diff --git a/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain.py b/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..f9d8b9f9432692979945e08cbae17a87f58213b3 --- /dev/null +++ b/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain.py @@ -0,0 +1,214 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from mmengine.dataset import DefaultSampler +from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook, + LoggerHook, ParamSchedulerHook) +from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR +from torch.optim import AdamW +from transformers import (AutoModelForCausalLM, AutoTokenizer, + BitsAndBytesConfig, CLIPImageProcessor, + CLIPVisionModel) + +from xtuner.dataset import LLaVADataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import llava_map_fn, template_map_fn_factory +from xtuner.engine.hooks import DatasetInfoHook, EvaluateChatHook +from xtuner.engine.runner import TrainLoop +from xtuner.model import LLaVAModel +from xtuner.utils import PROMPT_TEMPLATE + +####################################################################### +# PART 1 Settings # +####################################################################### +# Model +llm_name_or_path = 'internlm/internlm2-chat-1_8b' +visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336' + +# Data +data_root = './llava_data/' +data_path = data_root + 'LLaVA-Pretrain/blip_laion_cc_sbu_558k.json' +image_folder = data_root + 'LLaVA-Pretrain/images' +prompt_template = PROMPT_TEMPLATE.internlm2_chat +max_length = int(2048 - (336 / 14)**2) + +# Scheduler & Optimizer +batch_size = 40 # per_device +accumulative_counts = 7 +dataloader_num_workers = 2 +max_epochs = 1 +optim_type = AdamW +lr = 1e-3 +betas = (0.9, 0.999) +weight_decay = 0 +max_norm = 1 # grad clip +warmup_ratio = 0.03 + +# Save +save_steps = 500 +save_total_limit = 2 # Maximum checkpoints to keep (-1 means unlimited) + +# Evaluate the generation performance during the training +evaluation_freq = 500 +SYSTEM = '' +evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg' +evaluation_inputs = ['请描述一下这张照片', 'Please describe this picture'] + +####################################################################### +# PART 2 Model & Tokenizer & Image Processor # +####################################################################### +tokenizer = dict( + type=AutoTokenizer.from_pretrained, + pretrained_model_name_or_path=llm_name_or_path, + trust_remote_code=True, + padding_side='right') + +image_processor = dict( + type=CLIPImageProcessor.from_pretrained, + pretrained_model_name_or_path=visual_encoder_name_or_path, + trust_remote_code=True) + +model = dict( + type=LLaVAModel, + freeze_llm=True, + freeze_visual_encoder=True, + llm=dict( + type=AutoModelForCausalLM.from_pretrained, + pretrained_model_name_or_path=llm_name_or_path, + trust_remote_code=True, + torch_dtype=torch.float16, + quantization_config=dict( + type=BitsAndBytesConfig, + load_in_4bit=True, + load_in_8bit=False, + llm_int8_threshold=6.0, + llm_int8_has_fp16_weight=False, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4')), + visual_encoder=dict( + type=CLIPVisionModel.from_pretrained, + pretrained_model_name_or_path=visual_encoder_name_or_path)) + +####################################################################### +# PART 3 Dataset & Dataloader # +####################################################################### +llava_dataset = dict( + type=LLaVADataset, + data_path=data_path, + image_folder=image_folder, + tokenizer=tokenizer, + image_processor=image_processor, + dataset_map_fn=llava_map_fn, + template_map_fn=dict( + type=template_map_fn_factory, template=prompt_template), + max_length=max_length, + pad_image_to_square=False) + +train_dataloader = dict( + batch_size=batch_size, + num_workers=dataloader_num_workers, + dataset=llava_dataset, + sampler=dict(type=DefaultSampler, shuffle=True), + collate_fn=dict(type=default_collate_fn)) + +####################################################################### +# PART 4 Scheduler & Optimizer # +####################################################################### +# optimizer +optim_wrapper = dict( + type=AmpOptimWrapper, + optimizer=dict( + type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay), + clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False), + accumulative_counts=accumulative_counts, + loss_scale='dynamic', + dtype='float16') + +# learning policy +# More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md # noqa: E501 +param_scheduler = [ + dict( + type=LinearLR, + start_factor=1e-5, + by_epoch=True, + begin=0, + end=warmup_ratio * max_epochs, + convert_to_iter_based=True), + dict( + type=CosineAnnealingLR, + eta_min=0.0, + by_epoch=True, + begin=warmup_ratio * max_epochs, + end=max_epochs, + convert_to_iter_based=True) +] + +# train, val, test setting +train_cfg = dict(type=TrainLoop, max_epochs=max_epochs) + +####################################################################### +# PART 5 Runtime # +####################################################################### +# Log the dialogue periodically during the training process, optional +custom_hooks = [ + dict(type=DatasetInfoHook, tokenizer=tokenizer), + dict( + type=EvaluateChatHook, + tokenizer=tokenizer, + image_processor=image_processor, + every_n_iters=evaluation_freq, + evaluation_inputs=evaluation_inputs, + evaluation_images=evaluation_images, + system=SYSTEM, + prompt_template=prompt_template) +] + +# configure default hooks +default_hooks = dict( + # record the time of every iteration. + timer=dict(type=IterTimerHook), + # print log every 10 iterations. + logger=dict(type=LoggerHook, log_metric_by_epoch=False, interval=10), + # enable the parameter scheduler. + param_scheduler=dict(type=ParamSchedulerHook), + # save checkpoint per `save_steps`. + checkpoint=dict( + type=CheckpointHook, + by_epoch=False, + interval=save_steps, + max_keep_ckpts=save_total_limit), + # set sampler seed in distributed evrionment. + sampler_seed=dict(type=DistSamplerSeedHook), +) + +# configure environment +env_cfg = dict( + # whether to enable cudnn benchmark + cudnn_benchmark=False, + # set multi process parameters + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), + # set distributed parameters + dist_cfg=dict(backend='nccl'), +) + +# set visualizer +from mmengine.visualization import Visualizer, TensorboardVisBackend +visualizer = dict( + type=Visualizer, + vis_backends=[dict(type=TensorboardVisBackend)] +) + +# set log level +log_level = 'INFO' + +# load from which checkpoint +load_from = None + +# whether to resume training from the loaded checkpoint +resume = False + +# Defaults to use random seed and disable `deterministic` +randomness = dict(seed=None, deterministic=False) + +# set log processor +log_processor = dict(by_epoch=False) diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/20240220_050613.log b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/20240220_050613.log new file mode 100644 index 0000000000000000000000000000000000000000..ecc99455fad4ede67062bd53814c47470fc0af97 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/20240220_050613.log @@ -0,0 +1,2272 @@ +2024/02/20 05:06:14 - mmengine - INFO - +------------------------------------------------------------ +System environment: + sys.platform: linux + Python: 3.10.11 (main, Apr 20 2023, 19:02:41) [GCC 11.2.0] + CUDA available: True + MUSA available: False + numpy_random_seed: 575607930 + GPU 0: NVIDIA RTX 6000 Ada Generation + CUDA_HOME: /usr/local/cuda + NVCC: Cuda compilation tools, release 11.7, V11.7.99 + GCC: gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 + PyTorch: 2.0.1 + PyTorch compiling details: PyTorch built with: + - GCC 9.3 + - C++ Version: 201703 + - Intel(R) oneAPI Math Kernel Library Version 2023.1-Product Build 20230303 for Intel(R) 64 architecture applications + - Intel(R) MKL-DNN v2.7.3 (Git Hash 6dbeffbae1f23cbbeae17adb7b5b13f1f37c080e) + - OpenMP 201511 (a.k.a. OpenMP 4.5) + - LAPACK is enabled (usually provided by MKL) + - NNPACK is enabled + - CPU capability usage: AVX2 + - CUDA Runtime 11.7 + - NVCC architecture flags: -gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_37,code=compute_37 + - CuDNN 8.5 + - Magma 2.6.1 + - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.7, CUDNN_VERSION=8.5.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -Wno-deprecated -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wunused-local-typedefs -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-error=deprecated-declarations -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=redundant-decls -Wno-error=old-style-cast -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_DISABLE_GPU_ASSERTS=ON, TORCH_VERSION=2.0.1, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, + + TorchVision: 0.15.2 + OpenCV: 4.9.0 + MMEngine: 0.10.3 + +Runtime environment: + launcher: none + randomness: {'seed': None, 'deterministic': False} + cudnn_benchmark: False + mp_cfg: {'mp_start_method': 'fork', 'opencv_num_threads': 0} + dist_cfg: {'backend': 'nccl'} + seed: None + deterministic: False + Distributed launcher: none + Distributed training: False + GPU number: 1 +------------------------------------------------------------ + +2024/02/20 05:06:14 - mmengine - INFO - Config: +SYSTEM = '' +accumulative_counts = 7 +batch_size = 40 +betas = ( + 0.9, + 0.999, +) +custom_hooks = [ + dict( + tokenizer=dict( + padding_side='right', + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + trust_remote_code=True, + type='transformers.AutoTokenizer.from_pretrained'), + type='xtuner.engine.hooks.DatasetInfoHook'), + dict( + evaluation_images='https://llava-vl.github.io/static/images/view.jpg', + evaluation_inputs=[ + '请描述一下这张照片', + 'Please describe this picture', + ], + every_n_iters=500, + image_processor=dict( + pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', + trust_remote_code=True, + type='transformers.CLIPImageProcessor.from_pretrained'), + prompt_template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat', + system='', + tokenizer=dict( + padding_side='right', + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + trust_remote_code=True, + type='transformers.AutoTokenizer.from_pretrained'), + type='xtuner.engine.hooks.EvaluateChatHook'), +] +data_path = './llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json' +data_root = './llava_data/' +dataloader_num_workers = 2 +default_hooks = dict( + checkpoint=dict( + by_epoch=False, + interval=500, + max_keep_ckpts=2, + type='mmengine.hooks.CheckpointHook'), + logger=dict( + interval=10, + log_metric_by_epoch=False, + type='mmengine.hooks.LoggerHook'), + param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'), + sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'), + timer=dict(type='mmengine.hooks.IterTimerHook')) +env_cfg = dict( + cudnn_benchmark=False, + dist_cfg=dict(backend='nccl'), + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0)) +evaluation_freq = 500 +evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg' +evaluation_inputs = [ + '请描述一下这张照片', + 'Please describe this picture', +] +image_folder = './llava_data/LLaVA-Pretrain/images' +image_processor = dict( + pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', + trust_remote_code=True, + type='transformers.CLIPImageProcessor.from_pretrained') +launcher = 'none' +llava_dataset = dict( + data_path='./llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json', + dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', + image_folder='./llava_data/LLaVA-Pretrain/images', + image_processor=dict( + pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', + trust_remote_code=True, + type='transformers.CLIPImageProcessor.from_pretrained'), + max_length=1472, + pad_image_to_square=False, + template_map_fn=dict( + template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat', + type='xtuner.dataset.map_fns.template_map_fn_factory'), + tokenizer=dict( + padding_side='right', + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + trust_remote_code=True, + type='transformers.AutoTokenizer.from_pretrained'), + type='xtuner.dataset.LLaVADataset') +llm_name_or_path = 'internlm/internlm2-chat-1_8b' +load_from = None +log_level = 'INFO' +log_processor = dict(by_epoch=False) +lr = 0.001 +max_epochs = 1 +max_length = 1472 +max_norm = 1 +model = dict( + freeze_llm=True, + freeze_visual_encoder=True, + llm=dict( + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + quantization_config=dict( + bnb_4bit_compute_dtype='torch.float16', + bnb_4bit_quant_type='nf4', + bnb_4bit_use_double_quant=True, + llm_int8_has_fp16_weight=False, + llm_int8_threshold=6.0, + load_in_4bit=True, + load_in_8bit=False, + type='transformers.BitsAndBytesConfig'), + torch_dtype='torch.float16', + trust_remote_code=True, + type='transformers.AutoModelForCausalLM.from_pretrained'), + type='xtuner.model.LLaVAModel', + visual_encoder=dict( + pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', + type='transformers.CLIPVisionModel.from_pretrained')) +optim_type = 'torch.optim.AdamW' +optim_wrapper = dict( + optimizer=dict( + betas=( + 0.9, + 0.999, + ), + lr=0.001, + type='torch.optim.AdamW', + weight_decay=0), + type='DeepSpeedOptimWrapper') +param_scheduler = [ + dict( + begin=0, + by_epoch=True, + convert_to_iter_based=True, + end=0.03, + start_factor=1e-05, + type='mmengine.optim.LinearLR'), + dict( + begin=0.03, + by_epoch=True, + convert_to_iter_based=True, + end=1, + eta_min=0.0, + type='mmengine.optim.CosineAnnealingLR'), +] +prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat' +randomness = dict(deterministic=False, seed=None) +resume = False +runner_type = 'FlexibleRunner' +save_steps = 500 +save_total_limit = 2 +strategy = dict( + config=dict( + bf16=dict(enabled=True), + fp16=dict(enabled=False, initial_scale_power=16), + gradient_accumulation_steps='auto', + gradient_clipping='auto', + train_micro_batch_size_per_gpu='auto', + zero_allow_untested_optimizer=True, + zero_force_ds_cpu_optimizer=False, + zero_optimization=dict(overlap_comm=True, stage=2)), + exclude_frozen_parameters=True, + gradient_accumulation_steps=7, + gradient_clipping=1, + train_micro_batch_size_per_gpu=40, + type='xtuner.engine.DeepSpeedStrategy') +tokenizer = dict( + padding_side='right', + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + trust_remote_code=True, + type='transformers.AutoTokenizer.from_pretrained') +train_cfg = dict(max_epochs=1, type='xtuner.engine.runner.TrainLoop') +train_dataloader = dict( + batch_size=40, + collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'), + dataset=dict( + data_path='./llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json', + dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', + image_folder='./llava_data/LLaVA-Pretrain/images', + image_processor=dict( + pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', + trust_remote_code=True, + type='transformers.CLIPImageProcessor.from_pretrained'), + max_length=1472, + pad_image_to_square=False, + template_map_fn=dict( + template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat', + type='xtuner.dataset.map_fns.template_map_fn_factory'), + tokenizer=dict( + padding_side='right', + pretrained_model_name_or_path='internlm/internlm2-chat-1_8b', + trust_remote_code=True, + type='transformers.AutoTokenizer.from_pretrained'), + type='xtuner.dataset.LLaVADataset'), + num_workers=2, + sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler')) +visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336' +visualizer = dict( + type='mmengine.visualization.Visualizer', + vis_backends=[ + dict(type='mmengine.visualization.TensorboardVisBackend'), + ]) +warmup_ratio = 0.03 +weight_decay = 0 +work_dir = './work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy' + +2024/02/20 05:06:14 - mmengine - WARNING - Failed to search registry with scope "mmengine" in the "builder" registry tree. As a workaround, the current "builder" registry in "xtuner" is used to build instance. This may cause unexpected failure when running the built modules. Please check whether "mmengine" is a correct scope, or whether the registry is initialized. +2024/02/20 05:06:17 - mmengine - INFO - Hooks will be executed in the following order: +before_run: +(VERY_HIGH ) RuntimeInfoHook +(BELOW_NORMAL) LoggerHook + -------------------- +before_train: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook +(NORMAL ) DatasetInfoHook +(LOW ) EvaluateChatHook +(VERY_LOW ) CheckpointHook + -------------------- +before_train_epoch: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook +(NORMAL ) DistSamplerSeedHook + -------------------- +before_train_iter: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook + -------------------- +after_train_iter: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook +(BELOW_NORMAL) LoggerHook +(LOW ) ParamSchedulerHook +(LOW ) EvaluateChatHook +(VERY_LOW ) CheckpointHook + -------------------- +after_train_epoch: +(NORMAL ) IterTimerHook +(LOW ) ParamSchedulerHook +(VERY_LOW ) CheckpointHook + -------------------- +before_val: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) DatasetInfoHook + -------------------- +before_val_epoch: +(NORMAL ) IterTimerHook + -------------------- +before_val_iter: +(NORMAL ) IterTimerHook + -------------------- +after_val_iter: +(NORMAL ) IterTimerHook +(BELOW_NORMAL) LoggerHook + -------------------- +after_val_epoch: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook +(BELOW_NORMAL) LoggerHook +(LOW ) ParamSchedulerHook +(VERY_LOW ) CheckpointHook + -------------------- +after_val: +(VERY_HIGH ) RuntimeInfoHook +(LOW ) EvaluateChatHook + -------------------- +after_train: +(VERY_HIGH ) RuntimeInfoHook +(LOW ) EvaluateChatHook +(VERY_LOW ) CheckpointHook + -------------------- +before_test: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) DatasetInfoHook + -------------------- +before_test_epoch: +(NORMAL ) IterTimerHook + -------------------- +before_test_iter: +(NORMAL ) IterTimerHook + -------------------- +after_test_iter: +(NORMAL ) IterTimerHook +(BELOW_NORMAL) LoggerHook + -------------------- +after_test_epoch: +(VERY_HIGH ) RuntimeInfoHook +(NORMAL ) IterTimerHook +(BELOW_NORMAL) LoggerHook + -------------------- +after_test: +(VERY_HIGH ) RuntimeInfoHook + -------------------- +after_run: +(BELOW_NORMAL) LoggerHook + -------------------- +2024/02/20 05:11:10 - mmengine - WARNING - Dataset LLaVADataset has no metainfo. ``dataset_meta`` in visualizer will be None. +2024/02/20 05:11:17 - mmengine - WARNING - Due to the implementation of the PyTorch version of flash attention, even when the `output_attentions` flag is set to True, it is not possible to return the `attn_weights`. +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - dispatch internlm2 attn forward +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:17 - mmengine - INFO - replace internlm2 rope +2024/02/20 05:11:19 - mmengine - INFO - Num train samples 558128 +2024/02/20 05:11:19 - mmengine - INFO - train example: +2024/02/20 05:11:19 - mmengine - INFO - <|im_start|>user + +Render a clear and concise summary of the photo.<|im_end|> +<|im_start|>assistant +select luxury furniture 3 - inch gel memory foam mattress topper<|im_end|> + +2024/02/20 05:11:19 - mmengine - INFO - before_train in EvaluateChatHook. +2024/02/20 05:11:19 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +<|im_end|> + +2024/02/20 05:11:19 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +<|im_end|> + +2024/02/20 05:11:19 - mmengine - WARNING - "FileClient" will be deprecated in future. Please use io functions in https://mmengine.readthedocs.io/en/latest/api/fileio.html#file-io +2024/02/20 05:11:19 - mmengine - WARNING - "HardDiskBackend" is the alias of "LocalBackend" and the former will be deprecated in future. +2024/02/20 05:11:19 - mmengine - INFO - Checkpoints will be saved to /workspace/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy. +2024/02/20 05:11:59 - mmengine - INFO - Iter(train) [ 10/13954] lr: 2.1593e-05 eta: 15:27:20 time: 3.9903 data_time: 0.0127 memory: 34632 loss: 11.3723 +2024/02/20 05:12:41 - mmengine - INFO - Iter(train) [ 20/13954] lr: 4.5573e-05 eta: 15:48:19 time: 4.1766 data_time: 0.0142 memory: 34588 loss: 10.2403 +2024/02/20 05:13:24 - mmengine - INFO - Iter(train) [ 30/13954] lr: 6.9554e-05 eta: 16:06:25 time: 4.3264 data_time: 0.0134 memory: 34727 loss: 9.0815 +2024/02/20 05:14:08 - mmengine - INFO - Iter(train) [ 40/13954] lr: 9.3534e-05 eta: 16:15:54 time: 4.3400 data_time: 0.0139 memory: 34821 loss: 8.6593 +2024/02/20 05:14:51 - mmengine - INFO - Iter(train) [ 50/13954] lr: 1.1751e-04 eta: 16:21:57 time: 4.3538 data_time: 0.0143 memory: 35523 loss: 8.3129 +2024/02/20 05:15:34 - mmengine - INFO - Iter(train) [ 60/13954] lr: 1.4150e-04 eta: 16:23:27 time: 4.2948 data_time: 0.0145 memory: 34680 loss: 8.2237 +2024/02/20 05:16:18 - mmengine - INFO - Iter(train) [ 70/13954] lr: 1.6548e-04 eta: 16:26:04 time: 4.3473 data_time: 0.0134 memory: 34821 loss: 7.9252 +2024/02/20 05:17:01 - mmengine - INFO - Iter(train) [ 80/13954] lr: 1.8946e-04 eta: 16:27:06 time: 4.3218 data_time: 0.0133 memory: 34727 loss: 7.5468 +2024/02/20 05:17:44 - mmengine - INFO - Iter(train) [ 90/13954] lr: 2.1344e-04 eta: 16:28:06 time: 4.3359 data_time: 0.0133 memory: 35008 loss: 6.2603 +2024/02/20 05:18:28 - mmengine - INFO - Iter(train) [ 100/13954] lr: 2.3742e-04 eta: 16:28:26 time: 4.3210 data_time: 0.0136 memory: 34727 loss: 5.2772 +2024/02/20 05:19:11 - mmengine - INFO - Iter(train) [ 110/13954] lr: 2.6140e-04 eta: 16:28:17 time: 4.3077 data_time: 0.0142 memory: 34727 loss: 4.9775 +2024/02/20 05:19:54 - mmengine - INFO - Iter(train) [ 120/13954] lr: 2.8538e-04 eta: 16:28:08 time: 4.3131 data_time: 0.0140 memory: 34715 loss: 4.7079 +2024/02/20 05:20:37 - mmengine - INFO - Iter(train) [ 130/13954] lr: 3.0936e-04 eta: 16:27:41 time: 4.3008 data_time: 0.0142 memory: 34774 loss: 4.5161 +2024/02/20 05:21:20 - mmengine - INFO - Iter(train) [ 140/13954] lr: 3.3334e-04 eta: 16:27:22 time: 4.3104 data_time: 0.0138 memory: 34680 loss: 4.4002 +2024/02/20 05:22:03 - mmengine - INFO - Iter(train) [ 150/13954] lr: 3.5732e-04 eta: 16:27:00 time: 4.3111 data_time: 0.0140 memory: 34492 loss: 4.4157 +2024/02/20 05:22:46 - mmengine - INFO - Iter(train) [ 160/13954] lr: 3.8130e-04 eta: 16:26:38 time: 4.3144 data_time: 0.0141 memory: 34621 loss: 4.3606 +2024/02/20 05:23:29 - mmengine - INFO - Iter(train) [ 170/13954] lr: 4.0528e-04 eta: 16:26:15 time: 4.3169 data_time: 0.0140 memory: 34903 loss: 4.1524 +2024/02/20 05:24:12 - mmengine - INFO - Iter(train) [ 180/13954] lr: 4.2926e-04 eta: 16:25:49 time: 4.3149 data_time: 0.0134 memory: 34774 loss: 4.1915 +2024/02/20 05:24:56 - mmengine - INFO - Iter(train) [ 190/13954] lr: 4.5324e-04 eta: 16:25:29 time: 4.3266 data_time: 0.0132 memory: 34774 loss: 4.3322 +2024/02/20 05:25:39 - mmengine - INFO - Iter(train) [ 200/13954] lr: 4.7722e-04 eta: 16:25:10 time: 4.3303 data_time: 0.0135 memory: 34821 loss: 4.2665 +2024/02/20 05:26:22 - mmengine - INFO - Iter(train) [ 210/13954] lr: 5.0120e-04 eta: 16:24:27 time: 4.2978 data_time: 0.0133 memory: 34680 loss: 4.0792 +2024/02/20 05:27:05 - mmengine - INFO - Iter(train) [ 220/13954] lr: 5.2518e-04 eta: 16:23:55 time: 4.3148 data_time: 0.0134 memory: 34773 loss: 4.0092 +2024/02/20 05:27:48 - mmengine - INFO - Iter(train) [ 230/13954] lr: 5.4917e-04 eta: 16:23:16 time: 4.3054 data_time: 0.0136 memory: 34773 loss: 4.0182 +2024/02/20 05:28:31 - mmengine - INFO - Iter(train) [ 240/13954] lr: 5.7315e-04 eta: 16:22:39 time: 4.3090 data_time: 0.0140 memory: 34727 loss: 4.0364 +2024/02/20 05:29:15 - mmengine - INFO - Iter(train) [ 250/13954] lr: 5.9713e-04 eta: 16:22:07 time: 4.3203 data_time: 0.0142 memory: 34856 loss: 3.9323 +2024/02/20 05:29:58 - mmengine - INFO - Iter(train) [ 260/13954] lr: 6.2111e-04 eta: 16:21:37 time: 4.3240 data_time: 0.0140 memory: 34774 loss: 3.8941 +2024/02/20 05:30:41 - mmengine - INFO - Iter(train) [ 270/13954] lr: 6.4509e-04 eta: 16:21:02 time: 4.3172 data_time: 0.0140 memory: 34868 loss: 3.8982 +2024/02/20 05:31:24 - mmengine - INFO - Iter(train) [ 280/13954] lr: 6.6907e-04 eta: 16:20:28 time: 4.3198 data_time: 0.0141 memory: 34868 loss: 3.8423 +2024/02/20 05:32:07 - mmengine - INFO - Iter(train) [ 290/13954] lr: 6.9305e-04 eta: 16:19:58 time: 4.3294 data_time: 0.0140 memory: 34727 loss: 3.8456 +2024/02/20 05:32:51 - mmengine - INFO - Iter(train) [ 300/13954] lr: 7.1703e-04 eta: 16:19:21 time: 4.3165 data_time: 0.0140 memory: 34868 loss: 3.7224 +2024/02/20 05:33:34 - mmengine - INFO - Iter(train) [ 310/13954] lr: 7.4101e-04 eta: 16:18:55 time: 4.3432 data_time: 0.0142 memory: 34727 loss: 3.7795 +2024/02/20 05:34:17 - mmengine - INFO - Iter(train) [ 320/13954] lr: 7.6499e-04 eta: 16:18:29 time: 4.3429 data_time: 0.0142 memory: 34727 loss: 3.6941 +2024/02/20 05:35:01 - mmengine - INFO - Iter(train) [ 330/13954] lr: 7.8897e-04 eta: 16:18:09 time: 4.3641 data_time: 0.0141 memory: 34868 loss: 3.6143 +2024/02/20 05:35:45 - mmengine - INFO - Iter(train) [ 340/13954] lr: 8.1295e-04 eta: 16:17:41 time: 4.3448 data_time: 0.0140 memory: 34633 loss: 3.6963 +2024/02/20 05:36:28 - mmengine - INFO - Iter(train) [ 350/13954] lr: 8.3693e-04 eta: 16:17:09 time: 4.3370 data_time: 0.0142 memory: 34727 loss: 3.6802 +2024/02/20 05:37:11 - mmengine - INFO - Iter(train) [ 360/13954] lr: 8.6091e-04 eta: 16:16:30 time: 4.3207 data_time: 0.0126 memory: 34588 loss: 3.6714 +2024/02/20 05:37:55 - mmengine - INFO - Iter(train) [ 370/13954] lr: 8.8489e-04 eta: 16:16:04 time: 4.3551 data_time: 0.0114 memory: 34915 loss: 3.6867 +2024/02/20 05:38:38 - mmengine - INFO - Iter(train) [ 380/13954] lr: 9.0887e-04 eta: 16:15:27 time: 4.3288 data_time: 0.0114 memory: 34622 loss: 3.5501 +2024/02/20 05:39:22 - mmengine - INFO - Iter(train) [ 390/13954] lr: 9.3285e-04 eta: 16:15:21 time: 4.4191 data_time: 0.0116 memory: 36789 loss: 3.5462 +2024/02/20 05:40:06 - mmengine - INFO - Iter(train) [ 400/13954] lr: 9.5683e-04 eta: 16:14:48 time: 4.3461 data_time: 0.0120 memory: 34668 loss: 3.5635 +2024/02/20 05:40:49 - mmengine - INFO - Iter(train) [ 410/13954] lr: 9.8082e-04 eta: 16:14:11 time: 4.3315 data_time: 0.0116 memory: 34727 loss: 3.5309 +2024/02/20 05:41:32 - mmengine - INFO - Iter(train) [ 420/13954] lr: 1.0000e-03 eta: 16:13:35 time: 4.3400 data_time: 0.0118 memory: 34915 loss: 3.5474 +2024/02/20 05:42:16 - mmengine - INFO - Iter(train) [ 430/13954] lr: 1.0000e-03 eta: 16:12:56 time: 4.3290 data_time: 0.0117 memory: 34915 loss: 3.5750 +2024/02/20 05:42:59 - mmengine - INFO - Iter(train) [ 440/13954] lr: 9.9999e-04 eta: 16:12:18 time: 4.3311 data_time: 0.0119 memory: 34634 loss: 3.5035 +2024/02/20 05:43:42 - mmengine - INFO - Iter(train) [ 450/13954] lr: 9.9999e-04 eta: 16:11:47 time: 4.3589 data_time: 0.0115 memory: 34915 loss: 3.4649 +2024/02/20 05:44:26 - mmengine - INFO - Iter(train) [ 460/13954] lr: 9.9998e-04 eta: 16:11:15 time: 4.3557 data_time: 0.0117 memory: 34540 loss: 3.5596 +2024/02/20 05:45:09 - mmengine - INFO - Iter(train) [ 470/13954] lr: 9.9996e-04 eta: 16:10:38 time: 4.3400 data_time: 0.0118 memory: 34869 loss: 3.3992 +2024/02/20 05:45:53 - mmengine - INFO - Iter(train) [ 480/13954] lr: 9.9995e-04 eta: 16:10:00 time: 4.3371 data_time: 0.0119 memory: 34634 loss: 3.4535 +2024/02/20 05:46:36 - mmengine - INFO - Iter(train) [ 490/13954] lr: 9.9993e-04 eta: 16:09:27 time: 4.3563 data_time: 0.0118 memory: 34727 loss: 3.3685 +2024/02/20 05:47:20 - mmengine - INFO - Iter(train) [ 500/13954] lr: 9.9991e-04 eta: 16:08:54 time: 4.3591 data_time: 0.0117 memory: 34915 loss: 3.4570 +2024/02/20 05:47:20 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 05:47:20 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a wooden bridge in the mountains with a view of the lake<|im_end|> + +2024/02/20 05:47:21 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a wooden bridge in the mountains with a view of the water<|im_end|> + +2024/02/20 05:47:21 - mmengine - INFO - Saving checkpoint at 500 iterations +2024/02/20 05:48:04 - mmengine - INFO - Iter(train) [ 510/13954] lr: 9.9989e-04 eta: 16:08:41 time: 4.4343 data_time: 0.1079 memory: 34634 loss: 3.4298 +2024/02/20 05:48:48 - mmengine - INFO - Iter(train) [ 520/13954] lr: 9.9986e-04 eta: 16:08:04 time: 4.3493 data_time: 0.0125 memory: 34577 loss: 3.3673 +2024/02/20 05:49:31 - mmengine - INFO - Iter(train) [ 530/13954] lr: 9.9983e-04 eta: 16:07:24 time: 4.3357 data_time: 0.0118 memory: 34680 loss: 3.3873 +2024/02/20 05:50:15 - mmengine - INFO - Iter(train) [ 540/13954] lr: 9.9980e-04 eta: 16:06:45 time: 4.3392 data_time: 0.0121 memory: 34762 loss: 3.4219 +2024/02/20 05:50:58 - mmengine - INFO - Iter(train) [ 550/13954] lr: 9.9977e-04 eta: 16:06:04 time: 4.3346 data_time: 0.0118 memory: 34727 loss: 3.3686 +2024/02/20 05:51:42 - mmengine - INFO - Iter(train) [ 560/13954] lr: 9.9973e-04 eta: 16:05:30 time: 4.3650 data_time: 0.0120 memory: 34633 loss: 3.3303 +2024/02/20 05:52:25 - mmengine - INFO - Iter(train) [ 570/13954] lr: 9.9969e-04 eta: 16:04:59 time: 4.3747 data_time: 0.0119 memory: 34634 loss: 3.3739 +2024/02/20 05:53:09 - mmengine - INFO - Iter(train) [ 580/13954] lr: 9.9965e-04 eta: 16:04:25 time: 4.3683 data_time: 0.0130 memory: 34715 loss: 3.3700 +2024/02/20 05:53:53 - mmengine - INFO - Iter(train) [ 590/13954] lr: 9.9961e-04 eta: 16:03:50 time: 4.3632 data_time: 0.0139 memory: 34903 loss: 3.1969 +2024/02/20 05:54:36 - mmengine - INFO - Iter(train) [ 600/13954] lr: 9.9956e-04 eta: 16:03:08 time: 4.3309 data_time: 0.0143 memory: 34774 loss: 3.3398 +2024/02/20 05:55:19 - mmengine - INFO - Iter(train) [ 610/13954] lr: 9.9951e-04 eta: 16:02:27 time: 4.3410 data_time: 0.0134 memory: 34915 loss: 3.4129 +2024/02/20 05:56:03 - mmengine - INFO - Iter(train) [ 620/13954] lr: 9.9946e-04 eta: 16:01:48 time: 4.3436 data_time: 0.0130 memory: 34822 loss: 3.3868 +2024/02/20 05:56:46 - mmengine - INFO - Iter(train) [ 630/13954] lr: 9.9940e-04 eta: 16:01:02 time: 4.3163 data_time: 0.0130 memory: 34680 loss: 3.2215 +2024/02/20 05:57:29 - mmengine - INFO - Iter(train) [ 640/13954] lr: 9.9934e-04 eta: 16:00:15 time: 4.3085 data_time: 0.0130 memory: 34774 loss: 3.2632 +2024/02/20 05:58:12 - mmengine - INFO - Iter(train) [ 650/13954] lr: 9.9928e-04 eta: 15:59:25 time: 4.2981 data_time: 0.0125 memory: 34869 loss: 3.2154 +2024/02/20 05:58:55 - mmengine - INFO - Iter(train) [ 660/13954] lr: 9.9922e-04 eta: 15:58:42 time: 4.3259 data_time: 0.0118 memory: 34715 loss: 3.2602 +2024/02/20 05:59:39 - mmengine - INFO - Iter(train) [ 670/13954] lr: 9.9915e-04 eta: 15:57:59 time: 4.3300 data_time: 0.0117 memory: 34822 loss: 3.2396 +2024/02/20 06:00:22 - mmengine - INFO - Iter(train) [ 680/13954] lr: 9.9908e-04 eta: 15:57:15 time: 4.3216 data_time: 0.0114 memory: 34668 loss: 3.1923 +2024/02/20 06:01:05 - mmengine - INFO - Iter(train) [ 690/13954] lr: 9.9901e-04 eta: 15:56:34 time: 4.3386 data_time: 0.0139 memory: 34810 loss: 3.2882 +2024/02/20 06:01:48 - mmengine - INFO - Iter(train) [ 700/13954] lr: 9.9894e-04 eta: 15:55:49 time: 4.3198 data_time: 0.0137 memory: 34634 loss: 3.3043 +2024/02/20 06:02:31 - mmengine - INFO - Iter(train) [ 710/13954] lr: 9.9886e-04 eta: 15:55:03 time: 4.3124 data_time: 0.0138 memory: 34680 loss: 3.1727 +2024/02/20 06:03:15 - mmengine - INFO - Iter(train) [ 720/13954] lr: 9.9878e-04 eta: 15:54:19 time: 4.3203 data_time: 0.0119 memory: 35196 loss: 3.2970 +2024/02/20 06:03:58 - mmengine - INFO - Iter(train) [ 730/13954] lr: 9.9870e-04 eta: 15:53:35 time: 4.3253 data_time: 0.0116 memory: 34868 loss: 3.2582 +2024/02/20 06:04:41 - mmengine - INFO - Iter(train) [ 740/13954] lr: 9.9861e-04 eta: 15:52:48 time: 4.3062 data_time: 0.0119 memory: 34589 loss: 3.2044 +2024/02/20 06:05:24 - mmengine - INFO - Iter(train) [ 750/13954] lr: 9.9853e-04 eta: 15:52:07 time: 4.3364 data_time: 0.0118 memory: 34727 loss: 3.2647 +2024/02/20 06:06:08 - mmengine - INFO - Iter(train) [ 760/13954] lr: 9.9843e-04 eta: 15:51:21 time: 4.3133 data_time: 0.0118 memory: 34589 loss: 3.2080 +2024/02/20 06:06:51 - mmengine - INFO - Iter(train) [ 770/13954] lr: 9.9834e-04 eta: 15:50:39 time: 4.3338 data_time: 0.0118 memory: 34634 loss: 3.2164 +2024/02/20 06:07:34 - mmengine - INFO - Iter(train) [ 780/13954] lr: 9.9825e-04 eta: 15:49:55 time: 4.3206 data_time: 0.0120 memory: 34762 loss: 3.1150 +2024/02/20 06:08:17 - mmengine - INFO - Iter(train) [ 790/13954] lr: 9.9815e-04 eta: 15:49:12 time: 4.3253 data_time: 0.0118 memory: 35056 loss: 3.1560 +2024/02/20 06:09:01 - mmengine - INFO - Iter(train) [ 800/13954] lr: 9.9805e-04 eta: 15:48:30 time: 4.3360 data_time: 0.0121 memory: 34774 loss: 3.1612 +2024/02/20 06:09:44 - mmengine - INFO - Iter(train) [ 810/13954] lr: 9.9794e-04 eta: 15:47:48 time: 4.3339 data_time: 0.0120 memory: 34727 loss: 3.1057 +2024/02/20 06:10:27 - mmengine - INFO - Iter(train) [ 820/13954] lr: 9.9784e-04 eta: 15:47:03 time: 4.3159 data_time: 0.0120 memory: 34822 loss: 3.1140 +2024/02/20 06:11:11 - mmengine - INFO - Iter(train) [ 830/13954] lr: 9.9773e-04 eta: 15:46:21 time: 4.3346 data_time: 0.0123 memory: 34821 loss: 3.1542 +2024/02/20 06:11:54 - mmengine - INFO - Iter(train) [ 840/13954] lr: 9.9762e-04 eta: 15:45:37 time: 4.3217 data_time: 0.0121 memory: 34680 loss: 3.1597 +2024/02/20 06:12:37 - mmengine - INFO - Iter(train) [ 850/13954] lr: 9.9750e-04 eta: 15:44:57 time: 4.3490 data_time: 0.0122 memory: 34774 loss: 3.1972 +2024/02/20 06:13:21 - mmengine - INFO - Iter(train) [ 860/13954] lr: 9.9738e-04 eta: 15:44:22 time: 4.3801 data_time: 0.0120 memory: 35899 loss: 3.1401 +2024/02/20 06:14:05 - mmengine - INFO - Iter(train) [ 870/13954] lr: 9.9726e-04 eta: 15:43:42 time: 4.3509 data_time: 0.0120 memory: 34903 loss: 3.1080 +2024/02/20 06:14:48 - mmengine - INFO - Iter(train) [ 880/13954] lr: 9.9714e-04 eta: 15:43:01 time: 4.3413 data_time: 0.0119 memory: 34589 loss: 3.2416 +2024/02/20 06:15:31 - mmengine - INFO - Iter(train) [ 890/13954] lr: 9.9702e-04 eta: 15:42:18 time: 4.3276 data_time: 0.0130 memory: 34589 loss: 3.2193 +2024/02/20 06:16:15 - mmengine - INFO - Iter(train) [ 900/13954] lr: 9.9689e-04 eta: 15:41:36 time: 4.3384 data_time: 0.0126 memory: 34668 loss: 3.1061 +2024/02/20 06:16:58 - mmengine - INFO - Iter(train) [ 910/13954] lr: 9.9676e-04 eta: 15:40:54 time: 4.3353 data_time: 0.0120 memory: 35009 loss: 3.1469 +2024/02/20 06:17:42 - mmengine - INFO - Iter(train) [ 920/13954] lr: 9.9662e-04 eta: 15:40:15 time: 4.3577 data_time: 0.0121 memory: 34634 loss: 3.2620 +2024/02/20 06:18:25 - mmengine - INFO - Iter(train) [ 930/13954] lr: 9.9649e-04 eta: 15:39:34 time: 4.3484 data_time: 0.0118 memory: 34868 loss: 3.1240 +2024/02/20 06:19:08 - mmengine - INFO - Iter(train) [ 940/13954] lr: 9.9635e-04 eta: 15:38:51 time: 4.3287 data_time: 0.0119 memory: 34633 loss: 3.2422 +2024/02/20 06:19:52 - mmengine - INFO - Iter(train) [ 950/13954] lr: 9.9621e-04 eta: 15:38:11 time: 4.3534 data_time: 0.0117 memory: 34962 loss: 3.1763 +2024/02/20 06:20:35 - mmengine - INFO - Iter(train) [ 960/13954] lr: 9.9606e-04 eta: 15:37:28 time: 4.3274 data_time: 0.0117 memory: 34762 loss: 3.0729 +2024/02/20 06:21:18 - mmengine - INFO - Iter(train) [ 970/13954] lr: 9.9592e-04 eta: 15:36:43 time: 4.3196 data_time: 0.0117 memory: 34633 loss: 3.0640 +2024/02/20 06:22:02 - mmengine - INFO - Iter(train) [ 980/13954] lr: 9.9577e-04 eta: 15:36:00 time: 4.3339 data_time: 0.0138 memory: 35102 loss: 3.1399 +2024/02/20 06:22:45 - mmengine - INFO - Iter(train) [ 990/13954] lr: 9.9562e-04 eta: 15:35:16 time: 4.3230 data_time: 0.0138 memory: 34589 loss: 3.1392 +2024/02/20 06:23:28 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 06:23:28 - mmengine - INFO - Iter(train) [ 1000/13954] lr: 9.9546e-04 eta: 15:34:33 time: 4.3305 data_time: 0.0140 memory: 34962 loss: 2.9948 +2024/02/20 06:23:28 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 06:23:29 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a mountain range in the background<|im_end|> + +2024/02/20 06:23:29 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with the sun setting over the water<|im_end|> + +2024/02/20 06:23:29 - mmengine - INFO - Saving checkpoint at 1000 iterations +2024/02/20 06:24:13 - mmengine - INFO - Iter(train) [ 1010/13954] lr: 9.9530e-04 eta: 15:34:07 time: 4.4573 data_time: 0.1139 memory: 34822 loss: 3.1105 +2024/02/20 06:24:56 - mmengine - INFO - Iter(train) [ 1020/13954] lr: 9.9514e-04 eta: 15:33:26 time: 4.3545 data_time: 0.0142 memory: 34821 loss: 3.1797 +2024/02/20 06:25:40 - mmengine - INFO - Iter(train) [ 1030/13954] lr: 9.9498e-04 eta: 15:32:44 time: 4.3413 data_time: 0.0136 memory: 34727 loss: 3.1558 +2024/02/20 06:26:23 - mmengine - INFO - Iter(train) [ 1040/13954] lr: 9.9482e-04 eta: 15:32:04 time: 4.3501 data_time: 0.0129 memory: 35055 loss: 3.1212 +2024/02/20 06:27:06 - mmengine - INFO - Iter(train) [ 1050/13954] lr: 9.9465e-04 eta: 15:31:18 time: 4.3123 data_time: 0.0133 memory: 34773 loss: 3.1397 +2024/02/20 06:27:50 - mmengine - INFO - Iter(train) [ 1060/13954] lr: 9.9448e-04 eta: 15:30:37 time: 4.3466 data_time: 0.0139 memory: 34589 loss: 3.0877 +2024/02/20 06:28:33 - mmengine - INFO - Iter(train) [ 1070/13954] lr: 9.9430e-04 eta: 15:29:53 time: 4.3232 data_time: 0.0137 memory: 35009 loss: 3.1737 +2024/02/20 06:29:16 - mmengine - INFO - Iter(train) [ 1080/13954] lr: 9.9413e-04 eta: 15:29:09 time: 4.3289 data_time: 0.0139 memory: 34822 loss: 3.1794 +2024/02/20 06:30:00 - mmengine - INFO - Iter(train) [ 1090/13954] lr: 9.9395e-04 eta: 15:28:25 time: 4.3243 data_time: 0.0140 memory: 34822 loss: 3.1427 +2024/02/20 06:30:43 - mmengine - INFO - Iter(train) [ 1100/13954] lr: 9.9377e-04 eta: 15:27:42 time: 4.3295 data_time: 0.0141 memory: 34634 loss: 3.0936 +2024/02/20 06:31:27 - mmengine - INFO - Iter(train) [ 1110/13954] lr: 9.9358e-04 eta: 15:27:06 time: 4.3994 data_time: 0.0142 memory: 36648 loss: 3.1285 +2024/02/20 06:32:10 - mmengine - INFO - Iter(train) [ 1120/13954] lr: 9.9340e-04 eta: 15:26:26 time: 4.3587 data_time: 0.0142 memory: 35757 loss: 3.0524 +2024/02/20 06:32:54 - mmengine - INFO - Iter(train) [ 1130/13954] lr: 9.9321e-04 eta: 15:25:41 time: 4.3152 data_time: 0.0141 memory: 34868 loss: 3.0691 +2024/02/20 06:33:37 - mmengine - INFO - Iter(train) [ 1140/13954] lr: 9.9302e-04 eta: 15:24:59 time: 4.3412 data_time: 0.0142 memory: 35289 loss: 3.0406 +2024/02/20 06:34:20 - mmengine - INFO - Iter(train) [ 1150/13954] lr: 9.9282e-04 eta: 15:24:14 time: 4.3174 data_time: 0.0135 memory: 34962 loss: 3.1032 +2024/02/20 06:35:04 - mmengine - INFO - Iter(train) [ 1160/13954] lr: 9.9262e-04 eta: 15:23:31 time: 4.3320 data_time: 0.0132 memory: 35336 loss: 3.0781 +2024/02/20 06:35:47 - mmengine - INFO - Iter(train) [ 1170/13954] lr: 9.9242e-04 eta: 15:22:48 time: 4.3333 data_time: 0.0130 memory: 34680 loss: 3.0852 +2024/02/20 06:36:30 - mmengine - INFO - Iter(train) [ 1180/13954] lr: 9.9222e-04 eta: 15:22:05 time: 4.3336 data_time: 0.0136 memory: 34950 loss: 2.9945 +2024/02/20 06:37:14 - mmengine - INFO - Iter(train) [ 1190/13954] lr: 9.9202e-04 eta: 15:21:25 time: 4.3606 data_time: 0.0137 memory: 36226 loss: 2.9895 +2024/02/20 06:37:57 - mmengine - INFO - Iter(train) [ 1200/13954] lr: 9.9181e-04 eta: 15:20:39 time: 4.3064 data_time: 0.0135 memory: 34680 loss: 3.0700 +2024/02/20 06:38:40 - mmengine - INFO - Iter(train) [ 1210/13954] lr: 9.9160e-04 eta: 15:19:55 time: 4.3316 data_time: 0.0129 memory: 34915 loss: 3.0873 +2024/02/20 06:39:23 - mmengine - INFO - Iter(train) [ 1220/13954] lr: 9.9138e-04 eta: 15:19:10 time: 4.3070 data_time: 0.0129 memory: 34821 loss: 3.1750 +2024/02/20 06:40:06 - mmengine - INFO - Iter(train) [ 1230/13954] lr: 9.9117e-04 eta: 15:18:24 time: 4.3123 data_time: 0.0124 memory: 34868 loss: 3.0912 +2024/02/20 06:40:50 - mmengine - INFO - Iter(train) [ 1240/13954] lr: 9.9095e-04 eta: 15:17:41 time: 4.3328 data_time: 0.0133 memory: 34680 loss: 3.0680 +2024/02/20 06:41:33 - mmengine - INFO - Iter(train) [ 1250/13954] lr: 9.9073e-04 eta: 15:16:59 time: 4.3439 data_time: 0.0127 memory: 36133 loss: 3.0496 +2024/02/20 06:42:16 - mmengine - INFO - Iter(train) [ 1260/13954] lr: 9.9051e-04 eta: 15:16:13 time: 4.3016 data_time: 0.0128 memory: 34634 loss: 3.1013 +2024/02/20 06:42:59 - mmengine - INFO - Iter(train) [ 1270/13954] lr: 9.9028e-04 eta: 15:15:29 time: 4.3257 data_time: 0.0124 memory: 34915 loss: 3.0769 +2024/02/20 06:43:43 - mmengine - INFO - Iter(train) [ 1280/13954] lr: 9.9005e-04 eta: 15:14:45 time: 4.3217 data_time: 0.0129 memory: 34680 loss: 3.0098 +2024/02/20 06:44:26 - mmengine - INFO - Iter(train) [ 1290/13954] lr: 9.8982e-04 eta: 15:14:02 time: 4.3305 data_time: 0.0126 memory: 34774 loss: 3.0806 +2024/02/20 06:45:09 - mmengine - INFO - Iter(train) [ 1300/13954] lr: 9.8958e-04 eta: 15:13:18 time: 4.3234 data_time: 0.0126 memory: 35102 loss: 3.0178 +2024/02/20 06:45:52 - mmengine - INFO - Iter(train) [ 1310/13954] lr: 9.8935e-04 eta: 15:12:33 time: 4.3197 data_time: 0.0127 memory: 34634 loss: 2.9542 +2024/02/20 06:46:36 - mmengine - INFO - Iter(train) [ 1320/13954] lr: 9.8911e-04 eta: 15:11:51 time: 4.3419 data_time: 0.0127 memory: 34681 loss: 3.0361 +2024/02/20 06:47:19 - mmengine - INFO - Iter(train) [ 1330/13954] lr: 9.8887e-04 eta: 15:11:08 time: 4.3306 data_time: 0.0126 memory: 34634 loss: 3.1690 +2024/02/20 06:48:02 - mmengine - INFO - Iter(train) [ 1340/13954] lr: 9.8862e-04 eta: 15:10:21 time: 4.2956 data_time: 0.0129 memory: 34774 loss: 3.0097 +2024/02/20 06:48:45 - mmengine - INFO - Iter(train) [ 1350/13954] lr: 9.8837e-04 eta: 15:09:38 time: 4.3266 data_time: 0.0130 memory: 34774 loss: 2.9924 +2024/02/20 06:49:28 - mmengine - INFO - Iter(train) [ 1360/13954] lr: 9.8812e-04 eta: 15:08:52 time: 4.3055 data_time: 0.0127 memory: 34727 loss: 2.9320 +2024/02/20 06:50:12 - mmengine - INFO - Iter(train) [ 1370/13954] lr: 9.8787e-04 eta: 15:08:09 time: 4.3362 data_time: 0.0132 memory: 34822 loss: 3.1567 +2024/02/20 06:50:55 - mmengine - INFO - Iter(train) [ 1380/13954] lr: 9.8761e-04 eta: 15:07:25 time: 4.3251 data_time: 0.0131 memory: 34680 loss: 3.0085 +2024/02/20 06:51:38 - mmengine - INFO - Iter(train) [ 1390/13954] lr: 9.8736e-04 eta: 15:06:41 time: 4.3219 data_time: 0.0130 memory: 34915 loss: 3.0918 +2024/02/20 06:52:21 - mmengine - INFO - Iter(train) [ 1400/13954] lr: 9.8710e-04 eta: 15:05:57 time: 4.3147 data_time: 0.0122 memory: 34774 loss: 3.0401 +2024/02/20 06:53:05 - mmengine - INFO - Iter(train) [ 1410/13954] lr: 9.8683e-04 eta: 15:05:13 time: 4.3295 data_time: 0.0120 memory: 35055 loss: 2.9394 +2024/02/20 06:53:48 - mmengine - INFO - Iter(train) [ 1420/13954] lr: 9.8657e-04 eta: 15:04:29 time: 4.3192 data_time: 0.0121 memory: 35196 loss: 3.0733 +2024/02/20 06:54:31 - mmengine - INFO - Iter(train) [ 1430/13954] lr: 9.8630e-04 eta: 15:03:46 time: 4.3252 data_time: 0.0121 memory: 34774 loss: 3.0413 +2024/02/20 06:55:14 - mmengine - INFO - Iter(train) [ 1440/13954] lr: 9.8603e-04 eta: 15:03:03 time: 4.3374 data_time: 0.0123 memory: 34727 loss: 3.0325 +2024/02/20 06:55:57 - mmengine - INFO - Iter(train) [ 1450/13954] lr: 9.8575e-04 eta: 15:02:17 time: 4.3056 data_time: 0.0121 memory: 34589 loss: 3.1449 +2024/02/20 06:56:41 - mmengine - INFO - Iter(train) [ 1460/13954] lr: 9.8548e-04 eta: 15:01:32 time: 4.3054 data_time: 0.0124 memory: 34540 loss: 2.9926 +2024/02/20 06:57:24 - mmengine - INFO - Iter(train) [ 1470/13954] lr: 9.8520e-04 eta: 15:00:49 time: 4.3288 data_time: 0.0123 memory: 34774 loss: 3.0185 +2024/02/20 06:58:07 - mmengine - INFO - Iter(train) [ 1480/13954] lr: 9.8492e-04 eta: 15:00:06 time: 4.3377 data_time: 0.0122 memory: 35009 loss: 3.0108 +2024/02/20 06:58:50 - mmengine - INFO - Iter(train) [ 1490/13954] lr: 9.8463e-04 eta: 14:59:23 time: 4.3268 data_time: 0.0123 memory: 34680 loss: 3.0263 +2024/02/20 06:59:34 - mmengine - INFO - Iter(train) [ 1500/13954] lr: 9.8435e-04 eta: 14:58:40 time: 4.3353 data_time: 0.0123 memory: 34727 loss: 3.0179 +2024/02/20 06:59:34 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 06:59:34 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a view of the water<|im_end|> + +2024/02/20 06:59:35 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a view of the water<|im_end|> + +2024/02/20 06:59:35 - mmengine - INFO - Saving checkpoint at 1500 iterations +2024/02/20 07:00:18 - mmengine - INFO - Iter(train) [ 1510/13954] lr: 9.8406e-04 eta: 14:58:02 time: 4.4007 data_time: 0.1066 memory: 35055 loss: 3.0855 +2024/02/20 07:01:01 - mmengine - INFO - Iter(train) [ 1520/13954] lr: 9.8376e-04 eta: 14:57:16 time: 4.2934 data_time: 0.0122 memory: 34680 loss: 3.0438 +2024/02/20 07:01:44 - mmengine - INFO - Iter(train) [ 1530/13954] lr: 9.8347e-04 eta: 14:56:31 time: 4.3131 data_time: 0.0124 memory: 34634 loss: 3.0363 +2024/02/20 07:02:27 - mmengine - INFO - Iter(train) [ 1540/13954] lr: 9.8317e-04 eta: 14:55:46 time: 4.3095 data_time: 0.0122 memory: 34727 loss: 3.0654 +2024/02/20 07:03:10 - mmengine - INFO - Iter(train) [ 1550/13954] lr: 9.8287e-04 eta: 14:55:02 time: 4.3196 data_time: 0.0123 memory: 34868 loss: 2.9905 +2024/02/20 07:03:53 - mmengine - INFO - Iter(train) [ 1560/13954] lr: 9.8257e-04 eta: 14:54:19 time: 4.3294 data_time: 0.0122 memory: 35324 loss: 3.0209 +2024/02/20 07:04:37 - mmengine - INFO - Iter(train) [ 1570/13954] lr: 9.8227e-04 eta: 14:53:34 time: 4.3106 data_time: 0.0122 memory: 34680 loss: 3.1030 +2024/02/20 07:05:20 - mmengine - INFO - Iter(train) [ 1580/13954] lr: 9.8196e-04 eta: 14:52:51 time: 4.3234 data_time: 0.0123 memory: 34821 loss: 2.9627 +2024/02/20 07:06:03 - mmengine - INFO - Iter(train) [ 1590/13954] lr: 9.8165e-04 eta: 14:52:06 time: 4.3089 data_time: 0.0123 memory: 34680 loss: 2.9637 +2024/02/20 07:06:46 - mmengine - INFO - Iter(train) [ 1600/13954] lr: 9.8133e-04 eta: 14:51:23 time: 4.3310 data_time: 0.0124 memory: 35102 loss: 3.0391 +2024/02/20 07:07:29 - mmengine - INFO - Iter(train) [ 1610/13954] lr: 9.8102e-04 eta: 14:50:38 time: 4.3183 data_time: 0.0125 memory: 34774 loss: 3.0120 +2024/02/20 07:08:13 - mmengine - INFO - Iter(train) [ 1620/13954] lr: 9.8070e-04 eta: 14:49:54 time: 4.3083 data_time: 0.0124 memory: 35710 loss: 3.0331 +2024/02/20 07:08:56 - mmengine - INFO - Iter(train) [ 1630/13954] lr: 9.8038e-04 eta: 14:49:10 time: 4.3289 data_time: 0.0124 memory: 34774 loss: 3.0144 +2024/02/20 07:09:39 - mmengine - INFO - Iter(train) [ 1640/13954] lr: 9.8006e-04 eta: 14:48:26 time: 4.3129 data_time: 0.0120 memory: 34822 loss: 3.0376 +2024/02/20 07:10:22 - mmengine - INFO - Iter(train) [ 1650/13954] lr: 9.7973e-04 eta: 14:47:41 time: 4.3163 data_time: 0.0119 memory: 34727 loss: 3.0952 +2024/02/20 07:11:05 - mmengine - INFO - Iter(train) [ 1660/13954] lr: 9.7940e-04 eta: 14:46:56 time: 4.3000 data_time: 0.0119 memory: 34915 loss: 3.0768 +2024/02/20 07:11:48 - mmengine - INFO - Iter(train) [ 1670/13954] lr: 9.7907e-04 eta: 14:46:12 time: 4.3196 data_time: 0.0119 memory: 34680 loss: 2.9219 +2024/02/20 07:12:31 - mmengine - INFO - Iter(train) [ 1680/13954] lr: 9.7874e-04 eta: 14:45:27 time: 4.3005 data_time: 0.0120 memory: 34589 loss: 3.0349 +2024/02/20 07:13:14 - mmengine - INFO - Iter(train) [ 1690/13954] lr: 9.7840e-04 eta: 14:44:42 time: 4.3099 data_time: 0.0118 memory: 34668 loss: 3.0135 +2024/02/20 07:13:58 - mmengine - INFO - Iter(train) [ 1700/13954] lr: 9.7806e-04 eta: 14:43:58 time: 4.3211 data_time: 0.0119 memory: 34634 loss: 3.0822 +2024/02/20 07:14:41 - mmengine - INFO - Iter(train) [ 1710/13954] lr: 9.7772e-04 eta: 14:43:14 time: 4.3100 data_time: 0.0120 memory: 34540 loss: 3.0527 +2024/02/20 07:15:24 - mmengine - INFO - Iter(train) [ 1720/13954] lr: 9.7738e-04 eta: 14:42:29 time: 4.3142 data_time: 0.0120 memory: 34668 loss: 3.0687 +2024/02/20 07:16:07 - mmengine - INFO - Iter(train) [ 1730/13954] lr: 9.7703e-04 eta: 14:41:45 time: 4.3089 data_time: 0.0120 memory: 34622 loss: 3.0644 +2024/02/20 07:16:50 - mmengine - INFO - Iter(train) [ 1740/13954] lr: 9.7668e-04 eta: 14:41:01 time: 4.3253 data_time: 0.0122 memory: 34822 loss: 3.0155 +2024/02/20 07:17:33 - mmengine - INFO - Iter(train) [ 1750/13954] lr: 9.7633e-04 eta: 14:40:16 time: 4.2979 data_time: 0.0119 memory: 34540 loss: 3.0006 +2024/02/20 07:18:16 - mmengine - INFO - Iter(train) [ 1760/13954] lr: 9.7598e-04 eta: 14:39:30 time: 4.2834 data_time: 0.0120 memory: 34680 loss: 3.0215 +2024/02/20 07:18:59 - mmengine - INFO - Iter(train) [ 1770/13954] lr: 9.7562e-04 eta: 14:38:45 time: 4.3058 data_time: 0.0119 memory: 34962 loss: 2.9528 +2024/02/20 07:19:42 - mmengine - INFO - Iter(train) [ 1780/13954] lr: 9.7526e-04 eta: 14:38:02 time: 4.3333 data_time: 0.0118 memory: 35009 loss: 2.9988 +2024/02/20 07:20:25 - mmengine - INFO - Iter(train) [ 1790/13954] lr: 9.7490e-04 eta: 14:37:17 time: 4.3077 data_time: 0.0121 memory: 34680 loss: 3.0618 +2024/02/20 07:21:09 - mmengine - INFO - Iter(train) [ 1800/13954] lr: 9.7454e-04 eta: 14:36:34 time: 4.3276 data_time: 0.0120 memory: 35149 loss: 3.0126 +2024/02/20 07:21:52 - mmengine - INFO - Iter(train) [ 1810/13954] lr: 9.7417e-04 eta: 14:35:50 time: 4.3128 data_time: 0.0120 memory: 34822 loss: 3.0116 +2024/02/20 07:22:35 - mmengine - INFO - Iter(train) [ 1820/13954] lr: 9.7380e-04 eta: 14:35:06 time: 4.3214 data_time: 0.0118 memory: 34727 loss: 2.9556 +2024/02/20 07:23:18 - mmengine - INFO - Iter(train) [ 1830/13954] lr: 9.7343e-04 eta: 14:34:23 time: 4.3323 data_time: 0.0119 memory: 34589 loss: 2.9793 +2024/02/20 07:24:02 - mmengine - INFO - Iter(train) [ 1840/13954] lr: 9.7305e-04 eta: 14:33:40 time: 4.3206 data_time: 0.0123 memory: 34634 loss: 3.0020 +2024/02/20 07:24:45 - mmengine - INFO - Iter(train) [ 1850/13954] lr: 9.7268e-04 eta: 14:32:55 time: 4.3137 data_time: 0.0122 memory: 34868 loss: 3.0062 +2024/02/20 07:25:28 - mmengine - INFO - Iter(train) [ 1860/13954] lr: 9.7230e-04 eta: 14:32:12 time: 4.3187 data_time: 0.0124 memory: 34962 loss: 3.0818 +2024/02/20 07:26:11 - mmengine - INFO - Iter(train) [ 1870/13954] lr: 9.7191e-04 eta: 14:31:27 time: 4.3133 data_time: 0.0123 memory: 34727 loss: 2.9318 +2024/02/20 07:26:54 - mmengine - INFO - Iter(train) [ 1880/13954] lr: 9.7153e-04 eta: 14:30:44 time: 4.3172 data_time: 0.0124 memory: 34680 loss: 2.9974 +2024/02/20 07:27:37 - mmengine - INFO - Iter(train) [ 1890/13954] lr: 9.7114e-04 eta: 14:29:59 time: 4.3105 data_time: 0.0119 memory: 34634 loss: 3.0604 +2024/02/20 07:28:20 - mmengine - INFO - Iter(train) [ 1900/13954] lr: 9.7075e-04 eta: 14:29:15 time: 4.3078 data_time: 0.0119 memory: 34680 loss: 3.0546 +2024/02/20 07:29:04 - mmengine - INFO - Iter(train) [ 1910/13954] lr: 9.7036e-04 eta: 14:28:31 time: 4.3244 data_time: 0.0118 memory: 34822 loss: 3.0557 +2024/02/20 07:29:47 - mmengine - INFO - Iter(train) [ 1920/13954] lr: 9.6997e-04 eta: 14:27:47 time: 4.3152 data_time: 0.0118 memory: 34774 loss: 3.0193 +2024/02/20 07:30:30 - mmengine - INFO - Iter(train) [ 1930/13954] lr: 9.6957e-04 eta: 14:27:04 time: 4.3183 data_time: 0.0119 memory: 34680 loss: 2.9848 +2024/02/20 07:31:13 - mmengine - INFO - Iter(train) [ 1940/13954] lr: 9.6917e-04 eta: 14:26:19 time: 4.3120 data_time: 0.0118 memory: 34589 loss: 3.0242 +2024/02/20 07:31:56 - mmengine - INFO - Iter(train) [ 1950/13954] lr: 9.6877e-04 eta: 14:25:36 time: 4.3287 data_time: 0.0119 memory: 34810 loss: 2.9558 +2024/02/20 07:32:40 - mmengine - INFO - Iter(train) [ 1960/13954] lr: 9.6836e-04 eta: 14:24:54 time: 4.3369 data_time: 0.0120 memory: 34856 loss: 3.0266 +2024/02/20 07:33:23 - mmengine - INFO - Iter(train) [ 1970/13954] lr: 9.6795e-04 eta: 14:24:10 time: 4.3263 data_time: 0.0118 memory: 34622 loss: 2.9902 +2024/02/20 07:34:06 - mmengine - INFO - Iter(train) [ 1980/13954] lr: 9.6754e-04 eta: 14:23:27 time: 4.3260 data_time: 0.0121 memory: 34680 loss: 2.9136 +2024/02/20 07:34:50 - mmengine - INFO - Iter(train) [ 1990/13954] lr: 9.6713e-04 eta: 14:22:44 time: 4.3277 data_time: 0.0125 memory: 34727 loss: 2.9777 +2024/02/20 07:35:33 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 07:35:33 - mmengine - INFO - Iter(train) [ 2000/13954] lr: 9.6672e-04 eta: 14:22:00 time: 4.3105 data_time: 0.0134 memory: 34774 loss: 2.9139 +2024/02/20 07:35:33 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 07:35:33 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a wooden pier on a lake in the mountains<|im_end|> + +2024/02/20 07:35:33 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a wooden pier with a boat on the water<|im_end|> + +2024/02/20 07:35:33 - mmengine - INFO - Saving checkpoint at 2000 iterations +2024/02/20 07:36:17 - mmengine - INFO - Iter(train) [ 2010/13954] lr: 9.6630e-04 eta: 14:21:21 time: 4.4110 data_time: 0.0873 memory: 35149 loss: 2.9832 +2024/02/20 07:37:00 - mmengine - INFO - Iter(train) [ 2020/13954] lr: 9.6588e-04 eta: 14:20:38 time: 4.3294 data_time: 0.0142 memory: 34856 loss: 2.9064 +2024/02/20 07:37:43 - mmengine - INFO - Iter(train) [ 2030/13954] lr: 9.6546e-04 eta: 14:19:55 time: 4.3220 data_time: 0.0136 memory: 34962 loss: 2.9600 +2024/02/20 07:38:27 - mmengine - INFO - Iter(train) [ 2040/13954] lr: 9.6503e-04 eta: 14:19:12 time: 4.3380 data_time: 0.0128 memory: 34774 loss: 2.9989 +2024/02/20 07:39:10 - mmengine - INFO - Iter(train) [ 2050/13954] lr: 9.6460e-04 eta: 14:18:28 time: 4.3068 data_time: 0.0122 memory: 34577 loss: 3.0155 +2024/02/20 07:39:53 - mmengine - INFO - Iter(train) [ 2060/13954] lr: 9.6417e-04 eta: 14:17:44 time: 4.3300 data_time: 0.0117 memory: 34622 loss: 2.8779 +2024/02/20 07:40:36 - mmengine - INFO - Iter(train) [ 2070/13954] lr: 9.6374e-04 eta: 14:17:00 time: 4.3102 data_time: 0.0118 memory: 34680 loss: 2.9381 +2024/02/20 07:41:19 - mmengine - INFO - Iter(train) [ 2080/13954] lr: 9.6330e-04 eta: 14:16:16 time: 4.3148 data_time: 0.0116 memory: 34680 loss: 3.0414 +2024/02/20 07:42:03 - mmengine - INFO - Iter(train) [ 2090/13954] lr: 9.6287e-04 eta: 14:15:33 time: 4.3340 data_time: 0.0130 memory: 34634 loss: 3.0314 +2024/02/20 07:42:46 - mmengine - INFO - Iter(train) [ 2100/13954] lr: 9.6243e-04 eta: 14:14:51 time: 4.3387 data_time: 0.0140 memory: 35711 loss: 3.1238 +2024/02/20 07:43:29 - mmengine - INFO - Iter(train) [ 2110/13954] lr: 9.6198e-04 eta: 14:14:07 time: 4.3131 data_time: 0.0140 memory: 34727 loss: 2.9741 +2024/02/20 07:44:13 - mmengine - INFO - Iter(train) [ 2120/13954] lr: 9.6154e-04 eta: 14:13:24 time: 4.3438 data_time: 0.0140 memory: 34822 loss: 2.8904 +2024/02/20 07:44:56 - mmengine - INFO - Iter(train) [ 2130/13954] lr: 9.6109e-04 eta: 14:12:41 time: 4.3200 data_time: 0.0139 memory: 34539 loss: 2.9095 +2024/02/20 07:45:39 - mmengine - INFO - Iter(train) [ 2140/13954] lr: 9.6064e-04 eta: 14:11:59 time: 4.3480 data_time: 0.0141 memory: 35009 loss: 3.0543 +2024/02/20 07:46:23 - mmengine - INFO - Iter(train) [ 2150/13954] lr: 9.6019e-04 eta: 14:11:15 time: 4.3224 data_time: 0.0138 memory: 35230 loss: 3.0347 +2024/02/20 07:47:06 - mmengine - INFO - Iter(train) [ 2160/13954] lr: 9.5973e-04 eta: 14:10:32 time: 4.3325 data_time: 0.0139 memory: 34822 loss: 2.9575 +2024/02/20 07:47:49 - mmengine - INFO - Iter(train) [ 2170/13954] lr: 9.5928e-04 eta: 14:09:49 time: 4.3255 data_time: 0.0138 memory: 36273 loss: 2.9654 +2024/02/20 07:48:32 - mmengine - INFO - Iter(train) [ 2180/13954] lr: 9.5882e-04 eta: 14:09:06 time: 4.3389 data_time: 0.0141 memory: 34903 loss: 2.9281 +2024/02/20 07:49:16 - mmengine - INFO - Iter(train) [ 2190/13954] lr: 9.5835e-04 eta: 14:08:24 time: 4.3457 data_time: 0.0139 memory: 34727 loss: 2.9721 +2024/02/20 07:49:59 - mmengine - INFO - Iter(train) [ 2200/13954] lr: 9.5789e-04 eta: 14:07:42 time: 4.3456 data_time: 0.0138 memory: 35009 loss: 3.0255 +2024/02/20 07:50:43 - mmengine - INFO - Iter(train) [ 2210/13954] lr: 9.5742e-04 eta: 14:06:58 time: 4.3225 data_time: 0.0139 memory: 35149 loss: 2.9330 +2024/02/20 07:51:26 - mmengine - INFO - Iter(train) [ 2220/13954] lr: 9.5695e-04 eta: 14:06:16 time: 4.3574 data_time: 0.0140 memory: 35605 loss: 2.9517 +2024/02/20 07:52:10 - mmengine - INFO - Iter(train) [ 2230/13954] lr: 9.5648e-04 eta: 14:05:34 time: 4.3472 data_time: 0.0139 memory: 34809 loss: 2.9311 +2024/02/20 07:52:53 - mmengine - INFO - Iter(train) [ 2240/13954] lr: 9.5600e-04 eta: 14:04:50 time: 4.3155 data_time: 0.0140 memory: 34634 loss: 2.9517 +2024/02/20 07:53:36 - mmengine - INFO - Iter(train) [ 2250/13954] lr: 9.5553e-04 eta: 14:04:07 time: 4.3258 data_time: 0.0139 memory: 34822 loss: 2.9947 +2024/02/20 07:54:19 - mmengine - INFO - Iter(train) [ 2260/13954] lr: 9.5505e-04 eta: 14:03:24 time: 4.3256 data_time: 0.0140 memory: 34822 loss: 2.9077 +2024/02/20 07:55:02 - mmengine - INFO - Iter(train) [ 2270/13954] lr: 9.5457e-04 eta: 14:02:40 time: 4.3125 data_time: 0.0139 memory: 34539 loss: 3.0288 +2024/02/20 07:55:46 - mmengine - INFO - Iter(train) [ 2280/13954] lr: 9.5408e-04 eta: 14:01:56 time: 4.3251 data_time: 0.0138 memory: 34856 loss: 3.0063 +2024/02/20 07:56:29 - mmengine - INFO - Iter(train) [ 2290/13954] lr: 9.5359e-04 eta: 14:01:13 time: 4.3331 data_time: 0.0139 memory: 34962 loss: 2.8935 +2024/02/20 07:57:13 - mmengine - INFO - Iter(train) [ 2300/13954] lr: 9.5310e-04 eta: 14:00:31 time: 4.3512 data_time: 0.0140 memory: 35184 loss: 2.9879 +2024/02/20 07:57:56 - mmengine - INFO - Iter(train) [ 2310/13954] lr: 9.5261e-04 eta: 13:59:47 time: 4.3196 data_time: 0.0139 memory: 34680 loss: 2.9729 +2024/02/20 07:58:39 - mmengine - INFO - Iter(train) [ 2320/13954] lr: 9.5212e-04 eta: 13:59:05 time: 4.3414 data_time: 0.0140 memory: 35711 loss: 2.9172 +2024/02/20 07:59:22 - mmengine - INFO - Iter(train) [ 2330/13954] lr: 9.5162e-04 eta: 13:58:21 time: 4.3144 data_time: 0.0140 memory: 34868 loss: 2.9261 +2024/02/20 08:00:05 - mmengine - INFO - Iter(train) [ 2340/13954] lr: 9.5112e-04 eta: 13:57:37 time: 4.3172 data_time: 0.0140 memory: 34774 loss: 3.0227 +2024/02/20 08:00:48 - mmengine - INFO - Iter(train) [ 2350/13954] lr: 9.5062e-04 eta: 13:56:52 time: 4.2971 data_time: 0.0139 memory: 34715 loss: 3.0084 +2024/02/20 08:01:32 - mmengine - INFO - Iter(train) [ 2360/13954] lr: 9.5012e-04 eta: 13:56:09 time: 4.3161 data_time: 0.0140 memory: 34822 loss: 3.0012 +2024/02/20 08:02:15 - mmengine - INFO - Iter(train) [ 2370/13954] lr: 9.4961e-04 eta: 13:55:25 time: 4.3163 data_time: 0.0139 memory: 34821 loss: 2.8879 +2024/02/20 08:02:58 - mmengine - INFO - Iter(train) [ 2380/13954] lr: 9.4910e-04 eta: 13:54:41 time: 4.3082 data_time: 0.0139 memory: 34822 loss: 2.9190 +2024/02/20 08:03:41 - mmengine - INFO - Iter(train) [ 2390/13954] lr: 9.4859e-04 eta: 13:53:57 time: 4.3232 data_time: 0.0140 memory: 34774 loss: 2.9701 +2024/02/20 08:04:24 - mmengine - INFO - Iter(train) [ 2400/13954] lr: 9.4808e-04 eta: 13:53:14 time: 4.3199 data_time: 0.0139 memory: 34810 loss: 3.0154 +2024/02/20 08:05:07 - mmengine - INFO - Iter(train) [ 2410/13954] lr: 9.4756e-04 eta: 13:52:29 time: 4.3039 data_time: 0.0139 memory: 35044 loss: 2.9989 +2024/02/20 08:05:50 - mmengine - INFO - Iter(train) [ 2420/13954] lr: 9.4704e-04 eta: 13:51:45 time: 4.3083 data_time: 0.0141 memory: 34774 loss: 2.9506 +2024/02/20 08:06:33 - mmengine - INFO - Iter(train) [ 2430/13954] lr: 9.4652e-04 eta: 13:51:00 time: 4.2981 data_time: 0.0141 memory: 34727 loss: 2.9137 +2024/02/20 08:07:17 - mmengine - INFO - Iter(train) [ 2440/13954] lr: 9.4600e-04 eta: 13:50:18 time: 4.3375 data_time: 0.0140 memory: 34822 loss: 2.9205 +2024/02/20 08:08:00 - mmengine - INFO - Iter(train) [ 2450/13954] lr: 9.4547e-04 eta: 13:49:34 time: 4.3293 data_time: 0.0140 memory: 34822 loss: 2.8619 +2024/02/20 08:08:43 - mmengine - INFO - Iter(train) [ 2460/13954] lr: 9.4494e-04 eta: 13:48:51 time: 4.3199 data_time: 0.0141 memory: 34810 loss: 2.9553 +2024/02/20 08:09:26 - mmengine - INFO - Iter(train) [ 2470/13954] lr: 9.4441e-04 eta: 13:48:06 time: 4.2990 data_time: 0.0139 memory: 34634 loss: 2.8706 +2024/02/20 08:10:09 - mmengine - INFO - Iter(train) [ 2480/13954] lr: 9.4388e-04 eta: 13:47:22 time: 4.3064 data_time: 0.0141 memory: 34634 loss: 2.9166 +2024/02/20 08:10:53 - mmengine - INFO - Iter(train) [ 2490/13954] lr: 9.4334e-04 eta: 13:46:39 time: 4.3221 data_time: 0.0141 memory: 34634 loss: 2.9887 +2024/02/20 08:11:36 - mmengine - INFO - Iter(train) [ 2500/13954] lr: 9.4281e-04 eta: 13:45:56 time: 4.3347 data_time: 0.0141 memory: 36167 loss: 2.9520 +2024/02/20 08:11:36 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 08:11:36 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + +2024/02/20 08:11:37 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock in the lake with a boat on the shore<|im_end|> + +2024/02/20 08:11:37 - mmengine - INFO - Saving checkpoint at 2500 iterations +2024/02/20 08:12:20 - mmengine - INFO - Iter(train) [ 2510/13954] lr: 9.4227e-04 eta: 13:45:18 time: 4.4516 data_time: 0.1131 memory: 36273 loss: 2.9634 +2024/02/20 08:13:04 - mmengine - INFO - Iter(train) [ 2520/13954] lr: 9.4172e-04 eta: 13:44:34 time: 4.3149 data_time: 0.0139 memory: 34762 loss: 2.9623 +2024/02/20 08:13:47 - mmengine - INFO - Iter(train) [ 2530/13954] lr: 9.4118e-04 eta: 13:43:50 time: 4.3057 data_time: 0.0140 memory: 34680 loss: 2.9321 +2024/02/20 08:14:30 - mmengine - INFO - Iter(train) [ 2540/13954] lr: 9.4063e-04 eta: 13:43:07 time: 4.3253 data_time: 0.0139 memory: 34589 loss: 2.9777 +2024/02/20 08:15:13 - mmengine - INFO - Iter(train) [ 2550/13954] lr: 9.4008e-04 eta: 13:42:24 time: 4.3309 data_time: 0.0140 memory: 34774 loss: 2.8817 +2024/02/20 08:15:56 - mmengine - INFO - Iter(train) [ 2560/13954] lr: 9.3953e-04 eta: 13:41:40 time: 4.3169 data_time: 0.0141 memory: 34916 loss: 2.9670 +2024/02/20 08:16:40 - mmengine - INFO - Iter(train) [ 2570/13954] lr: 9.3898e-04 eta: 13:40:57 time: 4.3267 data_time: 0.0140 memory: 35184 loss: 2.9541 +2024/02/20 08:17:23 - mmengine - INFO - Iter(train) [ 2580/13954] lr: 9.3842e-04 eta: 13:40:13 time: 4.3137 data_time: 0.0142 memory: 34774 loss: 2.9548 +2024/02/20 08:18:06 - mmengine - INFO - Iter(train) [ 2590/13954] lr: 9.3786e-04 eta: 13:39:30 time: 4.3325 data_time: 0.0140 memory: 35009 loss: 2.9540 +2024/02/20 08:18:49 - mmengine - INFO - Iter(train) [ 2600/13954] lr: 9.3730e-04 eta: 13:38:46 time: 4.3136 data_time: 0.0141 memory: 34774 loss: 3.0137 +2024/02/20 08:19:32 - mmengine - INFO - Iter(train) [ 2610/13954] lr: 9.3673e-04 eta: 13:38:03 time: 4.3227 data_time: 0.0142 memory: 34822 loss: 2.8681 +2024/02/20 08:20:16 - mmengine - INFO - Iter(train) [ 2620/13954] lr: 9.3617e-04 eta: 13:37:19 time: 4.3199 data_time: 0.0140 memory: 34774 loss: 2.9709 +2024/02/20 08:20:59 - mmengine - INFO - Iter(train) [ 2630/13954] lr: 9.3560e-04 eta: 13:36:36 time: 4.3333 data_time: 0.0141 memory: 35336 loss: 2.9410 +2024/02/20 08:21:42 - mmengine - INFO - Iter(train) [ 2640/13954] lr: 9.3503e-04 eta: 13:35:52 time: 4.3019 data_time: 0.0141 memory: 34822 loss: 2.9277 +2024/02/20 08:22:25 - mmengine - INFO - Iter(train) [ 2650/13954] lr: 9.3446e-04 eta: 13:35:07 time: 4.3046 data_time: 0.0142 memory: 34774 loss: 2.9697 +2024/02/20 08:23:08 - mmengine - INFO - Iter(train) [ 2660/13954] lr: 9.3388e-04 eta: 13:34:23 time: 4.3060 data_time: 0.0141 memory: 34822 loss: 2.8863 +2024/02/20 08:23:51 - mmengine - INFO - Iter(train) [ 2670/13954] lr: 9.3330e-04 eta: 13:33:40 time: 4.3273 data_time: 0.0145 memory: 34634 loss: 2.9379 +2024/02/20 08:24:35 - mmengine - INFO - Iter(train) [ 2680/13954] lr: 9.3272e-04 eta: 13:32:57 time: 4.3364 data_time: 0.0142 memory: 34727 loss: 3.0236 +2024/02/20 08:25:18 - mmengine - INFO - Iter(train) [ 2690/13954] lr: 9.3214e-04 eta: 13:32:14 time: 4.3365 data_time: 0.0141 memory: 34810 loss: 2.9196 +2024/02/20 08:26:01 - mmengine - INFO - Iter(train) [ 2700/13954] lr: 9.3155e-04 eta: 13:31:31 time: 4.3356 data_time: 0.0139 memory: 34821 loss: 2.9968 +2024/02/20 08:26:45 - mmengine - INFO - Iter(train) [ 2710/13954] lr: 9.3097e-04 eta: 13:30:48 time: 4.3327 data_time: 0.0142 memory: 34997 loss: 2.9354 +2024/02/20 08:27:28 - mmengine - INFO - Iter(train) [ 2720/13954] lr: 9.3038e-04 eta: 13:30:05 time: 4.3336 data_time: 0.0142 memory: 34774 loss: 2.9260 +2024/02/20 08:28:11 - mmengine - INFO - Iter(train) [ 2730/13954] lr: 9.2979e-04 eta: 13:29:22 time: 4.3238 data_time: 0.0141 memory: 34903 loss: 3.0421 +2024/02/20 08:28:54 - mmengine - INFO - Iter(train) [ 2740/13954] lr: 9.2919e-04 eta: 13:28:38 time: 4.3137 data_time: 0.0141 memory: 34634 loss: 2.9984 +2024/02/20 08:29:38 - mmengine - INFO - Iter(train) [ 2750/13954] lr: 9.2860e-04 eta: 13:27:55 time: 4.3228 data_time: 0.0138 memory: 34774 loss: 2.9362 +2024/02/20 08:30:21 - mmengine - INFO - Iter(train) [ 2760/13954] lr: 9.2800e-04 eta: 13:27:12 time: 4.3261 data_time: 0.0141 memory: 34962 loss: 2.9278 +2024/02/20 08:31:04 - mmengine - INFO - Iter(train) [ 2770/13954] lr: 9.2740e-04 eta: 13:26:29 time: 4.3365 data_time: 0.0141 memory: 34774 loss: 2.9970 +2024/02/20 08:31:48 - mmengine - INFO - Iter(train) [ 2780/13954] lr: 9.2679e-04 eta: 13:25:46 time: 4.3381 data_time: 0.0141 memory: 35196 loss: 2.9469 +2024/02/20 08:32:31 - mmengine - INFO - Iter(train) [ 2790/13954] lr: 9.2619e-04 eta: 13:25:01 time: 4.2941 data_time: 0.0141 memory: 34669 loss: 2.8859 +2024/02/20 08:33:14 - mmengine - INFO - Iter(train) [ 2800/13954] lr: 9.2558e-04 eta: 13:24:19 time: 4.3470 data_time: 0.0141 memory: 35570 loss: 2.9112 +2024/02/20 08:33:58 - mmengine - INFO - Iter(train) [ 2810/13954] lr: 9.2497e-04 eta: 13:23:37 time: 4.3633 data_time: 0.0140 memory: 36226 loss: 2.9861 +2024/02/20 08:34:41 - mmengine - INFO - Iter(train) [ 2820/13954] lr: 9.2436e-04 eta: 13:22:53 time: 4.3129 data_time: 0.0142 memory: 34822 loss: 3.0245 +2024/02/20 08:35:24 - mmengine - INFO - Iter(train) [ 2830/13954] lr: 9.2374e-04 eta: 13:22:10 time: 4.3263 data_time: 0.0141 memory: 34762 loss: 2.8534 +2024/02/20 08:36:07 - mmengine - INFO - Iter(train) [ 2840/13954] lr: 9.2312e-04 eta: 13:21:26 time: 4.3158 data_time: 0.0141 memory: 34589 loss: 2.9786 +2024/02/20 08:36:51 - mmengine - INFO - Iter(train) [ 2850/13954] lr: 9.2250e-04 eta: 13:20:43 time: 4.3325 data_time: 0.0141 memory: 34715 loss: 2.9827 +2024/02/20 08:37:34 - mmengine - INFO - Iter(train) [ 2860/13954] lr: 9.2188e-04 eta: 13:19:59 time: 4.3024 data_time: 0.0142 memory: 34774 loss: 2.9676 +2024/02/20 08:38:17 - mmengine - INFO - Iter(train) [ 2870/13954] lr: 9.2126e-04 eta: 13:19:15 time: 4.3065 data_time: 0.0140 memory: 34634 loss: 2.9441 +2024/02/20 08:39:00 - mmengine - INFO - Iter(train) [ 2880/13954] lr: 9.2063e-04 eta: 13:18:31 time: 4.3145 data_time: 0.0141 memory: 34634 loss: 2.8268 +2024/02/20 08:39:43 - mmengine - INFO - Iter(train) [ 2890/13954] lr: 9.2000e-04 eta: 13:17:48 time: 4.3133 data_time: 0.0142 memory: 34680 loss: 2.9681 +2024/02/20 08:40:26 - mmengine - INFO - Iter(train) [ 2900/13954] lr: 9.1937e-04 eta: 13:17:03 time: 4.3048 data_time: 0.0142 memory: 34634 loss: 2.9448 +2024/02/20 08:41:10 - mmengine - INFO - Iter(train) [ 2910/13954] lr: 9.1874e-04 eta: 13:16:21 time: 4.3493 data_time: 0.0141 memory: 36133 loss: 2.8670 +2024/02/20 08:41:53 - mmengine - INFO - Iter(train) [ 2920/13954] lr: 9.1810e-04 eta: 13:15:37 time: 4.3125 data_time: 0.0141 memory: 34822 loss: 2.7905 +2024/02/20 08:42:36 - mmengine - INFO - Iter(train) [ 2930/13954] lr: 9.1747e-04 eta: 13:14:54 time: 4.3290 data_time: 0.0142 memory: 34680 loss: 2.9631 +2024/02/20 08:43:19 - mmengine - INFO - Iter(train) [ 2940/13954] lr: 9.1683e-04 eta: 13:14:11 time: 4.3287 data_time: 0.0140 memory: 34589 loss: 2.9269 +2024/02/20 08:44:02 - mmengine - INFO - Iter(train) [ 2950/13954] lr: 9.1619e-04 eta: 13:13:27 time: 4.3081 data_time: 0.0141 memory: 34822 loss: 2.8976 +2024/02/20 08:44:46 - mmengine - INFO - Iter(train) [ 2960/13954] lr: 9.1554e-04 eta: 13:12:44 time: 4.3274 data_time: 0.0140 memory: 34634 loss: 2.9083 +2024/02/20 08:45:29 - mmengine - INFO - Iter(train) [ 2970/13954] lr: 9.1489e-04 eta: 13:12:01 time: 4.3334 data_time: 0.0140 memory: 34822 loss: 2.9632 +2024/02/20 08:46:12 - mmengine - INFO - Iter(train) [ 2980/13954] lr: 9.1425e-04 eta: 13:11:17 time: 4.3217 data_time: 0.0142 memory: 34634 loss: 2.8902 +2024/02/20 08:46:56 - mmengine - INFO - Iter(train) [ 2990/13954] lr: 9.1360e-04 eta: 13:10:35 time: 4.3411 data_time: 0.0141 memory: 34962 loss: 2.9209 +2024/02/20 08:47:39 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 08:47:39 - mmengine - INFO - Iter(train) [ 3000/13954] lr: 9.1294e-04 eta: 13:09:51 time: 4.3142 data_time: 0.0142 memory: 34634 loss: 2.9830 +2024/02/20 08:47:39 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 08:47:39 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + +2024/02/20 08:47:39 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + +2024/02/20 08:47:39 - mmengine - INFO - Saving checkpoint at 3000 iterations +2024/02/20 08:48:23 - mmengine - INFO - Iter(train) [ 3010/13954] lr: 9.1229e-04 eta: 13:09:11 time: 4.4102 data_time: 0.0960 memory: 34774 loss: 2.9059 +2024/02/20 08:49:06 - mmengine - INFO - Iter(train) [ 3020/13954] lr: 9.1163e-04 eta: 13:08:27 time: 4.3272 data_time: 0.0139 memory: 34715 loss: 2.9134 +2024/02/20 08:49:49 - mmengine - INFO - Iter(train) [ 3030/13954] lr: 9.1097e-04 eta: 13:07:45 time: 4.3410 data_time: 0.0142 memory: 34915 loss: 2.8378 +2024/02/20 08:50:33 - mmengine - INFO - Iter(train) [ 3040/13954] lr: 9.1031e-04 eta: 13:07:01 time: 4.3254 data_time: 0.0140 memory: 35102 loss: 2.9707 +2024/02/20 08:51:16 - mmengine - INFO - Iter(train) [ 3050/13954] lr: 9.0964e-04 eta: 13:06:17 time: 4.2976 data_time: 0.0141 memory: 34589 loss: 2.9332 +2024/02/20 08:51:59 - mmengine - INFO - Iter(train) [ 3060/13954] lr: 9.0898e-04 eta: 13:05:33 time: 4.3156 data_time: 0.0140 memory: 34634 loss: 2.8930 +2024/02/20 08:52:42 - mmengine - INFO - Iter(train) [ 3070/13954] lr: 9.0831e-04 eta: 13:04:50 time: 4.3370 data_time: 0.0140 memory: 34680 loss: 2.9204 +2024/02/20 08:53:25 - mmengine - INFO - Iter(train) [ 3080/13954] lr: 9.0764e-04 eta: 13:04:07 time: 4.3123 data_time: 0.0139 memory: 34774 loss: 2.9305 +2024/02/20 08:54:09 - mmengine - INFO - Iter(train) [ 3090/13954] lr: 9.0696e-04 eta: 13:03:23 time: 4.3253 data_time: 0.0140 memory: 34668 loss: 2.8467 +2024/02/20 08:54:52 - mmengine - INFO - Iter(train) [ 3100/13954] lr: 9.0629e-04 eta: 13:02:40 time: 4.3173 data_time: 0.0139 memory: 34589 loss: 2.9845 +2024/02/20 08:55:35 - mmengine - INFO - Iter(train) [ 3110/13954] lr: 9.0561e-04 eta: 13:01:56 time: 4.3133 data_time: 0.0139 memory: 34868 loss: 2.9507 +2024/02/20 08:56:18 - mmengine - INFO - Iter(train) [ 3120/13954] lr: 9.0493e-04 eta: 13:01:13 time: 4.3268 data_time: 0.0140 memory: 34868 loss: 2.8667 +2024/02/20 08:57:01 - mmengine - INFO - Iter(train) [ 3130/13954] lr: 9.0425e-04 eta: 13:00:29 time: 4.3210 data_time: 0.0134 memory: 34774 loss: 3.0036 +2024/02/20 08:57:45 - mmengine - INFO - Iter(train) [ 3140/13954] lr: 9.0357e-04 eta: 12:59:47 time: 4.3498 data_time: 0.0130 memory: 34774 loss: 2.9615 +2024/02/20 08:58:28 - mmengine - INFO - Iter(train) [ 3150/13954] lr: 9.0288e-04 eta: 12:59:04 time: 4.3389 data_time: 0.0127 memory: 35664 loss: 2.9165 +2024/02/20 08:59:12 - mmengine - INFO - Iter(train) [ 3160/13954] lr: 9.0219e-04 eta: 12:58:21 time: 4.3383 data_time: 0.0128 memory: 35430 loss: 2.8582 +2024/02/20 08:59:55 - mmengine - INFO - Iter(train) [ 3170/13954] lr: 9.0150e-04 eta: 12:57:38 time: 4.3280 data_time: 0.0129 memory: 34680 loss: 2.8734 +2024/02/20 09:00:38 - mmengine - INFO - Iter(train) [ 3180/13954] lr: 9.0081e-04 eta: 12:56:55 time: 4.3260 data_time: 0.0127 memory: 34727 loss: 2.8359 +2024/02/20 09:01:22 - mmengine - INFO - Iter(train) [ 3190/13954] lr: 9.0011e-04 eta: 12:56:12 time: 4.3391 data_time: 0.0129 memory: 34822 loss: 2.8580 +2024/02/20 09:02:05 - mmengine - INFO - Iter(train) [ 3200/13954] lr: 8.9942e-04 eta: 12:55:29 time: 4.3383 data_time: 0.0126 memory: 34727 loss: 2.8697 +2024/02/20 09:02:48 - mmengine - INFO - Iter(train) [ 3210/13954] lr: 8.9872e-04 eta: 12:54:46 time: 4.3333 data_time: 0.0139 memory: 34774 loss: 2.9627 +2024/02/20 09:03:32 - mmengine - INFO - Iter(train) [ 3220/13954] lr: 8.9802e-04 eta: 12:54:03 time: 4.3247 data_time: 0.0140 memory: 34868 loss: 2.9039 +2024/02/20 09:04:15 - mmengine - INFO - Iter(train) [ 3230/13954] lr: 8.9731e-04 eta: 12:53:19 time: 4.3167 data_time: 0.0140 memory: 34822 loss: 2.9221 +2024/02/20 09:04:58 - mmengine - INFO - Iter(train) [ 3240/13954] lr: 8.9661e-04 eta: 12:52:36 time: 4.3245 data_time: 0.0134 memory: 34589 loss: 2.9376 +2024/02/20 09:05:41 - mmengine - INFO - Iter(train) [ 3250/13954] lr: 8.9590e-04 eta: 12:51:52 time: 4.3164 data_time: 0.0136 memory: 34774 loss: 2.8344 +2024/02/20 09:06:25 - mmengine - INFO - Iter(train) [ 3260/13954] lr: 8.9519e-04 eta: 12:51:09 time: 4.3358 data_time: 0.0131 memory: 34915 loss: 2.9543 +2024/02/20 09:07:07 - mmengine - INFO - Iter(train) [ 3270/13954] lr: 8.9448e-04 eta: 12:50:25 time: 4.2959 data_time: 0.0129 memory: 34680 loss: 2.9264 +2024/02/20 09:07:51 - mmengine - INFO - Iter(train) [ 3280/13954] lr: 8.9376e-04 eta: 12:49:41 time: 4.3127 data_time: 0.0129 memory: 34634 loss: 2.9522 +2024/02/20 09:08:34 - mmengine - INFO - Iter(train) [ 3290/13954] lr: 8.9305e-04 eta: 12:48:59 time: 4.3550 data_time: 0.0136 memory: 35946 loss: 2.8945 +2024/02/20 09:09:17 - mmengine - INFO - Iter(train) [ 3300/13954] lr: 8.9233e-04 eta: 12:48:15 time: 4.3280 data_time: 0.0138 memory: 34539 loss: 2.9480 +2024/02/20 09:10:01 - mmengine - INFO - Iter(train) [ 3310/13954] lr: 8.9161e-04 eta: 12:47:32 time: 4.3104 data_time: 0.0134 memory: 34774 loss: 2.8843 +2024/02/20 09:10:44 - mmengine - INFO - Iter(train) [ 3320/13954] lr: 8.9088e-04 eta: 12:46:49 time: 4.3390 data_time: 0.0249 memory: 35699 loss: 2.8585 +2024/02/20 09:11:27 - mmengine - INFO - Iter(train) [ 3330/13954] lr: 8.9016e-04 eta: 12:46:05 time: 4.3190 data_time: 0.0137 memory: 34589 loss: 2.8629 +2024/02/20 09:12:10 - mmengine - INFO - Iter(train) [ 3340/13954] lr: 8.8943e-04 eta: 12:45:22 time: 4.3314 data_time: 0.0133 memory: 34774 loss: 2.9453 +2024/02/20 09:12:54 - mmengine - INFO - Iter(train) [ 3350/13954] lr: 8.8870e-04 eta: 12:44:39 time: 4.3309 data_time: 0.0134 memory: 34868 loss: 2.8735 +2024/02/20 09:13:37 - mmengine - INFO - Iter(train) [ 3360/13954] lr: 8.8797e-04 eta: 12:43:56 time: 4.3381 data_time: 0.0133 memory: 35009 loss: 2.9990 +2024/02/20 09:14:20 - mmengine - INFO - Iter(train) [ 3370/13954] lr: 8.8724e-04 eta: 12:43:13 time: 4.3376 data_time: 0.0133 memory: 34680 loss: 2.9085 +2024/02/20 09:15:04 - mmengine - INFO - Iter(train) [ 3380/13954] lr: 8.8651e-04 eta: 12:42:31 time: 4.3517 data_time: 0.0136 memory: 34589 loss: 2.9519 +2024/02/20 09:15:47 - mmengine - INFO - Iter(train) [ 3390/13954] lr: 8.8577e-04 eta: 12:41:48 time: 4.3345 data_time: 0.0134 memory: 34622 loss: 3.0319 +2024/02/20 09:16:31 - mmengine - INFO - Iter(train) [ 3400/13954] lr: 8.8503e-04 eta: 12:41:05 time: 4.3476 data_time: 0.0138 memory: 36648 loss: 2.8960 +2024/02/20 09:17:14 - mmengine - INFO - Iter(train) [ 3410/13954] lr: 8.8429e-04 eta: 12:40:23 time: 4.3614 data_time: 0.0136 memory: 34727 loss: 2.8914 +2024/02/20 09:17:58 - mmengine - INFO - Iter(train) [ 3420/13954] lr: 8.8354e-04 eta: 12:39:41 time: 4.3659 data_time: 0.0138 memory: 34950 loss: 2.8875 +2024/02/20 09:18:41 - mmengine - INFO - Iter(train) [ 3430/13954] lr: 8.8280e-04 eta: 12:38:57 time: 4.3180 data_time: 0.0139 memory: 34634 loss: 2.9353 +2024/02/20 09:19:25 - mmengine - INFO - Iter(train) [ 3440/13954] lr: 8.8205e-04 eta: 12:38:14 time: 4.3317 data_time: 0.0140 memory: 34774 loss: 2.8751 +2024/02/20 09:20:08 - mmengine - INFO - Iter(train) [ 3450/13954] lr: 8.8130e-04 eta: 12:37:32 time: 4.3649 data_time: 0.0137 memory: 34821 loss: 2.9469 +2024/02/20 09:20:52 - mmengine - INFO - Iter(train) [ 3460/13954] lr: 8.8055e-04 eta: 12:36:49 time: 4.3310 data_time: 0.0137 memory: 34915 loss: 2.8623 +2024/02/20 09:21:35 - mmengine - INFO - Iter(train) [ 3470/13954] lr: 8.7980e-04 eta: 12:36:06 time: 4.3455 data_time: 0.0134 memory: 34540 loss: 2.8824 +2024/02/20 09:22:18 - mmengine - INFO - Iter(train) [ 3480/13954] lr: 8.7904e-04 eta: 12:35:23 time: 4.3415 data_time: 0.0138 memory: 34680 loss: 2.9762 +2024/02/20 09:23:02 - mmengine - INFO - Iter(train) [ 3490/13954] lr: 8.7828e-04 eta: 12:34:42 time: 4.3886 data_time: 0.0136 memory: 36214 loss: 2.8699 +2024/02/20 09:23:46 - mmengine - INFO - Iter(train) [ 3500/13954] lr: 8.7752e-04 eta: 12:33:59 time: 4.3406 data_time: 0.0136 memory: 34727 loss: 2.8806 +2024/02/20 09:23:46 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 09:23:46 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock in the background<|im_end|> + +2024/02/20 09:23:47 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a boat dock in the background<|im_end|> + +2024/02/20 09:23:47 - mmengine - INFO - Saving checkpoint at 3500 iterations +2024/02/20 09:24:30 - mmengine - INFO - Iter(train) [ 3510/13954] lr: 8.7676e-04 eta: 12:33:18 time: 4.4194 data_time: 0.1118 memory: 34774 loss: 2.9730 +2024/02/20 09:25:13 - mmengine - INFO - Iter(train) [ 3520/13954] lr: 8.7600e-04 eta: 12:32:36 time: 4.3470 data_time: 0.0135 memory: 34680 loss: 2.9912 +2024/02/20 09:25:57 - mmengine - INFO - Iter(train) [ 3530/13954] lr: 8.7523e-04 eta: 12:31:53 time: 4.3419 data_time: 0.0133 memory: 35289 loss: 2.8562 +2024/02/20 09:26:40 - mmengine - INFO - Iter(train) [ 3540/13954] lr: 8.7446e-04 eta: 12:31:10 time: 4.3381 data_time: 0.0136 memory: 34540 loss: 2.9332 +2024/02/20 09:27:24 - mmengine - INFO - Iter(train) [ 3550/13954] lr: 8.7369e-04 eta: 12:30:27 time: 4.3494 data_time: 0.0134 memory: 34727 loss: 2.9562 +2024/02/20 09:28:07 - mmengine - INFO - Iter(train) [ 3560/13954] lr: 8.7292e-04 eta: 12:29:45 time: 4.3542 data_time: 0.0134 memory: 34680 loss: 2.8964 +2024/02/20 09:28:50 - mmengine - INFO - Iter(train) [ 3570/13954] lr: 8.7215e-04 eta: 12:29:01 time: 4.3252 data_time: 0.0134 memory: 34680 loss: 2.9159 +2024/02/20 09:29:34 - mmengine - INFO - Iter(train) [ 3580/13954] lr: 8.7137e-04 eta: 12:28:19 time: 4.3454 data_time: 0.0134 memory: 34774 loss: 2.9254 +2024/02/20 09:30:17 - mmengine - INFO - Iter(train) [ 3590/13954] lr: 8.7059e-04 eta: 12:27:36 time: 4.3455 data_time: 0.0135 memory: 34962 loss: 3.0096 +2024/02/20 09:31:01 - mmengine - INFO - Iter(train) [ 3600/13954] lr: 8.6981e-04 eta: 12:26:53 time: 4.3502 data_time: 0.0139 memory: 34668 loss: 2.9372 +2024/02/20 09:31:44 - mmengine - INFO - Iter(train) [ 3610/13954] lr: 8.6903e-04 eta: 12:26:10 time: 4.3269 data_time: 0.0138 memory: 34634 loss: 2.9346 +2024/02/20 09:32:27 - mmengine - INFO - Iter(train) [ 3620/13954] lr: 8.6825e-04 eta: 12:25:26 time: 4.3220 data_time: 0.0139 memory: 34774 loss: 2.9262 +2024/02/20 09:33:11 - mmengine - INFO - Iter(train) [ 3630/13954] lr: 8.6746e-04 eta: 12:24:44 time: 4.3518 data_time: 0.0140 memory: 34822 loss: 2.8401 +2024/02/20 09:33:54 - mmengine - INFO - Iter(train) [ 3640/13954] lr: 8.6667e-04 eta: 12:24:01 time: 4.3514 data_time: 0.0138 memory: 34715 loss: 2.9506 +2024/02/20 09:34:38 - mmengine - INFO - Iter(train) [ 3650/13954] lr: 8.6588e-04 eta: 12:23:18 time: 4.3395 data_time: 0.0138 memory: 34822 loss: 2.7048 +2024/02/20 09:35:21 - mmengine - INFO - Iter(train) [ 3660/13954] lr: 8.6509e-04 eta: 12:22:35 time: 4.3349 data_time: 0.0134 memory: 34915 loss: 2.8616 +2024/02/20 09:36:04 - mmengine - INFO - Iter(train) [ 3670/13954] lr: 8.6430e-04 eta: 12:21:52 time: 4.3176 data_time: 0.0135 memory: 34622 loss: 2.8741 +2024/02/20 09:36:48 - mmengine - INFO - Iter(train) [ 3680/13954] lr: 8.6350e-04 eta: 12:21:08 time: 4.3333 data_time: 0.0133 memory: 34589 loss: 2.8850 +2024/02/20 09:37:31 - mmengine - INFO - Iter(train) [ 3690/13954] lr: 8.6270e-04 eta: 12:20:25 time: 4.3229 data_time: 0.0134 memory: 34822 loss: 2.8619 +2024/02/20 09:38:14 - mmengine - INFO - Iter(train) [ 3700/13954] lr: 8.6190e-04 eta: 12:19:42 time: 4.3436 data_time: 0.0133 memory: 34727 loss: 2.8683 +2024/02/20 09:38:57 - mmengine - INFO - Iter(train) [ 3710/13954] lr: 8.6110e-04 eta: 12:18:58 time: 4.3141 data_time: 0.0135 memory: 34493 loss: 2.8765 +2024/02/20 09:39:41 - mmengine - INFO - Iter(train) [ 3720/13954] lr: 8.6030e-04 eta: 12:18:15 time: 4.3144 data_time: 0.0135 memory: 34680 loss: 2.8671 +2024/02/20 09:40:24 - mmengine - INFO - Iter(train) [ 3730/13954] lr: 8.5949e-04 eta: 12:17:33 time: 4.3735 data_time: 0.0135 memory: 36320 loss: 2.8836 +2024/02/20 09:41:08 - mmengine - INFO - Iter(train) [ 3740/13954] lr: 8.5868e-04 eta: 12:16:49 time: 4.3251 data_time: 0.0139 memory: 34774 loss: 2.9107 +2024/02/20 09:41:51 - mmengine - INFO - Iter(train) [ 3750/13954] lr: 8.5788e-04 eta: 12:16:06 time: 4.3367 data_time: 0.0137 memory: 34962 loss: 2.9284 +2024/02/20 09:42:34 - mmengine - INFO - Iter(train) [ 3760/13954] lr: 8.5706e-04 eta: 12:15:23 time: 4.3418 data_time: 0.0133 memory: 35570 loss: 3.0065 +2024/02/20 09:43:18 - mmengine - INFO - Iter(train) [ 3770/13954] lr: 8.5625e-04 eta: 12:14:40 time: 4.3332 data_time: 0.0134 memory: 34668 loss: 2.8536 +2024/02/20 09:44:01 - mmengine - INFO - Iter(train) [ 3780/13954] lr: 8.5544e-04 eta: 12:13:57 time: 4.3286 data_time: 0.0134 memory: 34869 loss: 2.8295 +2024/02/20 09:44:44 - mmengine - INFO - Iter(train) [ 3790/13954] lr: 8.5462e-04 eta: 12:13:14 time: 4.3266 data_time: 0.0133 memory: 34774 loss: 2.8238 +2024/02/20 09:45:28 - mmengine - INFO - Iter(train) [ 3800/13954] lr: 8.5380e-04 eta: 12:12:30 time: 4.3257 data_time: 0.0132 memory: 34540 loss: 2.9144 +2024/02/20 09:46:11 - mmengine - INFO - Iter(train) [ 3810/13954] lr: 8.5298e-04 eta: 12:11:48 time: 4.3536 data_time: 0.0135 memory: 34727 loss: 3.0229 +2024/02/20 09:46:54 - mmengine - INFO - Iter(train) [ 3820/13954] lr: 8.5216e-04 eta: 12:11:04 time: 4.3338 data_time: 0.0133 memory: 34774 loss: 2.8953 +2024/02/20 09:47:38 - mmengine - INFO - Iter(train) [ 3830/13954] lr: 8.5133e-04 eta: 12:10:21 time: 4.3123 data_time: 0.0135 memory: 34856 loss: 2.8403 +2024/02/20 09:48:21 - mmengine - INFO - Iter(train) [ 3840/13954] lr: 8.5050e-04 eta: 12:09:37 time: 4.3162 data_time: 0.0135 memory: 34634 loss: 2.9537 +2024/02/20 09:49:04 - mmengine - INFO - Iter(train) [ 3850/13954] lr: 8.4968e-04 eta: 12:08:53 time: 4.3119 data_time: 0.0135 memory: 34715 loss: 2.8690 +2024/02/20 09:49:47 - mmengine - INFO - Iter(train) [ 3860/13954] lr: 8.4885e-04 eta: 12:08:10 time: 4.3226 data_time: 0.0135 memory: 34774 loss: 2.8505 +2024/02/20 09:50:30 - mmengine - INFO - Iter(train) [ 3870/13954] lr: 8.4801e-04 eta: 12:07:26 time: 4.3114 data_time: 0.0136 memory: 34774 loss: 2.8416 +2024/02/20 09:51:13 - mmengine - INFO - Iter(train) [ 3880/13954] lr: 8.4718e-04 eta: 12:06:43 time: 4.3242 data_time: 0.0136 memory: 34809 loss: 2.9550 +2024/02/20 09:51:57 - mmengine - INFO - Iter(train) [ 3890/13954] lr: 8.4634e-04 eta: 12:06:00 time: 4.3378 data_time: 0.0135 memory: 34856 loss: 2.8902 +2024/02/20 09:52:40 - mmengine - INFO - Iter(train) [ 3900/13954] lr: 8.4550e-04 eta: 12:05:17 time: 4.3441 data_time: 0.0133 memory: 34680 loss: 2.9012 +2024/02/20 09:53:24 - mmengine - INFO - Iter(train) [ 3910/13954] lr: 8.4467e-04 eta: 12:04:34 time: 4.3343 data_time: 0.0135 memory: 34634 loss: 2.8426 +2024/02/20 09:54:07 - mmengine - INFO - Iter(train) [ 3920/13954] lr: 8.4382e-04 eta: 12:03:50 time: 4.3130 data_time: 0.0135 memory: 34539 loss: 2.8166 +2024/02/20 09:54:50 - mmengine - INFO - Iter(train) [ 3930/13954] lr: 8.4298e-04 eta: 12:03:07 time: 4.3295 data_time: 0.0135 memory: 34634 loss: 2.8196 +2024/02/20 09:55:33 - mmengine - INFO - Iter(train) [ 3940/13954] lr: 8.4213e-04 eta: 12:02:23 time: 4.3098 data_time: 0.0136 memory: 34774 loss: 2.8360 +2024/02/20 09:56:16 - mmengine - INFO - Iter(train) [ 3950/13954] lr: 8.4129e-04 eta: 12:01:40 time: 4.3248 data_time: 0.0135 memory: 34727 loss: 2.8419 +2024/02/20 09:57:00 - mmengine - INFO - Iter(train) [ 3960/13954] lr: 8.4044e-04 eta: 12:00:57 time: 4.3432 data_time: 0.0137 memory: 34680 loss: 2.9057 +2024/02/20 09:57:43 - mmengine - INFO - Iter(train) [ 3970/13954] lr: 8.3959e-04 eta: 12:00:13 time: 4.3222 data_time: 0.0136 memory: 34774 loss: 2.9212 +2024/02/20 09:58:26 - mmengine - INFO - Iter(train) [ 3980/13954] lr: 8.3874e-04 eta: 11:59:31 time: 4.3487 data_time: 0.0133 memory: 34868 loss: 2.9016 +2024/02/20 09:59:10 - mmengine - INFO - Iter(train) [ 3990/13954] lr: 8.3788e-04 eta: 11:58:48 time: 4.3403 data_time: 0.0136 memory: 34962 loss: 2.8965 +2024/02/20 09:59:53 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 09:59:53 - mmengine - INFO - Iter(train) [ 4000/13954] lr: 8.3702e-04 eta: 11:58:04 time: 4.3233 data_time: 0.0135 memory: 34589 loss: 2.9515 +2024/02/20 09:59:53 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 09:59:53 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock with a boat and a lake in the background<|im_end|> + +2024/02/20 09:59:54 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock with a boat on the water and a bridge over it<|im_end|> + +2024/02/20 09:59:54 - mmengine - INFO - Saving checkpoint at 4000 iterations +2024/02/20 10:00:37 - mmengine - INFO - Iter(train) [ 4010/13954] lr: 8.3617e-04 eta: 11:57:23 time: 4.4263 data_time: 0.0996 memory: 34680 loss: 2.8541 +2024/02/20 10:01:21 - mmengine - INFO - Iter(train) [ 4020/13954] lr: 8.3531e-04 eta: 11:56:41 time: 4.3448 data_time: 0.0134 memory: 34634 loss: 2.8231 +2024/02/20 10:02:04 - mmengine - INFO - Iter(train) [ 4030/13954] lr: 8.3444e-04 eta: 11:55:58 time: 4.3666 data_time: 0.0135 memory: 34680 loss: 2.8985 +2024/02/20 10:02:48 - mmengine - INFO - Iter(train) [ 4040/13954] lr: 8.3358e-04 eta: 11:55:15 time: 4.3523 data_time: 0.0136 memory: 35196 loss: 2.7591 +2024/02/20 10:03:31 - mmengine - INFO - Iter(train) [ 4050/13954] lr: 8.3272e-04 eta: 11:54:32 time: 4.3321 data_time: 0.0136 memory: 34680 loss: 2.8584 +2024/02/20 10:04:15 - mmengine - INFO - Iter(train) [ 4060/13954] lr: 8.3185e-04 eta: 11:53:49 time: 4.3425 data_time: 0.0136 memory: 34869 loss: 2.9092 +2024/02/20 10:04:58 - mmengine - INFO - Iter(train) [ 4070/13954] lr: 8.3098e-04 eta: 11:53:06 time: 4.3387 data_time: 0.0136 memory: 34868 loss: 2.8599 +2024/02/20 10:05:41 - mmengine - INFO - Iter(train) [ 4080/13954] lr: 8.3011e-04 eta: 11:52:23 time: 4.3243 data_time: 0.0137 memory: 34680 loss: 2.8337 +2024/02/20 10:06:25 - mmengine - INFO - Iter(train) [ 4090/13954] lr: 8.2924e-04 eta: 11:51:40 time: 4.3280 data_time: 0.0137 memory: 34822 loss: 2.8628 +2024/02/20 10:07:08 - mmengine - INFO - Iter(train) [ 4100/13954] lr: 8.2836e-04 eta: 11:50:56 time: 4.3246 data_time: 0.0137 memory: 34668 loss: 2.8389 +2024/02/20 10:07:51 - mmengine - INFO - Iter(train) [ 4110/13954] lr: 8.2749e-04 eta: 11:50:13 time: 4.3175 data_time: 0.0138 memory: 34680 loss: 2.8716 +2024/02/20 10:08:34 - mmengine - INFO - Iter(train) [ 4120/13954] lr: 8.2661e-04 eta: 11:49:29 time: 4.3363 data_time: 0.0136 memory: 34727 loss: 2.9073 +2024/02/20 10:09:18 - mmengine - INFO - Iter(train) [ 4130/13954] lr: 8.2573e-04 eta: 11:48:46 time: 4.3335 data_time: 0.0134 memory: 34809 loss: 2.9101 +2024/02/20 10:10:01 - mmengine - INFO - Iter(train) [ 4140/13954] lr: 8.2485e-04 eta: 11:48:03 time: 4.3379 data_time: 0.0137 memory: 35149 loss: 2.8793 +2024/02/20 10:10:44 - mmengine - INFO - Iter(train) [ 4150/13954] lr: 8.2396e-04 eta: 11:47:20 time: 4.3187 data_time: 0.0135 memory: 34774 loss: 2.9469 +2024/02/20 10:11:28 - mmengine - INFO - Iter(train) [ 4160/13954] lr: 8.2308e-04 eta: 11:46:37 time: 4.3323 data_time: 0.0137 memory: 34634 loss: 2.9489 +2024/02/20 10:12:11 - mmengine - INFO - Iter(train) [ 4170/13954] lr: 8.2219e-04 eta: 11:45:54 time: 4.3501 data_time: 0.0134 memory: 34715 loss: 3.0369 +2024/02/20 10:12:54 - mmengine - INFO - Iter(train) [ 4180/13954] lr: 8.2130e-04 eta: 11:45:10 time: 4.3222 data_time: 0.0135 memory: 34774 loss: 2.8452 +2024/02/20 10:13:38 - mmengine - INFO - Iter(train) [ 4190/13954] lr: 8.2041e-04 eta: 11:44:27 time: 4.3196 data_time: 0.0135 memory: 34869 loss: 2.9612 +2024/02/20 10:14:21 - mmengine - INFO - Iter(train) [ 4200/13954] lr: 8.1952e-04 eta: 11:43:44 time: 4.3467 data_time: 0.0136 memory: 34680 loss: 2.8573 +2024/02/20 10:15:04 - mmengine - INFO - Iter(train) [ 4210/13954] lr: 8.1863e-04 eta: 11:43:01 time: 4.3296 data_time: 0.0137 memory: 34634 loss: 2.8684 +2024/02/20 10:15:48 - mmengine - INFO - Iter(train) [ 4220/13954] lr: 8.1773e-04 eta: 11:42:18 time: 4.3379 data_time: 0.0136 memory: 34774 loss: 2.9046 +2024/02/20 10:16:31 - mmengine - INFO - Iter(train) [ 4230/13954] lr: 8.1684e-04 eta: 11:41:34 time: 4.3176 data_time: 0.0135 memory: 34540 loss: 2.8539 +2024/02/20 10:17:14 - mmengine - INFO - Iter(train) [ 4240/13954] lr: 8.1594e-04 eta: 11:40:51 time: 4.3581 data_time: 0.0136 memory: 35992 loss: 2.8180 +2024/02/20 10:17:58 - mmengine - INFO - Iter(train) [ 4250/13954] lr: 8.1504e-04 eta: 11:40:08 time: 4.3459 data_time: 0.0138 memory: 34715 loss: 2.7912 +2024/02/20 10:18:41 - mmengine - INFO - Iter(train) [ 4260/13954] lr: 8.1414e-04 eta: 11:39:25 time: 4.3278 data_time: 0.0137 memory: 34634 loss: 2.8174 +2024/02/20 10:19:25 - mmengine - INFO - Iter(train) [ 4270/13954] lr: 8.1323e-04 eta: 11:38:42 time: 4.3390 data_time: 0.0131 memory: 34727 loss: 2.8796 +2024/02/20 10:20:08 - mmengine - INFO - Iter(train) [ 4280/13954] lr: 8.1233e-04 eta: 11:37:59 time: 4.3270 data_time: 0.0129 memory: 35090 loss: 2.9008 +2024/02/20 10:20:51 - mmengine - INFO - Iter(train) [ 4290/13954] lr: 8.1142e-04 eta: 11:37:15 time: 4.3272 data_time: 0.0127 memory: 34869 loss: 2.9223 +2024/02/20 10:21:34 - mmengine - INFO - Iter(train) [ 4300/13954] lr: 8.1051e-04 eta: 11:36:32 time: 4.3222 data_time: 0.0132 memory: 34774 loss: 2.8557 +2024/02/20 10:22:18 - mmengine - INFO - Iter(train) [ 4310/13954] lr: 8.0960e-04 eta: 11:35:48 time: 4.3212 data_time: 0.0139 memory: 34915 loss: 2.8537 +2024/02/20 10:23:01 - mmengine - INFO - Iter(train) [ 4320/13954] lr: 8.0869e-04 eta: 11:35:05 time: 4.3345 data_time: 0.0138 memory: 34727 loss: 2.8992 +2024/02/20 10:23:44 - mmengine - INFO - Iter(train) [ 4330/13954] lr: 8.0778e-04 eta: 11:34:23 time: 4.3506 data_time: 0.0139 memory: 34857 loss: 2.9091 +2024/02/20 10:24:28 - mmengine - INFO - Iter(train) [ 4340/13954] lr: 8.0686e-04 eta: 11:33:39 time: 4.3230 data_time: 0.0139 memory: 34774 loss: 2.8917 +2024/02/20 10:25:11 - mmengine - INFO - Iter(train) [ 4350/13954] lr: 8.0594e-04 eta: 11:32:56 time: 4.3228 data_time: 0.0139 memory: 34634 loss: 2.8620 +2024/02/20 10:25:54 - mmengine - INFO - Iter(train) [ 4360/13954] lr: 8.0502e-04 eta: 11:32:13 time: 4.3378 data_time: 0.0140 memory: 35289 loss: 2.8724 +2024/02/20 10:26:38 - mmengine - INFO - Iter(train) [ 4370/13954] lr: 8.0410e-04 eta: 11:31:30 time: 4.3414 data_time: 0.0140 memory: 34962 loss: 2.7952 +2024/02/20 10:27:21 - mmengine - INFO - Iter(train) [ 4380/13954] lr: 8.0318e-04 eta: 11:30:46 time: 4.3206 data_time: 0.0140 memory: 34774 loss: 2.8678 +2024/02/20 10:28:04 - mmengine - INFO - Iter(train) [ 4390/13954] lr: 8.0226e-04 eta: 11:30:03 time: 4.3508 data_time: 0.0139 memory: 34727 loss: 2.8780 +2024/02/20 10:28:48 - mmengine - INFO - Iter(train) [ 4400/13954] lr: 8.0133e-04 eta: 11:29:20 time: 4.3266 data_time: 0.0140 memory: 34822 loss: 2.8347 +2024/02/20 10:29:31 - mmengine - INFO - Iter(train) [ 4410/13954] lr: 8.0041e-04 eta: 11:28:38 time: 4.3753 data_time: 0.0135 memory: 35570 loss: 2.9020 +2024/02/20 10:30:15 - mmengine - INFO - Iter(train) [ 4420/13954] lr: 7.9948e-04 eta: 11:27:54 time: 4.3372 data_time: 0.0136 memory: 34727 loss: 2.8668 +2024/02/20 10:30:58 - mmengine - INFO - Iter(train) [ 4430/13954] lr: 7.9855e-04 eta: 11:27:11 time: 4.3301 data_time: 0.0138 memory: 34715 loss: 2.8572 +2024/02/20 10:31:41 - mmengine - INFO - Iter(train) [ 4440/13954] lr: 7.9762e-04 eta: 11:26:28 time: 4.3306 data_time: 0.0135 memory: 34762 loss: 2.8497 +2024/02/20 10:32:25 - mmengine - INFO - Iter(train) [ 4450/13954] lr: 7.9668e-04 eta: 11:25:45 time: 4.3414 data_time: 0.0135 memory: 34915 loss: 2.7919 +2024/02/20 10:33:08 - mmengine - INFO - Iter(train) [ 4460/13954] lr: 7.9575e-04 eta: 11:25:02 time: 4.3274 data_time: 0.0134 memory: 34762 loss: 2.8921 +2024/02/20 10:33:51 - mmengine - INFO - Iter(train) [ 4470/13954] lr: 7.9481e-04 eta: 11:24:18 time: 4.3226 data_time: 0.0136 memory: 34822 loss: 2.8389 +2024/02/20 10:34:34 - mmengine - INFO - Iter(train) [ 4480/13954] lr: 7.9387e-04 eta: 11:23:34 time: 4.3099 data_time: 0.0136 memory: 34540 loss: 2.8850 +2024/02/20 10:35:18 - mmengine - INFO - Iter(train) [ 4490/13954] lr: 7.9293e-04 eta: 11:22:51 time: 4.3275 data_time: 0.0134 memory: 34727 loss: 2.9210 +2024/02/20 10:36:01 - mmengine - INFO - Iter(train) [ 4500/13954] lr: 7.9199e-04 eta: 11:22:08 time: 4.3156 data_time: 0.0132 memory: 35009 loss: 2.9257 +2024/02/20 10:36:01 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 10:36:01 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a pier on a lake with a boat on the pier<|im_end|> + +2024/02/20 10:36:01 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + +2024/02/20 10:36:01 - mmengine - INFO - Saving checkpoint at 4500 iterations +2024/02/20 10:36:45 - mmengine - INFO - Iter(train) [ 4510/13954] lr: 7.9105e-04 eta: 11:21:26 time: 4.3946 data_time: 0.0952 memory: 35102 loss: 2.8619 +2024/02/20 10:37:28 - mmengine - INFO - Iter(train) [ 4520/13954] lr: 7.9011e-04 eta: 11:20:42 time: 4.3097 data_time: 0.0133 memory: 34868 loss: 2.8681 +2024/02/20 10:38:11 - mmengine - INFO - Iter(train) [ 4530/13954] lr: 7.8916e-04 eta: 11:19:58 time: 4.3071 data_time: 0.0131 memory: 34727 loss: 2.8696 +2024/02/20 10:38:54 - mmengine - INFO - Iter(train) [ 4540/13954] lr: 7.8821e-04 eta: 11:19:15 time: 4.3271 data_time: 0.0133 memory: 35196 loss: 2.7567 +2024/02/20 10:39:37 - mmengine - INFO - Iter(train) [ 4550/13954] lr: 7.8726e-04 eta: 11:18:31 time: 4.3004 data_time: 0.0132 memory: 34774 loss: 2.8246 +2024/02/20 10:40:20 - mmengine - INFO - Iter(train) [ 4560/13954] lr: 7.8631e-04 eta: 11:17:47 time: 4.3207 data_time: 0.0134 memory: 34634 loss: 2.8895 +2024/02/20 10:41:04 - mmengine - INFO - Iter(train) [ 4570/13954] lr: 7.8536e-04 eta: 11:17:04 time: 4.3088 data_time: 0.0136 memory: 34822 loss: 2.8457 +2024/02/20 10:41:47 - mmengine - INFO - Iter(train) [ 4580/13954] lr: 7.8441e-04 eta: 11:16:20 time: 4.3059 data_time: 0.0133 memory: 34634 loss: 2.8403 +2024/02/20 10:42:30 - mmengine - INFO - Iter(train) [ 4590/13954] lr: 7.8345e-04 eta: 11:15:36 time: 4.3068 data_time: 0.0132 memory: 34915 loss: 2.9097 +2024/02/20 10:43:13 - mmengine - INFO - Iter(train) [ 4600/13954] lr: 7.8249e-04 eta: 11:14:52 time: 4.2855 data_time: 0.0134 memory: 34634 loss: 2.8317 +2024/02/20 10:43:56 - mmengine - INFO - Iter(train) [ 4610/13954] lr: 7.8154e-04 eta: 11:14:09 time: 4.3419 data_time: 0.0132 memory: 34821 loss: 2.8694 +2024/02/20 10:44:39 - mmengine - INFO - Iter(train) [ 4620/13954] lr: 7.8058e-04 eta: 11:13:26 time: 4.3287 data_time: 0.0133 memory: 34540 loss: 2.8067 +2024/02/20 10:45:23 - mmengine - INFO - Iter(train) [ 4630/13954] lr: 7.7962e-04 eta: 11:12:42 time: 4.3288 data_time: 0.0134 memory: 34680 loss: 2.8597 +2024/02/20 10:46:06 - mmengine - INFO - Iter(train) [ 4640/13954] lr: 7.7865e-04 eta: 11:11:59 time: 4.3263 data_time: 0.0136 memory: 34774 loss: 2.8750 +2024/02/20 10:46:49 - mmengine - INFO - Iter(train) [ 4650/13954] lr: 7.7769e-04 eta: 11:11:16 time: 4.3228 data_time: 0.0134 memory: 34622 loss: 2.9046 +2024/02/20 10:47:32 - mmengine - INFO - Iter(train) [ 4660/13954] lr: 7.7672e-04 eta: 11:10:32 time: 4.3072 data_time: 0.0133 memory: 34633 loss: 2.8490 +2024/02/20 10:48:15 - mmengine - INFO - Iter(train) [ 4670/13954] lr: 7.7576e-04 eta: 11:09:49 time: 4.3200 data_time: 0.0132 memory: 34680 loss: 2.8289 +2024/02/20 10:48:59 - mmengine - INFO - Iter(train) [ 4680/13954] lr: 7.7479e-04 eta: 11:09:05 time: 4.3334 data_time: 0.0131 memory: 34915 loss: 2.9278 +2024/02/20 10:49:42 - mmengine - INFO - Iter(train) [ 4690/13954] lr: 7.7382e-04 eta: 11:08:22 time: 4.3425 data_time: 0.0131 memory: 35102 loss: 2.8524 +2024/02/20 10:50:25 - mmengine - INFO - Iter(train) [ 4700/13954] lr: 7.7284e-04 eta: 11:07:38 time: 4.2994 data_time: 0.0131 memory: 34680 loss: 2.7984 +2024/02/20 10:51:08 - mmengine - INFO - Iter(train) [ 4710/13954] lr: 7.7187e-04 eta: 11:06:55 time: 4.3352 data_time: 0.0132 memory: 34774 loss: 2.8936 +2024/02/20 10:51:52 - mmengine - INFO - Iter(train) [ 4720/13954] lr: 7.7090e-04 eta: 11:06:12 time: 4.3430 data_time: 0.0133 memory: 34822 loss: 2.8096 +2024/02/20 10:52:35 - mmengine - INFO - Iter(train) [ 4730/13954] lr: 7.6992e-04 eta: 11:05:29 time: 4.3078 data_time: 0.0133 memory: 34822 loss: 2.8380 +2024/02/20 10:53:18 - mmengine - INFO - Iter(train) [ 4740/13954] lr: 7.6894e-04 eta: 11:04:45 time: 4.3292 data_time: 0.0133 memory: 34727 loss: 2.8816 +2024/02/20 10:54:01 - mmengine - INFO - Iter(train) [ 4750/13954] lr: 7.6796e-04 eta: 11:04:02 time: 4.3173 data_time: 0.0132 memory: 34680 loss: 2.7401 +2024/02/20 10:54:44 - mmengine - INFO - Iter(train) [ 4760/13954] lr: 7.6698e-04 eta: 11:03:18 time: 4.3054 data_time: 0.0133 memory: 34589 loss: 2.9078 +2024/02/20 10:55:28 - mmengine - INFO - Iter(train) [ 4770/13954] lr: 7.6600e-04 eta: 11:02:35 time: 4.3214 data_time: 0.0131 memory: 34869 loss: 2.8319 +2024/02/20 10:56:11 - mmengine - INFO - Iter(train) [ 4780/13954] lr: 7.6502e-04 eta: 11:01:51 time: 4.3076 data_time: 0.0129 memory: 34589 loss: 2.8815 +2024/02/20 10:56:54 - mmengine - INFO - Iter(train) [ 4790/13954] lr: 7.6403e-04 eta: 11:01:07 time: 4.3156 data_time: 0.0130 memory: 34589 loss: 2.9092 +2024/02/20 10:57:37 - mmengine - INFO - Iter(train) [ 4800/13954] lr: 7.6305e-04 eta: 11:00:24 time: 4.3296 data_time: 0.0131 memory: 34822 loss: 2.8693 +2024/02/20 10:58:20 - mmengine - INFO - Iter(train) [ 4810/13954] lr: 7.6206e-04 eta: 10:59:41 time: 4.3323 data_time: 0.0129 memory: 34727 loss: 2.8181 +2024/02/20 10:59:04 - mmengine - INFO - Iter(train) [ 4820/13954] lr: 7.6107e-04 eta: 10:58:57 time: 4.3168 data_time: 0.0131 memory: 34774 loss: 2.7746 +2024/02/20 10:59:47 - mmengine - INFO - Iter(train) [ 4830/13954] lr: 7.6008e-04 eta: 10:58:14 time: 4.3244 data_time: 0.0133 memory: 34821 loss: 2.9547 +2024/02/20 11:00:30 - mmengine - INFO - Iter(train) [ 4840/13954] lr: 7.5909e-04 eta: 10:57:30 time: 4.3029 data_time: 0.0131 memory: 34680 loss: 2.7745 +2024/02/20 11:01:13 - mmengine - INFO - Iter(train) [ 4850/13954] lr: 7.5810e-04 eta: 10:56:47 time: 4.3196 data_time: 0.0132 memory: 35055 loss: 2.8491 +2024/02/20 11:01:56 - mmengine - INFO - Iter(train) [ 4860/13954] lr: 7.5710e-04 eta: 10:56:04 time: 4.3323 data_time: 0.0133 memory: 34774 loss: 2.8204 +2024/02/20 11:02:40 - mmengine - INFO - Iter(train) [ 4870/13954] lr: 7.5611e-04 eta: 10:55:20 time: 4.3163 data_time: 0.0133 memory: 34727 loss: 2.9481 +2024/02/20 11:03:23 - mmengine - INFO - Iter(train) [ 4880/13954] lr: 7.5511e-04 eta: 10:54:37 time: 4.3211 data_time: 0.0133 memory: 34727 loss: 2.8835 +2024/02/20 11:04:06 - mmengine - INFO - Iter(train) [ 4890/13954] lr: 7.5411e-04 eta: 10:53:53 time: 4.3334 data_time: 0.0131 memory: 34634 loss: 2.8582 +2024/02/20 11:04:49 - mmengine - INFO - Iter(train) [ 4900/13954] lr: 7.5311e-04 eta: 10:53:10 time: 4.3080 data_time: 0.0132 memory: 34727 loss: 2.9961 +2024/02/20 11:05:32 - mmengine - INFO - Iter(train) [ 4910/13954] lr: 7.5211e-04 eta: 10:52:26 time: 4.3176 data_time: 0.0131 memory: 34727 loss: 2.7477 +2024/02/20 11:06:16 - mmengine - INFO - Iter(train) [ 4920/13954] lr: 7.5110e-04 eta: 10:51:43 time: 4.3141 data_time: 0.0132 memory: 34680 loss: 2.9387 +2024/02/20 11:06:59 - mmengine - INFO - Iter(train) [ 4930/13954] lr: 7.5010e-04 eta: 10:51:00 time: 4.3455 data_time: 0.0132 memory: 34868 loss: 2.8986 +2024/02/20 11:07:42 - mmengine - INFO - Iter(train) [ 4940/13954] lr: 7.4909e-04 eta: 10:50:16 time: 4.3154 data_time: 0.0134 memory: 34589 loss: 2.8333 +2024/02/20 11:08:26 - mmengine - INFO - Iter(train) [ 4950/13954] lr: 7.4809e-04 eta: 10:49:33 time: 4.3356 data_time: 0.0131 memory: 34774 loss: 2.8927 +2024/02/20 11:09:09 - mmengine - INFO - Iter(train) [ 4960/13954] lr: 7.4708e-04 eta: 10:48:50 time: 4.3543 data_time: 0.0132 memory: 35102 loss: 2.8098 +2024/02/20 11:09:52 - mmengine - INFO - Iter(train) [ 4970/13954] lr: 7.4607e-04 eta: 10:48:07 time: 4.3412 data_time: 0.0131 memory: 34774 loss: 2.7700 +2024/02/20 11:10:36 - mmengine - INFO - Iter(train) [ 4980/13954] lr: 7.4506e-04 eta: 10:47:24 time: 4.3133 data_time: 0.0131 memory: 34774 loss: 2.8586 +2024/02/20 11:11:19 - mmengine - INFO - Iter(train) [ 4990/13954] lr: 7.4405e-04 eta: 10:46:40 time: 4.3122 data_time: 0.0133 memory: 34622 loss: 2.8227 +2024/02/20 11:12:02 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 11:12:02 - mmengine - INFO - Iter(train) [ 5000/13954] lr: 7.4303e-04 eta: 10:45:57 time: 4.3187 data_time: 0.0132 memory: 34915 loss: 2.8905 +2024/02/20 11:12:02 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 11:12:02 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat and a boat house<|im_end|> + +2024/02/20 11:12:03 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the water<|im_end|> + +2024/02/20 11:12:03 - mmengine - INFO - Saving checkpoint at 5000 iterations +2024/02/20 11:12:46 - mmengine - INFO - Iter(train) [ 5010/13954] lr: 7.4202e-04 eta: 10:45:15 time: 4.4142 data_time: 0.0972 memory: 34680 loss: 2.8716 +2024/02/20 11:13:29 - mmengine - INFO - Iter(train) [ 5020/13954] lr: 7.4100e-04 eta: 10:44:31 time: 4.3137 data_time: 0.0130 memory: 34727 loss: 2.7601 +2024/02/20 11:14:13 - mmengine - INFO - Iter(train) [ 5030/13954] lr: 7.3999e-04 eta: 10:43:48 time: 4.3363 data_time: 0.0132 memory: 34822 loss: 2.9411 +2024/02/20 11:14:56 - mmengine - INFO - Iter(train) [ 5040/13954] lr: 7.3897e-04 eta: 10:43:05 time: 4.3079 data_time: 0.0130 memory: 34868 loss: 2.8479 +2024/02/20 11:15:39 - mmengine - INFO - Iter(train) [ 5050/13954] lr: 7.3795e-04 eta: 10:42:21 time: 4.3190 data_time: 0.0130 memory: 34727 loss: 2.8246 +2024/02/20 11:16:22 - mmengine - INFO - Iter(train) [ 5060/13954] lr: 7.3693e-04 eta: 10:41:38 time: 4.3259 data_time: 0.0132 memory: 34540 loss: 2.8195 +2024/02/20 11:17:05 - mmengine - INFO - Iter(train) [ 5070/13954] lr: 7.3590e-04 eta: 10:40:54 time: 4.3297 data_time: 0.0129 memory: 34727 loss: 2.8232 +2024/02/20 11:17:49 - mmengine - INFO - Iter(train) [ 5080/13954] lr: 7.3488e-04 eta: 10:40:11 time: 4.3269 data_time: 0.0130 memory: 34680 loss: 2.8662 +2024/02/20 11:18:32 - mmengine - INFO - Iter(train) [ 5090/13954] lr: 7.3385e-04 eta: 10:39:28 time: 4.3132 data_time: 0.0130 memory: 34727 loss: 2.8056 +2024/02/20 11:19:15 - mmengine - INFO - Iter(train) [ 5100/13954] lr: 7.3283e-04 eta: 10:38:44 time: 4.3156 data_time: 0.0129 memory: 34668 loss: 2.7701 +2024/02/20 11:19:58 - mmengine - INFO - Iter(train) [ 5110/13954] lr: 7.3180e-04 eta: 10:38:01 time: 4.3357 data_time: 0.0130 memory: 35617 loss: 2.7672 +2024/02/20 11:20:41 - mmengine - INFO - Iter(train) [ 5120/13954] lr: 7.3077e-04 eta: 10:37:17 time: 4.3098 data_time: 0.0131 memory: 34680 loss: 2.8226 +2024/02/20 11:21:25 - mmengine - INFO - Iter(train) [ 5130/13954] lr: 7.2974e-04 eta: 10:36:34 time: 4.3254 data_time: 0.0130 memory: 34962 loss: 2.8951 +2024/02/20 11:22:08 - mmengine - INFO - Iter(train) [ 5140/13954] lr: 7.2871e-04 eta: 10:35:50 time: 4.3097 data_time: 0.0132 memory: 34774 loss: 2.8484 +2024/02/20 11:22:51 - mmengine - INFO - Iter(train) [ 5150/13954] lr: 7.2768e-04 eta: 10:35:07 time: 4.3211 data_time: 0.0135 memory: 34634 loss: 2.9132 +2024/02/20 11:23:34 - mmengine - INFO - Iter(train) [ 5160/13954] lr: 7.2664e-04 eta: 10:34:23 time: 4.3115 data_time: 0.0129 memory: 34774 loss: 2.8780 +2024/02/20 11:24:17 - mmengine - INFO - Iter(train) [ 5170/13954] lr: 7.2561e-04 eta: 10:33:40 time: 4.3152 data_time: 0.0133 memory: 34668 loss: 2.8941 +2024/02/20 11:25:00 - mmengine - INFO - Iter(train) [ 5180/13954] lr: 7.2457e-04 eta: 10:32:56 time: 4.3075 data_time: 0.0132 memory: 34915 loss: 2.7736 +2024/02/20 11:25:44 - mmengine - INFO - Iter(train) [ 5190/13954] lr: 7.2353e-04 eta: 10:32:13 time: 4.3269 data_time: 0.0131 memory: 34915 loss: 2.8586 +2024/02/20 11:26:27 - mmengine - INFO - Iter(train) [ 5200/13954] lr: 7.2250e-04 eta: 10:31:29 time: 4.3037 data_time: 0.0133 memory: 34915 loss: 2.7942 +2024/02/20 11:27:10 - mmengine - INFO - Iter(train) [ 5210/13954] lr: 7.2146e-04 eta: 10:30:46 time: 4.3185 data_time: 0.0132 memory: 34727 loss: 2.8710 +2024/02/20 11:27:53 - mmengine - INFO - Iter(train) [ 5220/13954] lr: 7.2042e-04 eta: 10:30:02 time: 4.3081 data_time: 0.0131 memory: 34589 loss: 2.7992 +2024/02/20 11:28:36 - mmengine - INFO - Iter(train) [ 5230/13954] lr: 7.1937e-04 eta: 10:29:19 time: 4.3358 data_time: 0.0130 memory: 34727 loss: 2.8097 +2024/02/20 11:29:20 - mmengine - INFO - Iter(train) [ 5240/13954] lr: 7.1833e-04 eta: 10:28:36 time: 4.3297 data_time: 0.0132 memory: 34915 loss: 2.7952 +2024/02/20 11:30:02 - mmengine - INFO - Iter(train) [ 5250/13954] lr: 7.1729e-04 eta: 10:27:52 time: 4.2934 data_time: 0.0130 memory: 34680 loss: 2.8134 +2024/02/20 11:30:46 - mmengine - INFO - Iter(train) [ 5260/13954] lr: 7.1624e-04 eta: 10:27:09 time: 4.3385 data_time: 0.0129 memory: 35523 loss: 2.8164 +2024/02/20 11:31:29 - mmengine - INFO - Iter(train) [ 5270/13954] lr: 7.1519e-04 eta: 10:26:25 time: 4.3033 data_time: 0.0130 memory: 34634 loss: 2.8188 +2024/02/20 11:32:12 - mmengine - INFO - Iter(train) [ 5280/13954] lr: 7.1414e-04 eta: 10:25:41 time: 4.2832 data_time: 0.0128 memory: 34589 loss: 2.7220 +2024/02/20 11:32:55 - mmengine - INFO - Iter(train) [ 5290/13954] lr: 7.1310e-04 eta: 10:24:57 time: 4.2994 data_time: 0.0133 memory: 34680 loss: 2.7560 +2024/02/20 11:33:38 - mmengine - INFO - Iter(train) [ 5300/13954] lr: 7.1205e-04 eta: 10:24:14 time: 4.3181 data_time: 0.0132 memory: 34822 loss: 2.7716 +2024/02/20 11:34:21 - mmengine - INFO - Iter(train) [ 5310/13954] lr: 7.1099e-04 eta: 10:23:31 time: 4.3284 data_time: 0.0132 memory: 34869 loss: 2.8457 +2024/02/20 11:35:04 - mmengine - INFO - Iter(train) [ 5320/13954] lr: 7.0994e-04 eta: 10:22:47 time: 4.3203 data_time: 0.0133 memory: 34727 loss: 2.8789 +2024/02/20 11:35:47 - mmengine - INFO - Iter(train) [ 5330/13954] lr: 7.0889e-04 eta: 10:22:04 time: 4.3117 data_time: 0.0132 memory: 34774 loss: 2.9025 +2024/02/20 11:36:30 - mmengine - INFO - Iter(train) [ 5340/13954] lr: 7.0783e-04 eta: 10:21:20 time: 4.2989 data_time: 0.0132 memory: 34680 loss: 2.7829 +2024/02/20 11:37:14 - mmengine - INFO - Iter(train) [ 5350/13954] lr: 7.0678e-04 eta: 10:20:36 time: 4.3075 data_time: 0.0133 memory: 35009 loss: 2.7968 +2024/02/20 11:37:57 - mmengine - INFO - Iter(train) [ 5360/13954] lr: 7.0572e-04 eta: 10:19:53 time: 4.3065 data_time: 0.0133 memory: 34680 loss: 2.9301 +2024/02/20 11:38:40 - mmengine - INFO - Iter(train) [ 5370/13954] lr: 7.0466e-04 eta: 10:19:09 time: 4.3186 data_time: 0.0133 memory: 35149 loss: 2.7518 +2024/02/20 11:39:23 - mmengine - INFO - Iter(train) [ 5380/13954] lr: 7.0360e-04 eta: 10:18:26 time: 4.3377 data_time: 0.0132 memory: 34822 loss: 2.7678 +2024/02/20 11:40:06 - mmengine - INFO - Iter(train) [ 5390/13954] lr: 7.0254e-04 eta: 10:17:42 time: 4.2943 data_time: 0.0131 memory: 35289 loss: 2.9090 +2024/02/20 11:40:49 - mmengine - INFO - Iter(train) [ 5400/13954] lr: 7.0148e-04 eta: 10:16:59 time: 4.2967 data_time: 0.0133 memory: 34774 loss: 2.8933 +2024/02/20 11:41:32 - mmengine - INFO - Iter(train) [ 5410/13954] lr: 7.0042e-04 eta: 10:16:15 time: 4.3060 data_time: 0.0134 memory: 34727 loss: 2.7510 +2024/02/20 11:42:15 - mmengine - INFO - Iter(train) [ 5420/13954] lr: 6.9935e-04 eta: 10:15:31 time: 4.3055 data_time: 0.0132 memory: 34589 loss: 2.8955 +2024/02/20 11:42:58 - mmengine - INFO - Iter(train) [ 5430/13954] lr: 6.9829e-04 eta: 10:14:47 time: 4.2809 data_time: 0.0133 memory: 34634 loss: 2.8196 +2024/02/20 11:43:41 - mmengine - INFO - Iter(train) [ 5440/13954] lr: 6.9722e-04 eta: 10:14:04 time: 4.3074 data_time: 0.0131 memory: 34962 loss: 2.7809 +2024/02/20 11:44:24 - mmengine - INFO - Iter(train) [ 5450/13954] lr: 6.9616e-04 eta: 10:13:20 time: 4.2965 data_time: 0.0130 memory: 34680 loss: 2.8389 +2024/02/20 11:45:07 - mmengine - INFO - Iter(train) [ 5460/13954] lr: 6.9509e-04 eta: 10:12:37 time: 4.3192 data_time: 0.0131 memory: 35324 loss: 2.8483 +2024/02/20 11:45:50 - mmengine - INFO - Iter(train) [ 5470/13954] lr: 6.9402e-04 eta: 10:11:53 time: 4.3086 data_time: 0.0131 memory: 36168 loss: 2.7938 +2024/02/20 11:46:33 - mmengine - INFO - Iter(train) [ 5480/13954] lr: 6.9295e-04 eta: 10:11:09 time: 4.2769 data_time: 0.0131 memory: 34589 loss: 2.8554 +2024/02/20 11:47:16 - mmengine - INFO - Iter(train) [ 5490/13954] lr: 6.9188e-04 eta: 10:10:26 time: 4.3237 data_time: 0.0132 memory: 35852 loss: 2.8552 +2024/02/20 11:47:59 - mmengine - INFO - Iter(train) [ 5500/13954] lr: 6.9081e-04 eta: 10:09:42 time: 4.2832 data_time: 0.0132 memory: 34774 loss: 2.7649 +2024/02/20 11:47:59 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 11:47:59 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock and pier on a lake with a boat<|im_end|> + +2024/02/20 11:48:00 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock with a boat on the water<|im_end|> + +2024/02/20 11:48:00 - mmengine - INFO - Saving checkpoint at 5500 iterations +2024/02/20 11:48:43 - mmengine - INFO - Iter(train) [ 5510/13954] lr: 6.8973e-04 eta: 10:08:59 time: 4.3652 data_time: 0.0856 memory: 34810 loss: 2.7930 +2024/02/20 11:49:26 - mmengine - INFO - Iter(train) [ 5520/13954] lr: 6.8866e-04 eta: 10:08:15 time: 4.3127 data_time: 0.0134 memory: 34634 loss: 2.8935 +2024/02/20 11:50:09 - mmengine - INFO - Iter(train) [ 5530/13954] lr: 6.8758e-04 eta: 10:07:32 time: 4.2914 data_time: 0.0136 memory: 34774 loss: 2.9195 +2024/02/20 11:50:52 - mmengine - INFO - Iter(train) [ 5540/13954] lr: 6.8651e-04 eta: 10:06:48 time: 4.3114 data_time: 0.0135 memory: 34727 loss: 2.7987 +2024/02/20 11:51:35 - mmengine - INFO - Iter(train) [ 5550/13954] lr: 6.8543e-04 eta: 10:06:05 time: 4.3318 data_time: 0.0133 memory: 34774 loss: 2.7892 +2024/02/20 11:52:19 - mmengine - INFO - Iter(train) [ 5560/13954] lr: 6.8435e-04 eta: 10:05:22 time: 4.3277 data_time: 0.0135 memory: 34774 loss: 2.8650 +2024/02/20 11:53:02 - mmengine - INFO - Iter(train) [ 5570/13954] lr: 6.8327e-04 eta: 10:04:38 time: 4.3143 data_time: 0.0135 memory: 35102 loss: 2.6897 +2024/02/20 11:53:45 - mmengine - INFO - Iter(train) [ 5580/13954] lr: 6.8219e-04 eta: 10:03:55 time: 4.3250 data_time: 0.0132 memory: 34727 loss: 2.8101 +2024/02/20 11:54:28 - mmengine - INFO - Iter(train) [ 5590/13954] lr: 6.8111e-04 eta: 10:03:11 time: 4.2961 data_time: 0.0133 memory: 34774 loss: 2.8592 +2024/02/20 11:55:11 - mmengine - INFO - Iter(train) [ 5600/13954] lr: 6.8003e-04 eta: 10:02:28 time: 4.3109 data_time: 0.0133 memory: 34634 loss: 2.8313 +2024/02/20 11:55:54 - mmengine - INFO - Iter(train) [ 5610/13954] lr: 6.7895e-04 eta: 10:01:45 time: 4.3351 data_time: 0.0133 memory: 35476 loss: 2.7835 +2024/02/20 11:56:37 - mmengine - INFO - Iter(train) [ 5620/13954] lr: 6.7786e-04 eta: 10:01:01 time: 4.3118 data_time: 0.0132 memory: 34680 loss: 2.8485 +2024/02/20 11:57:20 - mmengine - INFO - Iter(train) [ 5630/13954] lr: 6.7678e-04 eta: 10:00:17 time: 4.2963 data_time: 0.0133 memory: 34774 loss: 2.7788 +2024/02/20 11:58:04 - mmengine - INFO - Iter(train) [ 5640/13954] lr: 6.7569e-04 eta: 9:59:34 time: 4.3223 data_time: 0.0133 memory: 34727 loss: 2.7655 +2024/02/20 11:58:47 - mmengine - INFO - Iter(train) [ 5650/13954] lr: 6.7460e-04 eta: 9:58:50 time: 4.2962 data_time: 0.0135 memory: 34589 loss: 2.8064 +2024/02/20 11:59:30 - mmengine - INFO - Iter(train) [ 5660/13954] lr: 6.7352e-04 eta: 9:58:07 time: 4.3059 data_time: 0.0134 memory: 34774 loss: 2.7426 +2024/02/20 12:00:13 - mmengine - INFO - Iter(train) [ 5670/13954] lr: 6.7243e-04 eta: 9:57:23 time: 4.3163 data_time: 0.0135 memory: 34727 loss: 2.8814 +2024/02/20 12:00:56 - mmengine - INFO - Iter(train) [ 5680/13954] lr: 6.7134e-04 eta: 9:56:40 time: 4.3220 data_time: 0.0135 memory: 35055 loss: 2.7694 +2024/02/20 12:01:39 - mmengine - INFO - Iter(train) [ 5690/13954] lr: 6.7025e-04 eta: 9:55:57 time: 4.3209 data_time: 0.0135 memory: 34727 loss: 2.8444 +2024/02/20 12:02:23 - mmengine - INFO - Iter(train) [ 5700/13954] lr: 6.6915e-04 eta: 9:55:13 time: 4.3274 data_time: 0.0133 memory: 34810 loss: 2.8676 +2024/02/20 12:03:06 - mmengine - INFO - Iter(train) [ 5710/13954] lr: 6.6806e-04 eta: 9:54:30 time: 4.3250 data_time: 0.0133 memory: 34589 loss: 2.7557 +2024/02/20 12:03:49 - mmengine - INFO - Iter(train) [ 5720/13954] lr: 6.6697e-04 eta: 9:53:47 time: 4.3196 data_time: 0.0133 memory: 35009 loss: 2.8898 +2024/02/20 12:04:32 - mmengine - INFO - Iter(train) [ 5730/13954] lr: 6.6587e-04 eta: 9:53:03 time: 4.3169 data_time: 0.0133 memory: 34540 loss: 2.7398 +2024/02/20 12:05:15 - mmengine - INFO - Iter(train) [ 5740/13954] lr: 6.6478e-04 eta: 9:52:20 time: 4.3131 data_time: 0.0134 memory: 34540 loss: 2.8323 +2024/02/20 12:05:58 - mmengine - INFO - Iter(train) [ 5750/13954] lr: 6.6368e-04 eta: 9:51:36 time: 4.2993 data_time: 0.0131 memory: 34668 loss: 2.7651 +2024/02/20 12:06:41 - mmengine - INFO - Iter(train) [ 5760/13954] lr: 6.6259e-04 eta: 9:50:53 time: 4.3083 data_time: 0.0133 memory: 34727 loss: 2.8617 +2024/02/20 12:07:25 - mmengine - INFO - Iter(train) [ 5770/13954] lr: 6.6149e-04 eta: 9:50:09 time: 4.3372 data_time: 0.0131 memory: 34962 loss: 2.7980 +2024/02/20 12:08:08 - mmengine - INFO - Iter(train) [ 5780/13954] lr: 6.6039e-04 eta: 9:49:26 time: 4.3108 data_time: 0.0133 memory: 34727 loss: 2.8914 +2024/02/20 12:08:51 - mmengine - INFO - Iter(train) [ 5790/13954] lr: 6.5929e-04 eta: 9:48:42 time: 4.2908 data_time: 0.0134 memory: 34727 loss: 2.7199 +2024/02/20 12:09:34 - mmengine - INFO - Iter(train) [ 5800/13954] lr: 6.5819e-04 eta: 9:47:58 time: 4.2957 data_time: 0.0133 memory: 34634 loss: 2.7221 +2024/02/20 12:10:17 - mmengine - INFO - Iter(train) [ 5810/13954] lr: 6.5709e-04 eta: 9:47:15 time: 4.3081 data_time: 0.0134 memory: 34634 loss: 2.7816 +2024/02/20 12:11:00 - mmengine - INFO - Iter(train) [ 5820/13954] lr: 6.5599e-04 eta: 9:46:31 time: 4.2931 data_time: 0.0133 memory: 34680 loss: 2.7779 +2024/02/20 12:11:43 - mmengine - INFO - Iter(train) [ 5830/13954] lr: 6.5488e-04 eta: 9:45:48 time: 4.3257 data_time: 0.0133 memory: 34822 loss: 2.7733 +2024/02/20 12:12:26 - mmengine - INFO - Iter(train) [ 5840/13954] lr: 6.5378e-04 eta: 9:45:04 time: 4.2736 data_time: 0.0133 memory: 34589 loss: 2.7624 +2024/02/20 12:13:09 - mmengine - INFO - Iter(train) [ 5850/13954] lr: 6.5268e-04 eta: 9:44:20 time: 4.3125 data_time: 0.0135 memory: 34589 loss: 2.8396 +2024/02/20 12:13:52 - mmengine - INFO - Iter(train) [ 5860/13954] lr: 6.5157e-04 eta: 9:43:37 time: 4.3090 data_time: 0.0136 memory: 34528 loss: 2.9098 +2024/02/20 12:14:35 - mmengine - INFO - Iter(train) [ 5870/13954] lr: 6.5046e-04 eta: 9:42:54 time: 4.3260 data_time: 0.0137 memory: 34915 loss: 2.8292 +2024/02/20 12:15:18 - mmengine - INFO - Iter(train) [ 5880/13954] lr: 6.4936e-04 eta: 9:42:10 time: 4.2900 data_time: 0.0131 memory: 34668 loss: 2.7907 +2024/02/20 12:16:01 - mmengine - INFO - Iter(train) [ 5890/13954] lr: 6.4825e-04 eta: 9:41:26 time: 4.2987 data_time: 0.0128 memory: 34810 loss: 2.8133 +2024/02/20 12:16:44 - mmengine - INFO - Iter(train) [ 5900/13954] lr: 6.4714e-04 eta: 9:40:43 time: 4.3193 data_time: 0.0130 memory: 34715 loss: 2.8220 +2024/02/20 12:17:27 - mmengine - INFO - Iter(train) [ 5910/13954] lr: 6.4603e-04 eta: 9:39:59 time: 4.2881 data_time: 0.0129 memory: 34634 loss: 2.8033 +2024/02/20 12:18:10 - mmengine - INFO - Iter(train) [ 5920/13954] lr: 6.4492e-04 eta: 9:39:16 time: 4.3158 data_time: 0.0130 memory: 35617 loss: 2.8550 +2024/02/20 12:18:53 - mmengine - INFO - Iter(train) [ 5930/13954] lr: 6.4381e-04 eta: 9:38:32 time: 4.3047 data_time: 0.0130 memory: 34634 loss: 2.8448 +2024/02/20 12:19:36 - mmengine - INFO - Iter(train) [ 5940/13954] lr: 6.4270e-04 eta: 9:37:49 time: 4.2964 data_time: 0.0130 memory: 34680 loss: 2.8705 +2024/02/20 12:20:19 - mmengine - INFO - Iter(train) [ 5950/13954] lr: 6.4158e-04 eta: 9:37:05 time: 4.3084 data_time: 0.0130 memory: 34774 loss: 2.7933 +2024/02/20 12:21:02 - mmengine - INFO - Iter(train) [ 5960/13954] lr: 6.4047e-04 eta: 9:36:21 time: 4.3042 data_time: 0.0129 memory: 34727 loss: 2.8255 +2024/02/20 12:21:46 - mmengine - INFO - Iter(train) [ 5970/13954] lr: 6.3936e-04 eta: 9:35:38 time: 4.3176 data_time: 0.0129 memory: 35523 loss: 2.8122 +2024/02/20 12:22:29 - mmengine - INFO - Iter(train) [ 5980/13954] lr: 6.3824e-04 eta: 9:34:55 time: 4.3131 data_time: 0.0130 memory: 34962 loss: 2.8739 +2024/02/20 12:23:12 - mmengine - INFO - Iter(train) [ 5990/13954] lr: 6.3713e-04 eta: 9:34:11 time: 4.3170 data_time: 0.0129 memory: 34915 loss: 2.7592 +2024/02/20 12:23:55 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 12:23:55 - mmengine - INFO - Iter(train) [ 6000/13954] lr: 6.3601e-04 eta: 9:33:28 time: 4.3273 data_time: 0.0129 memory: 35009 loss: 2.7995 +2024/02/20 12:23:55 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 12:23:56 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on the water<|im_end|> + +2024/02/20 12:23:56 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on the water<|im_end|> + +2024/02/20 12:23:56 - mmengine - INFO - Saving checkpoint at 6000 iterations +2024/02/20 12:24:39 - mmengine - INFO - Iter(train) [ 6010/13954] lr: 6.3489e-04 eta: 9:32:46 time: 4.3862 data_time: 0.0949 memory: 34589 loss: 2.9470 +2024/02/20 12:25:22 - mmengine - INFO - Iter(train) [ 6020/13954] lr: 6.3378e-04 eta: 9:32:02 time: 4.3033 data_time: 0.0134 memory: 35009 loss: 2.8289 +2024/02/20 12:26:05 - mmengine - INFO - Iter(train) [ 6030/13954] lr: 6.3266e-04 eta: 9:31:19 time: 4.3249 data_time: 0.0134 memory: 35242 loss: 2.7475 +2024/02/20 12:26:48 - mmengine - INFO - Iter(train) [ 6040/13954] lr: 6.3154e-04 eta: 9:30:35 time: 4.2889 data_time: 0.0135 memory: 35336 loss: 2.8513 +2024/02/20 12:27:31 - mmengine - INFO - Iter(train) [ 6050/13954] lr: 6.3042e-04 eta: 9:29:52 time: 4.3134 data_time: 0.0135 memory: 34822 loss: 2.8497 +2024/02/20 12:28:14 - mmengine - INFO - Iter(train) [ 6060/13954] lr: 6.2930e-04 eta: 9:29:08 time: 4.2878 data_time: 0.0130 memory: 34589 loss: 2.8199 +2024/02/20 12:28:57 - mmengine - INFO - Iter(train) [ 6070/13954] lr: 6.2818e-04 eta: 9:28:24 time: 4.3048 data_time: 0.0130 memory: 34680 loss: 2.7555 +2024/02/20 12:29:40 - mmengine - INFO - Iter(train) [ 6080/13954] lr: 6.2705e-04 eta: 9:27:41 time: 4.3035 data_time: 0.0132 memory: 34680 loss: 2.7999 +2024/02/20 12:30:23 - mmengine - INFO - Iter(train) [ 6090/13954] lr: 6.2593e-04 eta: 9:26:57 time: 4.3051 data_time: 0.0130 memory: 34634 loss: 2.8540 +2024/02/20 12:31:07 - mmengine - INFO - Iter(train) [ 6100/13954] lr: 6.2481e-04 eta: 9:26:14 time: 4.3176 data_time: 0.0130 memory: 34622 loss: 2.7393 +2024/02/20 12:31:50 - mmengine - INFO - Iter(train) [ 6110/13954] lr: 6.2368e-04 eta: 9:25:31 time: 4.3250 data_time: 0.0131 memory: 34774 loss: 2.8119 +2024/02/20 12:32:33 - mmengine - INFO - Iter(train) [ 6120/13954] lr: 6.2256e-04 eta: 9:24:47 time: 4.3210 data_time: 0.0129 memory: 34680 loss: 2.8589 +2024/02/20 12:33:16 - mmengine - INFO - Iter(train) [ 6130/13954] lr: 6.2143e-04 eta: 9:24:04 time: 4.3353 data_time: 0.0130 memory: 34915 loss: 2.7789 +2024/02/20 12:34:00 - mmengine - INFO - Iter(train) [ 6140/13954] lr: 6.2031e-04 eta: 9:23:21 time: 4.3528 data_time: 0.0132 memory: 35009 loss: 2.8060 +2024/02/20 12:34:43 - mmengine - INFO - Iter(train) [ 6150/13954] lr: 6.1918e-04 eta: 9:22:38 time: 4.3278 data_time: 0.0131 memory: 34680 loss: 2.7992 +2024/02/20 12:35:26 - mmengine - INFO - Iter(train) [ 6160/13954] lr: 6.1805e-04 eta: 9:21:55 time: 4.3180 data_time: 0.0133 memory: 34727 loss: 2.8114 +2024/02/20 12:36:10 - mmengine - INFO - Iter(train) [ 6170/13954] lr: 6.1693e-04 eta: 9:21:12 time: 4.3445 data_time: 0.0132 memory: 34774 loss: 2.7468 +2024/02/20 12:36:53 - mmengine - INFO - Iter(train) [ 6180/13954] lr: 6.1580e-04 eta: 9:20:28 time: 4.3067 data_time: 0.0129 memory: 34622 loss: 2.8100 +2024/02/20 12:37:36 - mmengine - INFO - Iter(train) [ 6190/13954] lr: 6.1467e-04 eta: 9:19:45 time: 4.3071 data_time: 0.0131 memory: 34762 loss: 2.7887 +2024/02/20 12:38:19 - mmengine - INFO - Iter(train) [ 6200/13954] lr: 6.1354e-04 eta: 9:19:01 time: 4.3076 data_time: 0.0132 memory: 34774 loss: 2.8335 +2024/02/20 12:39:02 - mmengine - INFO - Iter(train) [ 6210/13954] lr: 6.1241e-04 eta: 9:18:18 time: 4.3202 data_time: 0.0132 memory: 34668 loss: 2.8653 +2024/02/20 12:39:45 - mmengine - INFO - Iter(train) [ 6220/13954] lr: 6.1128e-04 eta: 9:17:35 time: 4.3212 data_time: 0.0132 memory: 34622 loss: 2.8274 +2024/02/20 12:40:29 - mmengine - INFO - Iter(train) [ 6230/13954] lr: 6.1014e-04 eta: 9:16:51 time: 4.3177 data_time: 0.0132 memory: 34810 loss: 2.8409 +2024/02/20 12:41:12 - mmengine - INFO - Iter(train) [ 6240/13954] lr: 6.0901e-04 eta: 9:16:08 time: 4.3327 data_time: 0.0132 memory: 34822 loss: 2.8944 +2024/02/20 12:41:55 - mmengine - INFO - Iter(train) [ 6250/13954] lr: 6.0788e-04 eta: 9:15:25 time: 4.3489 data_time: 0.0132 memory: 34915 loss: 2.8572 +2024/02/20 12:42:39 - mmengine - INFO - Iter(train) [ 6260/13954] lr: 6.0675e-04 eta: 9:14:42 time: 4.3088 data_time: 0.0131 memory: 34822 loss: 2.8564 +2024/02/20 12:43:22 - mmengine - INFO - Iter(train) [ 6270/13954] lr: 6.0561e-04 eta: 9:13:58 time: 4.3074 data_time: 0.0128 memory: 34822 loss: 2.8559 +2024/02/20 12:44:05 - mmengine - INFO - Iter(train) [ 6280/13954] lr: 6.0448e-04 eta: 9:13:14 time: 4.2967 data_time: 0.0127 memory: 34868 loss: 2.7523 +2024/02/20 12:44:47 - mmengine - INFO - Iter(train) [ 6290/13954] lr: 6.0334e-04 eta: 9:12:31 time: 4.2882 data_time: 0.0131 memory: 34680 loss: 2.8576 +2024/02/20 12:45:30 - mmengine - INFO - Iter(train) [ 6300/13954] lr: 6.0221e-04 eta: 9:11:47 time: 4.2991 data_time: 0.0136 memory: 34634 loss: 2.8864 +2024/02/20 12:46:14 - mmengine - INFO - Iter(train) [ 6310/13954] lr: 6.0107e-04 eta: 9:11:04 time: 4.3233 data_time: 0.0133 memory: 34668 loss: 2.7124 +2024/02/20 12:46:57 - mmengine - INFO - Iter(train) [ 6320/13954] lr: 5.9993e-04 eta: 9:10:20 time: 4.2948 data_time: 0.0132 memory: 34868 loss: 2.8108 +2024/02/20 12:47:40 - mmengine - INFO - Iter(train) [ 6330/13954] lr: 5.9880e-04 eta: 9:09:37 time: 4.3499 data_time: 0.0129 memory: 35009 loss: 2.7943 +2024/02/20 12:48:24 - mmengine - INFO - Iter(train) [ 6340/13954] lr: 5.9766e-04 eta: 9:08:54 time: 4.3520 data_time: 0.0130 memory: 34762 loss: 2.7907 +2024/02/20 12:49:07 - mmengine - INFO - Iter(train) [ 6350/13954] lr: 5.9652e-04 eta: 9:08:11 time: 4.3198 data_time: 0.0131 memory: 34822 loss: 2.7281 +2024/02/20 12:49:50 - mmengine - INFO - Iter(train) [ 6360/13954] lr: 5.9538e-04 eta: 9:07:28 time: 4.3033 data_time: 0.0130 memory: 34727 loss: 2.7822 +2024/02/20 12:50:33 - mmengine - INFO - Iter(train) [ 6370/13954] lr: 5.9424e-04 eta: 9:06:44 time: 4.2998 data_time: 0.0132 memory: 34634 loss: 2.8694 +2024/02/20 12:51:16 - mmengine - INFO - Iter(train) [ 6380/13954] lr: 5.9310e-04 eta: 9:06:01 time: 4.3354 data_time: 0.0130 memory: 34822 loss: 2.8521 +2024/02/20 12:51:59 - mmengine - INFO - Iter(train) [ 6390/13954] lr: 5.9196e-04 eta: 9:05:17 time: 4.3044 data_time: 0.0128 memory: 34633 loss: 2.9144 +2024/02/20 12:52:42 - mmengine - INFO - Iter(train) [ 6400/13954] lr: 5.9082e-04 eta: 9:04:34 time: 4.2941 data_time: 0.0129 memory: 34589 loss: 2.8363 +2024/02/20 12:53:25 - mmengine - INFO - Iter(train) [ 6410/13954] lr: 5.8968e-04 eta: 9:03:50 time: 4.3206 data_time: 0.0131 memory: 34680 loss: 2.8725 +2024/02/20 12:54:09 - mmengine - INFO - Iter(train) [ 6420/13954] lr: 5.8854e-04 eta: 9:03:07 time: 4.3439 data_time: 0.0132 memory: 35946 loss: 2.7725 +2024/02/20 12:54:52 - mmengine - INFO - Iter(train) [ 6430/13954] lr: 5.8740e-04 eta: 9:02:24 time: 4.3339 data_time: 0.0130 memory: 34774 loss: 2.7498 +2024/02/20 12:55:35 - mmengine - INFO - Iter(train) [ 6440/13954] lr: 5.8625e-04 eta: 9:01:41 time: 4.3177 data_time: 0.0131 memory: 34589 loss: 2.7510 +2024/02/20 12:56:19 - mmengine - INFO - Iter(train) [ 6450/13954] lr: 5.8511e-04 eta: 9:00:58 time: 4.3242 data_time: 0.0132 memory: 34715 loss: 2.7500 +2024/02/20 12:57:02 - mmengine - INFO - Iter(train) [ 6460/13954] lr: 5.8397e-04 eta: 9:00:14 time: 4.3157 data_time: 0.0131 memory: 34962 loss: 2.8609 +2024/02/20 12:57:45 - mmengine - INFO - Iter(train) [ 6470/13954] lr: 5.8282e-04 eta: 8:59:31 time: 4.3604 data_time: 0.0130 memory: 36027 loss: 2.7264 +2024/02/20 12:58:29 - mmengine - INFO - Iter(train) [ 6480/13954] lr: 5.8168e-04 eta: 8:58:48 time: 4.3152 data_time: 0.0130 memory: 34493 loss: 2.7543 +2024/02/20 12:59:12 - mmengine - INFO - Iter(train) [ 6490/13954] lr: 5.8053e-04 eta: 8:58:05 time: 4.3237 data_time: 0.0129 memory: 34822 loss: 2.7852 +2024/02/20 12:59:55 - mmengine - INFO - Iter(train) [ 6500/13954] lr: 5.7939e-04 eta: 8:57:22 time: 4.3408 data_time: 0.0129 memory: 35009 loss: 2.7870 +2024/02/20 12:59:55 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 12:59:56 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + +2024/02/20 12:59:56 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + +2024/02/20 12:59:56 - mmengine - INFO - Saving checkpoint at 6500 iterations +2024/02/20 13:00:39 - mmengine - INFO - Iter(train) [ 6510/13954] lr: 5.7824e-04 eta: 8:56:39 time: 4.3733 data_time: 0.0943 memory: 34950 loss: 2.8624 +2024/02/20 13:01:22 - mmengine - INFO - Iter(train) [ 6520/13954] lr: 5.7709e-04 eta: 8:55:56 time: 4.3392 data_time: 0.0126 memory: 35196 loss: 2.8242 +2024/02/20 13:02:05 - mmengine - INFO - Iter(train) [ 6530/13954] lr: 5.7595e-04 eta: 8:55:12 time: 4.2988 data_time: 0.0128 memory: 34868 loss: 2.8217 +2024/02/20 13:02:49 - mmengine - INFO - Iter(train) [ 6540/13954] lr: 5.7480e-04 eta: 8:54:29 time: 4.3228 data_time: 0.0126 memory: 34774 loss: 2.8200 +2024/02/20 13:03:32 - mmengine - INFO - Iter(train) [ 6550/13954] lr: 5.7365e-04 eta: 8:53:46 time: 4.3156 data_time: 0.0128 memory: 34634 loss: 2.7351 +2024/02/20 13:04:15 - mmengine - INFO - Iter(train) [ 6560/13954] lr: 5.7250e-04 eta: 8:53:02 time: 4.3121 data_time: 0.0130 memory: 34680 loss: 2.8108 +2024/02/20 13:04:58 - mmengine - INFO - Iter(train) [ 6570/13954] lr: 5.7136e-04 eta: 8:52:19 time: 4.3090 data_time: 0.0128 memory: 35102 loss: 2.8443 +2024/02/20 13:05:41 - mmengine - INFO - Iter(train) [ 6580/13954] lr: 5.7021e-04 eta: 8:51:35 time: 4.3160 data_time: 0.0129 memory: 34774 loss: 2.7627 +2024/02/20 13:06:24 - mmengine - INFO - Iter(train) [ 6590/13954] lr: 5.6906e-04 eta: 8:50:52 time: 4.3205 data_time: 0.0127 memory: 34680 loss: 2.7245 +2024/02/20 13:07:07 - mmengine - INFO - Iter(train) [ 6600/13954] lr: 5.6791e-04 eta: 8:50:09 time: 4.3159 data_time: 0.0130 memory: 34727 loss: 2.7308 +2024/02/20 13:07:51 - mmengine - INFO - Iter(train) [ 6610/13954] lr: 5.6676e-04 eta: 8:49:25 time: 4.3170 data_time: 0.0127 memory: 35324 loss: 2.8441 +2024/02/20 13:08:34 - mmengine - INFO - Iter(train) [ 6620/13954] lr: 5.6561e-04 eta: 8:48:42 time: 4.3018 data_time: 0.0128 memory: 34810 loss: 2.8384 +2024/02/20 13:09:17 - mmengine - INFO - Iter(train) [ 6630/13954] lr: 5.6446e-04 eta: 8:47:58 time: 4.3010 data_time: 0.0129 memory: 34903 loss: 2.7759 +2024/02/20 13:10:00 - mmengine - INFO - Iter(train) [ 6640/13954] lr: 5.6331e-04 eta: 8:47:15 time: 4.3150 data_time: 0.0129 memory: 34822 loss: 2.8701 +2024/02/20 13:10:43 - mmengine - INFO - Iter(train) [ 6650/13954] lr: 5.6216e-04 eta: 8:46:31 time: 4.2932 data_time: 0.0126 memory: 34680 loss: 2.8110 +2024/02/20 13:11:26 - mmengine - INFO - Iter(train) [ 6660/13954] lr: 5.6100e-04 eta: 8:45:48 time: 4.3200 data_time: 0.0138 memory: 34915 loss: 2.7283 +2024/02/20 13:12:09 - mmengine - INFO - Iter(train) [ 6670/13954] lr: 5.5985e-04 eta: 8:45:05 time: 4.3254 data_time: 0.0131 memory: 34869 loss: 2.7546 +2024/02/20 13:12:52 - mmengine - INFO - Iter(train) [ 6680/13954] lr: 5.5870e-04 eta: 8:44:21 time: 4.3027 data_time: 0.0130 memory: 34727 loss: 2.7728 +2024/02/20 13:13:35 - mmengine - INFO - Iter(train) [ 6690/13954] lr: 5.5755e-04 eta: 8:43:38 time: 4.3067 data_time: 0.0130 memory: 34822 loss: 2.7622 +2024/02/20 13:14:18 - mmengine - INFO - Iter(train) [ 6700/13954] lr: 5.5639e-04 eta: 8:42:54 time: 4.3163 data_time: 0.0130 memory: 35055 loss: 2.7089 +2024/02/20 13:15:02 - mmengine - INFO - Iter(train) [ 6710/13954] lr: 5.5524e-04 eta: 8:42:11 time: 4.3302 data_time: 0.0129 memory: 34915 loss: 2.7757 +2024/02/20 13:15:45 - mmengine - INFO - Iter(train) [ 6720/13954] lr: 5.5409e-04 eta: 8:41:28 time: 4.3110 data_time: 0.0128 memory: 34774 loss: 2.8747 +2024/02/20 13:16:28 - mmengine - INFO - Iter(train) [ 6730/13954] lr: 5.5293e-04 eta: 8:40:44 time: 4.2869 data_time: 0.0132 memory: 35137 loss: 2.7912 +2024/02/20 13:17:11 - mmengine - INFO - Iter(train) [ 6740/13954] lr: 5.5178e-04 eta: 8:40:01 time: 4.3118 data_time: 0.0130 memory: 34869 loss: 2.7363 +2024/02/20 13:17:54 - mmengine - INFO - Iter(train) [ 6750/13954] lr: 5.5063e-04 eta: 8:39:18 time: 4.3260 data_time: 0.0130 memory: 34727 loss: 2.8198 +2024/02/20 13:18:37 - mmengine - INFO - Iter(train) [ 6760/13954] lr: 5.4947e-04 eta: 8:38:34 time: 4.3260 data_time: 0.0128 memory: 35196 loss: 2.7752 +2024/02/20 13:19:20 - mmengine - INFO - Iter(train) [ 6770/13954] lr: 5.4832e-04 eta: 8:37:51 time: 4.3123 data_time: 0.0128 memory: 34589 loss: 2.7508 +2024/02/20 13:20:04 - mmengine - INFO - Iter(train) [ 6780/13954] lr: 5.4716e-04 eta: 8:37:08 time: 4.3446 data_time: 0.0129 memory: 34727 loss: 2.7564 +2024/02/20 13:20:47 - mmengine - INFO - Iter(train) [ 6790/13954] lr: 5.4601e-04 eta: 8:36:25 time: 4.3335 data_time: 0.0129 memory: 34962 loss: 2.8380 +2024/02/20 13:21:30 - mmengine - INFO - Iter(train) [ 6800/13954] lr: 5.4485e-04 eta: 8:35:41 time: 4.3032 data_time: 0.0128 memory: 34727 loss: 2.7708 +2024/02/20 13:22:13 - mmengine - INFO - Iter(train) [ 6810/13954] lr: 5.4369e-04 eta: 8:34:58 time: 4.3196 data_time: 0.0129 memory: 34680 loss: 2.7799 +2024/02/20 13:22:57 - mmengine - INFO - Iter(train) [ 6820/13954] lr: 5.4254e-04 eta: 8:34:14 time: 4.3033 data_time: 0.0127 memory: 34868 loss: 2.7225 +2024/02/20 13:23:40 - mmengine - INFO - Iter(train) [ 6830/13954] lr: 5.4138e-04 eta: 8:33:31 time: 4.3317 data_time: 0.0128 memory: 34774 loss: 2.8087 +2024/02/20 13:24:23 - mmengine - INFO - Iter(train) [ 6840/13954] lr: 5.4022e-04 eta: 8:32:48 time: 4.3201 data_time: 0.0128 memory: 34868 loss: 2.8060 +2024/02/20 13:25:06 - mmengine - INFO - Iter(train) [ 6850/13954] lr: 5.3907e-04 eta: 8:32:05 time: 4.3192 data_time: 0.0129 memory: 34727 loss: 2.7547 +2024/02/20 13:25:49 - mmengine - INFO - Iter(train) [ 6860/13954] lr: 5.3791e-04 eta: 8:31:21 time: 4.3118 data_time: 0.0128 memory: 34634 loss: 2.7618 +2024/02/20 13:26:32 - mmengine - INFO - Iter(train) [ 6870/13954] lr: 5.3675e-04 eta: 8:30:38 time: 4.3113 data_time: 0.0128 memory: 34481 loss: 2.7823 +2024/02/20 13:27:16 - mmengine - INFO - Iter(train) [ 6880/13954] lr: 5.3560e-04 eta: 8:29:54 time: 4.3063 data_time: 0.0129 memory: 34950 loss: 2.7402 +2024/02/20 13:27:59 - mmengine - INFO - Iter(train) [ 6890/13954] lr: 5.3444e-04 eta: 8:29:11 time: 4.3143 data_time: 0.0130 memory: 34727 loss: 2.8419 +2024/02/20 13:28:42 - mmengine - INFO - Iter(train) [ 6900/13954] lr: 5.3328e-04 eta: 8:28:28 time: 4.3117 data_time: 0.0130 memory: 34903 loss: 2.7662 +2024/02/20 13:29:25 - mmengine - INFO - Iter(train) [ 6910/13954] lr: 5.3212e-04 eta: 8:27:45 time: 4.3348 data_time: 0.0129 memory: 34962 loss: 2.6575 +2024/02/20 13:30:08 - mmengine - INFO - Iter(train) [ 6920/13954] lr: 5.3096e-04 eta: 8:27:01 time: 4.3219 data_time: 0.0131 memory: 34810 loss: 2.8292 +2024/02/20 13:30:52 - mmengine - INFO - Iter(train) [ 6930/13954] lr: 5.2981e-04 eta: 8:26:18 time: 4.3234 data_time: 0.0127 memory: 34727 loss: 2.7611 +2024/02/20 13:31:35 - mmengine - INFO - Iter(train) [ 6940/13954] lr: 5.2865e-04 eta: 8:25:35 time: 4.3367 data_time: 0.0128 memory: 34868 loss: 2.7873 +2024/02/20 13:32:18 - mmengine - INFO - Iter(train) [ 6950/13954] lr: 5.2749e-04 eta: 8:24:52 time: 4.3105 data_time: 0.0126 memory: 34868 loss: 2.8250 +2024/02/20 13:33:01 - mmengine - INFO - Iter(train) [ 6960/13954] lr: 5.2633e-04 eta: 8:24:08 time: 4.3067 data_time: 0.0126 memory: 35056 loss: 2.7481 +2024/02/20 13:33:44 - mmengine - INFO - Iter(train) [ 6970/13954] lr: 5.2517e-04 eta: 8:23:25 time: 4.3156 data_time: 0.0125 memory: 34680 loss: 2.7252 +2024/02/20 13:34:28 - mmengine - INFO - Iter(train) [ 6980/13954] lr: 5.2401e-04 eta: 8:22:42 time: 4.3281 data_time: 0.0125 memory: 36121 loss: 2.7665 +2024/02/20 13:35:10 - mmengine - INFO - Iter(train) [ 6990/13954] lr: 5.2285e-04 eta: 8:21:58 time: 4.2900 data_time: 0.0124 memory: 34822 loss: 2.8636 +2024/02/20 13:35:53 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 13:35:53 - mmengine - INFO - Iter(train) [ 7000/13954] lr: 5.2169e-04 eta: 8:21:14 time: 4.2943 data_time: 0.0125 memory: 34680 loss: 2.7413 +2024/02/20 13:35:53 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 13:35:54 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + +2024/02/20 13:35:54 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + +2024/02/20 13:35:54 - mmengine - INFO - Saving checkpoint at 7000 iterations +2024/02/20 13:36:37 - mmengine - INFO - Iter(train) [ 7010/13954] lr: 5.2053e-04 eta: 8:20:32 time: 4.3788 data_time: 0.0952 memory: 34680 loss: 2.7679 +2024/02/20 13:37:20 - mmengine - INFO - Iter(train) [ 7020/13954] lr: 5.1937e-04 eta: 8:19:48 time: 4.3287 data_time: 0.0123 memory: 34727 loss: 2.6794 +2024/02/20 13:38:04 - mmengine - INFO - Iter(train) [ 7030/13954] lr: 5.1822e-04 eta: 8:19:05 time: 4.3172 data_time: 0.0123 memory: 34822 loss: 2.7621 +2024/02/20 13:38:47 - mmengine - INFO - Iter(train) [ 7040/13954] lr: 5.1706e-04 eta: 8:18:22 time: 4.3378 data_time: 0.0121 memory: 36273 loss: 2.7484 +2024/02/20 13:39:30 - mmengine - INFO - Iter(train) [ 7050/13954] lr: 5.1590e-04 eta: 8:17:39 time: 4.3438 data_time: 0.0120 memory: 34727 loss: 2.7917 +2024/02/20 13:40:14 - mmengine - INFO - Iter(train) [ 7060/13954] lr: 5.1474e-04 eta: 8:16:56 time: 4.3225 data_time: 0.0120 memory: 34727 loss: 2.7739 +2024/02/20 13:40:57 - mmengine - INFO - Iter(train) [ 7070/13954] lr: 5.1358e-04 eta: 8:16:12 time: 4.3126 data_time: 0.0126 memory: 34774 loss: 2.8245 +2024/02/20 13:41:40 - mmengine - INFO - Iter(train) [ 7080/13954] lr: 5.1242e-04 eta: 8:15:29 time: 4.2998 data_time: 0.0122 memory: 35009 loss: 2.6969 +2024/02/20 13:42:23 - mmengine - INFO - Iter(train) [ 7090/13954] lr: 5.1126e-04 eta: 8:14:45 time: 4.3052 data_time: 0.0121 memory: 34774 loss: 2.8236 +2024/02/20 13:43:06 - mmengine - INFO - Iter(train) [ 7100/13954] lr: 5.1010e-04 eta: 8:14:02 time: 4.3238 data_time: 0.0120 memory: 34668 loss: 2.7125 +2024/02/20 13:43:49 - mmengine - INFO - Iter(train) [ 7110/13954] lr: 5.0894e-04 eta: 8:13:19 time: 4.3184 data_time: 0.0120 memory: 35195 loss: 2.8273 +2024/02/20 13:44:32 - mmengine - INFO - Iter(train) [ 7120/13954] lr: 5.0777e-04 eta: 8:12:35 time: 4.2989 data_time: 0.0120 memory: 34962 loss: 2.8162 +2024/02/20 13:45:16 - mmengine - INFO - Iter(train) [ 7130/13954] lr: 5.0661e-04 eta: 8:11:52 time: 4.3319 data_time: 0.0121 memory: 34634 loss: 2.7814 +2024/02/20 13:45:59 - mmengine - INFO - Iter(train) [ 7140/13954] lr: 5.0545e-04 eta: 8:11:09 time: 4.3127 data_time: 0.0122 memory: 34774 loss: 2.8076 +2024/02/20 13:46:42 - mmengine - INFO - Iter(train) [ 7150/13954] lr: 5.0429e-04 eta: 8:10:25 time: 4.3028 data_time: 0.0121 memory: 34774 loss: 2.7278 +2024/02/20 13:47:25 - mmengine - INFO - Iter(train) [ 7160/13954] lr: 5.0313e-04 eta: 8:09:42 time: 4.3059 data_time: 0.0121 memory: 34668 loss: 2.8060 +2024/02/20 13:48:08 - mmengine - INFO - Iter(train) [ 7170/13954] lr: 5.0197e-04 eta: 8:08:59 time: 4.3358 data_time: 0.0122 memory: 34822 loss: 2.7808 +2024/02/20 13:48:51 - mmengine - INFO - Iter(train) [ 7180/13954] lr: 5.0081e-04 eta: 8:08:16 time: 4.3296 data_time: 0.0121 memory: 34680 loss: 2.8181 +2024/02/20 13:49:35 - mmengine - INFO - Iter(train) [ 7190/13954] lr: 4.9965e-04 eta: 8:07:32 time: 4.3249 data_time: 0.0121 memory: 34727 loss: 2.7893 +2024/02/20 13:50:18 - mmengine - INFO - Iter(train) [ 7200/13954] lr: 4.9849e-04 eta: 8:06:49 time: 4.3305 data_time: 0.0121 memory: 34868 loss: 2.8385 +2024/02/20 13:51:01 - mmengine - INFO - Iter(train) [ 7210/13954] lr: 4.9733e-04 eta: 8:06:06 time: 4.3149 data_time: 0.0122 memory: 34822 loss: 2.8402 +2024/02/20 13:51:44 - mmengine - INFO - Iter(train) [ 7220/13954] lr: 4.9617e-04 eta: 8:05:22 time: 4.3055 data_time: 0.0126 memory: 35102 loss: 2.8175 +2024/02/20 13:52:28 - mmengine - INFO - Iter(train) [ 7230/13954] lr: 4.9501e-04 eta: 8:04:39 time: 4.3324 data_time: 0.0133 memory: 34634 loss: 2.8557 +2024/02/20 13:53:11 - mmengine - INFO - Iter(train) [ 7240/13954] lr: 4.9385e-04 eta: 8:03:56 time: 4.3258 data_time: 0.0140 memory: 34915 loss: 2.7312 +2024/02/20 13:53:54 - mmengine - INFO - Iter(train) [ 7250/13954] lr: 4.9269e-04 eta: 8:03:12 time: 4.3003 data_time: 0.0141 memory: 34589 loss: 2.8158 +2024/02/20 13:54:37 - mmengine - INFO - Iter(train) [ 7260/13954] lr: 4.9153e-04 eta: 8:02:29 time: 4.2969 data_time: 0.0136 memory: 35558 loss: 2.8989 +2024/02/20 13:55:20 - mmengine - INFO - Iter(train) [ 7270/13954] lr: 4.9037e-04 eta: 8:01:46 time: 4.3073 data_time: 0.0133 memory: 34773 loss: 2.7614 +2024/02/20 13:56:03 - mmengine - INFO - Iter(train) [ 7280/13954] lr: 4.8921e-04 eta: 8:01:02 time: 4.3367 data_time: 0.0124 memory: 35009 loss: 2.8070 +2024/02/20 13:56:46 - mmengine - INFO - Iter(train) [ 7290/13954] lr: 4.8805e-04 eta: 8:00:19 time: 4.3185 data_time: 0.0127 memory: 34774 loss: 2.8285 +2024/02/20 13:57:30 - mmengine - INFO - Iter(train) [ 7300/13954] lr: 4.8689e-04 eta: 7:59:36 time: 4.3148 data_time: 0.0124 memory: 34589 loss: 2.7673 +2024/02/20 13:58:13 - mmengine - INFO - Iter(train) [ 7310/13954] lr: 4.8573e-04 eta: 7:58:52 time: 4.3104 data_time: 0.0133 memory: 34680 loss: 2.7336 +2024/02/20 13:58:56 - mmengine - INFO - Iter(train) [ 7320/13954] lr: 4.8457e-04 eta: 7:58:09 time: 4.3003 data_time: 0.0140 memory: 34634 loss: 2.8088 +2024/02/20 13:59:39 - mmengine - INFO - Iter(train) [ 7330/13954] lr: 4.8341e-04 eta: 7:57:26 time: 4.3228 data_time: 0.0140 memory: 34727 loss: 2.7706 +2024/02/20 14:00:22 - mmengine - INFO - Iter(train) [ 7340/13954] lr: 4.8225e-04 eta: 7:56:42 time: 4.3239 data_time: 0.0135 memory: 34634 loss: 2.7801 +2024/02/20 14:01:05 - mmengine - INFO - Iter(train) [ 7350/13954] lr: 4.8109e-04 eta: 7:55:59 time: 4.3218 data_time: 0.0131 memory: 34997 loss: 2.8134 +2024/02/20 14:01:48 - mmengine - INFO - Iter(train) [ 7360/13954] lr: 4.7993e-04 eta: 7:55:16 time: 4.2948 data_time: 0.0129 memory: 34589 loss: 2.8745 +2024/02/20 14:02:31 - mmengine - INFO - Iter(train) [ 7370/13954] lr: 4.7877e-04 eta: 7:54:32 time: 4.3022 data_time: 0.0130 memory: 34589 loss: 2.7341 +2024/02/20 14:03:14 - mmengine - INFO - Iter(train) [ 7380/13954] lr: 4.7761e-04 eta: 7:53:49 time: 4.3146 data_time: 0.0131 memory: 34634 loss: 2.8176 +2024/02/20 14:03:58 - mmengine - INFO - Iter(train) [ 7390/13954] lr: 4.7645e-04 eta: 7:53:05 time: 4.3075 data_time: 0.0129 memory: 34680 loss: 2.7818 +2024/02/20 14:04:41 - mmengine - INFO - Iter(train) [ 7400/13954] lr: 4.7529e-04 eta: 7:52:22 time: 4.3149 data_time: 0.0132 memory: 34727 loss: 2.7502 +2024/02/20 14:05:24 - mmengine - INFO - Iter(train) [ 7410/13954] lr: 4.7413e-04 eta: 7:51:39 time: 4.3035 data_time: 0.0130 memory: 34680 loss: 2.8242 +2024/02/20 14:06:07 - mmengine - INFO - Iter(train) [ 7420/13954] lr: 4.7297e-04 eta: 7:50:55 time: 4.2968 data_time: 0.0141 memory: 34727 loss: 2.7634 +2024/02/20 14:06:49 - mmengine - INFO - Iter(train) [ 7430/13954] lr: 4.7182e-04 eta: 7:50:12 time: 4.2802 data_time: 0.0141 memory: 34633 loss: 2.7090 +2024/02/20 14:07:33 - mmengine - INFO - Iter(train) [ 7440/13954] lr: 4.7066e-04 eta: 7:49:28 time: 4.3030 data_time: 0.0140 memory: 34774 loss: 2.7732 +2024/02/20 14:08:16 - mmengine - INFO - Iter(train) [ 7450/13954] lr: 4.6950e-04 eta: 7:48:45 time: 4.3047 data_time: 0.0136 memory: 34774 loss: 2.8023 +2024/02/20 14:08:59 - mmengine - INFO - Iter(train) [ 7460/13954] lr: 4.6834e-04 eta: 7:48:01 time: 4.3112 data_time: 0.0123 memory: 34540 loss: 2.8342 +2024/02/20 14:09:42 - mmengine - INFO - Iter(train) [ 7470/13954] lr: 4.6718e-04 eta: 7:47:18 time: 4.3064 data_time: 0.0128 memory: 34915 loss: 2.8135 +2024/02/20 14:10:25 - mmengine - INFO - Iter(train) [ 7480/13954] lr: 4.6602e-04 eta: 7:46:35 time: 4.3189 data_time: 0.0133 memory: 34634 loss: 2.7224 +2024/02/20 14:11:08 - mmengine - INFO - Iter(train) [ 7490/13954] lr: 4.6487e-04 eta: 7:45:51 time: 4.3005 data_time: 0.0132 memory: 34774 loss: 2.7945 +2024/02/20 14:11:51 - mmengine - INFO - Iter(train) [ 7500/13954] lr: 4.6371e-04 eta: 7:45:08 time: 4.3229 data_time: 0.0130 memory: 35149 loss: 2.7123 +2024/02/20 14:11:51 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 14:11:51 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + +2024/02/20 14:11:52 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + +2024/02/20 14:11:52 - mmengine - INFO - Saving checkpoint at 7500 iterations +2024/02/20 14:12:35 - mmengine - INFO - Iter(train) [ 7510/13954] lr: 4.6255e-04 eta: 7:44:25 time: 4.3952 data_time: 0.0946 memory: 34822 loss: 2.6779 +2024/02/20 14:13:18 - mmengine - INFO - Iter(train) [ 7520/13954] lr: 4.6140e-04 eta: 7:43:42 time: 4.3164 data_time: 0.0129 memory: 34822 loss: 2.7723 +2024/02/20 14:14:01 - mmengine - INFO - Iter(train) [ 7530/13954] lr: 4.6024e-04 eta: 7:42:59 time: 4.3186 data_time: 0.0128 memory: 34962 loss: 2.7279 +2024/02/20 14:14:44 - mmengine - INFO - Iter(train) [ 7540/13954] lr: 4.5908e-04 eta: 7:42:15 time: 4.2838 data_time: 0.0128 memory: 34622 loss: 2.9082 +2024/02/20 14:15:27 - mmengine - INFO - Iter(train) [ 7550/13954] lr: 4.5793e-04 eta: 7:41:32 time: 4.3101 data_time: 0.0129 memory: 34727 loss: 2.7980 +2024/02/20 14:16:10 - mmengine - INFO - Iter(train) [ 7560/13954] lr: 4.5677e-04 eta: 7:40:48 time: 4.3038 data_time: 0.0138 memory: 34589 loss: 2.7940 +2024/02/20 14:16:53 - mmengine - INFO - Iter(train) [ 7570/13954] lr: 4.5561e-04 eta: 7:40:05 time: 4.3049 data_time: 0.0140 memory: 34668 loss: 2.7460 +2024/02/20 14:17:37 - mmengine - INFO - Iter(train) [ 7580/13954] lr: 4.5446e-04 eta: 7:39:22 time: 4.3183 data_time: 0.0139 memory: 34822 loss: 2.7440 +2024/02/20 14:18:20 - mmengine - INFO - Iter(train) [ 7590/13954] lr: 4.5330e-04 eta: 7:38:38 time: 4.3086 data_time: 0.0139 memory: 34634 loss: 2.7704 +2024/02/20 14:19:03 - mmengine - INFO - Iter(train) [ 7600/13954] lr: 4.5215e-04 eta: 7:37:55 time: 4.3015 data_time: 0.0140 memory: 34589 loss: 2.7514 +2024/02/20 14:19:46 - mmengine - INFO - Iter(train) [ 7610/13954] lr: 4.5099e-04 eta: 7:37:11 time: 4.2780 data_time: 0.0138 memory: 34668 loss: 2.8252 +2024/02/20 14:20:29 - mmengine - INFO - Iter(train) [ 7620/13954] lr: 4.4984e-04 eta: 7:36:28 time: 4.3081 data_time: 0.0140 memory: 34822 loss: 2.6888 +2024/02/20 14:21:12 - mmengine - INFO - Iter(train) [ 7630/13954] lr: 4.4868e-04 eta: 7:35:44 time: 4.3029 data_time: 0.0141 memory: 34774 loss: 2.8609 +2024/02/20 14:21:55 - mmengine - INFO - Iter(train) [ 7640/13954] lr: 4.4753e-04 eta: 7:35:01 time: 4.2931 data_time: 0.0140 memory: 34774 loss: 2.8519 +2024/02/20 14:22:38 - mmengine - INFO - Iter(train) [ 7650/13954] lr: 4.4637e-04 eta: 7:34:17 time: 4.2945 data_time: 0.0139 memory: 34634 loss: 2.8634 +2024/02/20 14:23:20 - mmengine - INFO - Iter(train) [ 7660/13954] lr: 4.4522e-04 eta: 7:33:34 time: 4.2889 data_time: 0.0141 memory: 34774 loss: 2.8176 +2024/02/20 14:24:03 - mmengine - INFO - Iter(train) [ 7670/13954] lr: 4.4407e-04 eta: 7:32:50 time: 4.2809 data_time: 0.0142 memory: 34680 loss: 2.8086 +2024/02/20 14:24:46 - mmengine - INFO - Iter(train) [ 7680/13954] lr: 4.4291e-04 eta: 7:32:07 time: 4.2974 data_time: 0.0141 memory: 34680 loss: 2.7100 +2024/02/20 14:25:29 - mmengine - INFO - Iter(train) [ 7690/13954] lr: 4.4176e-04 eta: 7:31:23 time: 4.2913 data_time: 0.0141 memory: 34589 loss: 2.7899 +2024/02/20 14:26:12 - mmengine - INFO - Iter(train) [ 7700/13954] lr: 4.4061e-04 eta: 7:30:40 time: 4.2917 data_time: 0.0139 memory: 34577 loss: 2.7401 +2024/02/20 14:26:55 - mmengine - INFO - Iter(train) [ 7710/13954] lr: 4.3946e-04 eta: 7:29:56 time: 4.2983 data_time: 0.0140 memory: 34634 loss: 2.8143 +2024/02/20 14:27:38 - mmengine - INFO - Iter(train) [ 7720/13954] lr: 4.3831e-04 eta: 7:29:13 time: 4.2962 data_time: 0.0138 memory: 34633 loss: 2.9253 +2024/02/20 14:28:21 - mmengine - INFO - Iter(train) [ 7730/13954] lr: 4.3715e-04 eta: 7:28:30 time: 4.3122 data_time: 0.0139 memory: 34869 loss: 2.7320 +2024/02/20 14:29:04 - mmengine - INFO - Iter(train) [ 7740/13954] lr: 4.3600e-04 eta: 7:27:46 time: 4.3116 data_time: 0.0140 memory: 34680 loss: 2.7232 +2024/02/20 14:29:47 - mmengine - INFO - Iter(train) [ 7750/13954] lr: 4.3485e-04 eta: 7:27:03 time: 4.3075 data_time: 0.0142 memory: 34962 loss: 2.7286 +2024/02/20 14:30:30 - mmengine - INFO - Iter(train) [ 7760/13954] lr: 4.3370e-04 eta: 7:26:20 time: 4.3122 data_time: 0.0142 memory: 34774 loss: 2.8330 +2024/02/20 14:31:14 - mmengine - INFO - Iter(train) [ 7770/13954] lr: 4.3255e-04 eta: 7:25:36 time: 4.3198 data_time: 0.0144 memory: 34727 loss: 2.7790 +2024/02/20 14:31:57 - mmengine - INFO - Iter(train) [ 7780/13954] lr: 4.3140e-04 eta: 7:24:53 time: 4.3112 data_time: 0.0140 memory: 34577 loss: 2.7616 +2024/02/20 14:32:40 - mmengine - INFO - Iter(train) [ 7790/13954] lr: 4.3025e-04 eta: 7:24:10 time: 4.3271 data_time: 0.0140 memory: 34634 loss: 2.6668 +2024/02/20 14:33:23 - mmengine - INFO - Iter(train) [ 7800/13954] lr: 4.2910e-04 eta: 7:23:27 time: 4.3281 data_time: 0.0139 memory: 34727 loss: 2.8407 +2024/02/20 14:34:06 - mmengine - INFO - Iter(train) [ 7810/13954] lr: 4.2796e-04 eta: 7:22:43 time: 4.3073 data_time: 0.0141 memory: 34822 loss: 2.7786 +2024/02/20 14:34:50 - mmengine - INFO - Iter(train) [ 7820/13954] lr: 4.2681e-04 eta: 7:22:00 time: 4.3258 data_time: 0.0144 memory: 35009 loss: 2.7210 +2024/02/20 14:35:33 - mmengine - INFO - Iter(train) [ 7830/13954] lr: 4.2566e-04 eta: 7:21:17 time: 4.2998 data_time: 0.0141 memory: 34727 loss: 2.7418 +2024/02/20 14:36:16 - mmengine - INFO - Iter(train) [ 7840/13954] lr: 4.2451e-04 eta: 7:20:33 time: 4.2931 data_time: 0.0140 memory: 34774 loss: 2.7869 +2024/02/20 14:36:59 - mmengine - INFO - Iter(train) [ 7850/13954] lr: 4.2336e-04 eta: 7:19:50 time: 4.3386 data_time: 0.0143 memory: 35840 loss: 2.7759 +2024/02/20 14:37:42 - mmengine - INFO - Iter(train) [ 7860/13954] lr: 4.2222e-04 eta: 7:19:07 time: 4.2917 data_time: 0.0141 memory: 34680 loss: 2.8582 +2024/02/20 14:38:25 - mmengine - INFO - Iter(train) [ 7870/13954] lr: 4.2107e-04 eta: 7:18:23 time: 4.2986 data_time: 0.0140 memory: 34727 loss: 2.7012 +2024/02/20 14:39:08 - mmengine - INFO - Iter(train) [ 7880/13954] lr: 4.1993e-04 eta: 7:17:40 time: 4.3109 data_time: 0.0141 memory: 34727 loss: 2.7594 +2024/02/20 14:39:51 - mmengine - INFO - Iter(train) [ 7890/13954] lr: 4.1878e-04 eta: 7:16:56 time: 4.2796 data_time: 0.0142 memory: 34950 loss: 2.7104 +2024/02/20 14:40:34 - mmengine - INFO - Iter(train) [ 7900/13954] lr: 4.1764e-04 eta: 7:16:13 time: 4.2858 data_time: 0.0140 memory: 34680 loss: 2.7453 +2024/02/20 14:41:17 - mmengine - INFO - Iter(train) [ 7910/13954] lr: 4.1649e-04 eta: 7:15:29 time: 4.2911 data_time: 0.0142 memory: 34589 loss: 2.8467 +2024/02/20 14:41:59 - mmengine - INFO - Iter(train) [ 7920/13954] lr: 4.1535e-04 eta: 7:14:46 time: 4.2944 data_time: 0.0141 memory: 34915 loss: 2.6914 +2024/02/20 14:42:43 - mmengine - INFO - Iter(train) [ 7930/13954] lr: 4.1420e-04 eta: 7:14:02 time: 4.3081 data_time: 0.0142 memory: 34774 loss: 2.6935 +2024/02/20 14:43:25 - mmengine - INFO - Iter(train) [ 7940/13954] lr: 4.1306e-04 eta: 7:13:19 time: 4.2816 data_time: 0.0141 memory: 34634 loss: 2.8337 +2024/02/20 14:44:08 - mmengine - INFO - Iter(train) [ 7950/13954] lr: 4.1192e-04 eta: 7:12:35 time: 4.2963 data_time: 0.0140 memory: 34868 loss: 2.7618 +2024/02/20 14:44:51 - mmengine - INFO - Iter(train) [ 7960/13954] lr: 4.1078e-04 eta: 7:11:52 time: 4.2992 data_time: 0.0143 memory: 34727 loss: 2.7744 +2024/02/20 14:45:35 - mmengine - INFO - Iter(train) [ 7970/13954] lr: 4.0964e-04 eta: 7:11:09 time: 4.3236 data_time: 0.0141 memory: 34727 loss: 2.7881 +2024/02/20 14:46:18 - mmengine - INFO - Iter(train) [ 7980/13954] lr: 4.0849e-04 eta: 7:10:26 time: 4.3202 data_time: 0.0141 memory: 34962 loss: 2.8504 +2024/02/20 14:47:01 - mmengine - INFO - Iter(train) [ 7990/13954] lr: 4.0735e-04 eta: 7:09:42 time: 4.3201 data_time: 0.0141 memory: 35102 loss: 2.8028 +2024/02/20 14:47:44 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 14:47:44 - mmengine - INFO - Iter(train) [ 8000/13954] lr: 4.0621e-04 eta: 7:08:59 time: 4.3029 data_time: 0.0140 memory: 34589 loss: 2.7694 +2024/02/20 14:47:44 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 14:47:44 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore<|im_end|> + +2024/02/20 14:47:45 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore<|im_end|> + +2024/02/20 14:47:45 - mmengine - INFO - Saving checkpoint at 8000 iterations +2024/02/20 14:48:28 - mmengine - INFO - Iter(train) [ 8010/13954] lr: 4.0507e-04 eta: 7:08:16 time: 4.3708 data_time: 0.0912 memory: 34822 loss: 2.6855 +2024/02/20 14:49:11 - mmengine - INFO - Iter(train) [ 8020/13954] lr: 4.0393e-04 eta: 7:07:33 time: 4.2990 data_time: 0.0142 memory: 34589 loss: 2.8093 +2024/02/20 14:49:54 - mmengine - INFO - Iter(train) [ 8030/13954] lr: 4.0280e-04 eta: 7:06:49 time: 4.3119 data_time: 0.0141 memory: 34727 loss: 2.8537 +2024/02/20 14:50:37 - mmengine - INFO - Iter(train) [ 8040/13954] lr: 4.0166e-04 eta: 7:06:06 time: 4.2921 data_time: 0.0140 memory: 34774 loss: 2.7507 +2024/02/20 14:51:20 - mmengine - INFO - Iter(train) [ 8050/13954] lr: 4.0052e-04 eta: 7:05:23 time: 4.3192 data_time: 0.0141 memory: 34634 loss: 2.7070 +2024/02/20 14:52:03 - mmengine - INFO - Iter(train) [ 8060/13954] lr: 3.9938e-04 eta: 7:04:39 time: 4.2940 data_time: 0.0140 memory: 34856 loss: 2.7246 +2024/02/20 14:52:46 - mmengine - INFO - Iter(train) [ 8070/13954] lr: 3.9825e-04 eta: 7:03:56 time: 4.2973 data_time: 0.0140 memory: 34822 loss: 2.7217 +2024/02/20 14:53:29 - mmengine - INFO - Iter(train) [ 8080/13954] lr: 3.9711e-04 eta: 7:03:12 time: 4.2734 data_time: 0.0142 memory: 34680 loss: 2.7818 +2024/02/20 14:54:11 - mmengine - INFO - Iter(train) [ 8090/13954] lr: 3.9598e-04 eta: 7:02:29 time: 4.2902 data_time: 0.0143 memory: 34727 loss: 2.6489 +2024/02/20 14:54:54 - mmengine - INFO - Iter(train) [ 8100/13954] lr: 3.9484e-04 eta: 7:01:45 time: 4.2797 data_time: 0.0141 memory: 34680 loss: 2.8262 +2024/02/20 14:55:37 - mmengine - INFO - Iter(train) [ 8110/13954] lr: 3.9371e-04 eta: 7:01:02 time: 4.3184 data_time: 0.0140 memory: 34869 loss: 2.7141 +2024/02/20 14:56:21 - mmengine - INFO - Iter(train) [ 8120/13954] lr: 3.9257e-04 eta: 7:00:19 time: 4.3124 data_time: 0.0138 memory: 34915 loss: 2.7885 +2024/02/20 14:57:04 - mmengine - INFO - Iter(train) [ 8130/13954] lr: 3.9144e-04 eta: 6:59:35 time: 4.2938 data_time: 0.0139 memory: 34915 loss: 2.7770 +2024/02/20 14:57:46 - mmengine - INFO - Iter(train) [ 8140/13954] lr: 3.9031e-04 eta: 6:58:52 time: 4.2963 data_time: 0.0140 memory: 34727 loss: 2.7014 +2024/02/20 14:58:29 - mmengine - INFO - Iter(train) [ 8150/13954] lr: 3.8918e-04 eta: 6:58:08 time: 4.2867 data_time: 0.0138 memory: 34774 loss: 2.8499 +2024/02/20 14:59:12 - mmengine - INFO - Iter(train) [ 8160/13954] lr: 3.8804e-04 eta: 6:57:25 time: 4.2772 data_time: 0.0143 memory: 34680 loss: 2.7672 +2024/02/20 14:59:55 - mmengine - INFO - Iter(train) [ 8170/13954] lr: 3.8691e-04 eta: 6:56:41 time: 4.2870 data_time: 0.0140 memory: 34680 loss: 2.8176 +2024/02/20 15:00:38 - mmengine - INFO - Iter(train) [ 8180/13954] lr: 3.8578e-04 eta: 6:55:58 time: 4.3134 data_time: 0.0142 memory: 34727 loss: 2.8195 +2024/02/20 15:01:21 - mmengine - INFO - Iter(train) [ 8190/13954] lr: 3.8465e-04 eta: 6:55:15 time: 4.3062 data_time: 0.0141 memory: 34727 loss: 2.7625 +2024/02/20 15:02:04 - mmengine - INFO - Iter(train) [ 8200/13954] lr: 3.8353e-04 eta: 6:54:31 time: 4.3035 data_time: 0.0141 memory: 34822 loss: 2.7447 +2024/02/20 15:02:47 - mmengine - INFO - Iter(train) [ 8210/13954] lr: 3.8240e-04 eta: 6:53:48 time: 4.3112 data_time: 0.0141 memory: 34822 loss: 2.6813 +2024/02/20 15:03:30 - mmengine - INFO - Iter(train) [ 8220/13954] lr: 3.8127e-04 eta: 6:53:04 time: 4.2876 data_time: 0.0142 memory: 34493 loss: 2.8080 +2024/02/20 15:04:14 - mmengine - INFO - Iter(train) [ 8230/13954] lr: 3.8014e-04 eta: 6:52:21 time: 4.3404 data_time: 0.0141 memory: 34822 loss: 2.7885 +2024/02/20 15:04:57 - mmengine - INFO - Iter(train) [ 8240/13954] lr: 3.7902e-04 eta: 6:51:38 time: 4.3217 data_time: 0.0142 memory: 34680 loss: 2.7316 +2024/02/20 15:05:40 - mmengine - INFO - Iter(train) [ 8250/13954] lr: 3.7789e-04 eta: 6:50:55 time: 4.3077 data_time: 0.0142 memory: 34634 loss: 2.8100 +2024/02/20 15:06:23 - mmengine - INFO - Iter(train) [ 8260/13954] lr: 3.7677e-04 eta: 6:50:12 time: 4.3164 data_time: 0.0141 memory: 35711 loss: 2.7812 +2024/02/20 15:07:06 - mmengine - INFO - Iter(train) [ 8270/13954] lr: 3.7564e-04 eta: 6:49:28 time: 4.3324 data_time: 0.0141 memory: 34680 loss: 2.8033 +2024/02/20 15:07:49 - mmengine - INFO - Iter(train) [ 8280/13954] lr: 3.7452e-04 eta: 6:48:45 time: 4.2973 data_time: 0.0141 memory: 34868 loss: 2.7656 +2024/02/20 15:08:33 - mmengine - INFO - Iter(train) [ 8290/13954] lr: 3.7339e-04 eta: 6:48:02 time: 4.3243 data_time: 0.0141 memory: 35758 loss: 2.7205 +2024/02/20 15:09:16 - mmengine - INFO - Iter(train) [ 8300/13954] lr: 3.7227e-04 eta: 6:47:18 time: 4.3104 data_time: 0.0141 memory: 34822 loss: 2.7490 +2024/02/20 15:09:59 - mmengine - INFO - Iter(train) [ 8310/13954] lr: 3.7115e-04 eta: 6:46:35 time: 4.3055 data_time: 0.0140 memory: 34868 loss: 2.8418 +2024/02/20 15:10:42 - mmengine - INFO - Iter(train) [ 8320/13954] lr: 3.7003e-04 eta: 6:45:52 time: 4.3028 data_time: 0.0142 memory: 34962 loss: 2.7686 +2024/02/20 15:11:25 - mmengine - INFO - Iter(train) [ 8330/13954] lr: 3.6891e-04 eta: 6:45:08 time: 4.3076 data_time: 0.0140 memory: 34680 loss: 2.8256 +2024/02/20 15:12:08 - mmengine - INFO - Iter(train) [ 8340/13954] lr: 3.6779e-04 eta: 6:44:25 time: 4.3022 data_time: 0.0140 memory: 34622 loss: 2.7815 +2024/02/20 15:12:51 - mmengine - INFO - Iter(train) [ 8350/13954] lr: 3.6667e-04 eta: 6:43:42 time: 4.2860 data_time: 0.0134 memory: 34868 loss: 2.8504 +2024/02/20 15:13:34 - mmengine - INFO - Iter(train) [ 8360/13954] lr: 3.6555e-04 eta: 6:42:58 time: 4.2996 data_time: 0.0120 memory: 34915 loss: 2.6685 +2024/02/20 15:14:17 - mmengine - INFO - Iter(train) [ 8370/13954] lr: 3.6444e-04 eta: 6:42:15 time: 4.3292 data_time: 0.0122 memory: 34727 loss: 2.7313 +2024/02/20 15:15:00 - mmengine - INFO - Iter(train) [ 8380/13954] lr: 3.6332e-04 eta: 6:41:32 time: 4.2929 data_time: 0.0135 memory: 34589 loss: 2.7160 +2024/02/20 15:15:43 - mmengine - INFO - Iter(train) [ 8390/13954] lr: 3.6220e-04 eta: 6:40:48 time: 4.3192 data_time: 0.0137 memory: 34727 loss: 2.7878 +2024/02/20 15:16:26 - mmengine - INFO - Iter(train) [ 8400/13954] lr: 3.6109e-04 eta: 6:40:05 time: 4.3001 data_time: 0.0132 memory: 34762 loss: 2.6863 +2024/02/20 15:17:09 - mmengine - INFO - Iter(train) [ 8410/13954] lr: 3.5997e-04 eta: 6:39:22 time: 4.3019 data_time: 0.0125 memory: 34774 loss: 2.7743 +2024/02/20 15:17:52 - mmengine - INFO - Iter(train) [ 8420/13954] lr: 3.5886e-04 eta: 6:38:38 time: 4.2888 data_time: 0.0139 memory: 34540 loss: 2.7762 +2024/02/20 15:18:35 - mmengine - INFO - Iter(train) [ 8430/13954] lr: 3.5775e-04 eta: 6:37:55 time: 4.2995 data_time: 0.0137 memory: 34761 loss: 2.7221 +2024/02/20 15:19:18 - mmengine - INFO - Iter(train) [ 8440/13954] lr: 3.5664e-04 eta: 6:37:11 time: 4.2968 data_time: 0.0138 memory: 34680 loss: 2.7664 +2024/02/20 15:20:01 - mmengine - INFO - Iter(train) [ 8450/13954] lr: 3.5552e-04 eta: 6:36:28 time: 4.2977 data_time: 0.0128 memory: 34727 loss: 2.7323 +2024/02/20 15:20:44 - mmengine - INFO - Iter(train) [ 8460/13954] lr: 3.5441e-04 eta: 6:35:45 time: 4.3031 data_time: 0.0122 memory: 34727 loss: 2.7622 +2024/02/20 15:21:27 - mmengine - INFO - Iter(train) [ 8470/13954] lr: 3.5330e-04 eta: 6:35:02 time: 4.3175 data_time: 0.0122 memory: 34868 loss: 2.7194 +2024/02/20 15:22:10 - mmengine - INFO - Iter(train) [ 8480/13954] lr: 3.5219e-04 eta: 6:34:18 time: 4.3034 data_time: 0.0126 memory: 34540 loss: 2.7145 +2024/02/20 15:22:53 - mmengine - INFO - Iter(train) [ 8490/13954] lr: 3.5109e-04 eta: 6:33:35 time: 4.2983 data_time: 0.0125 memory: 34680 loss: 2.7411 +2024/02/20 15:23:36 - mmengine - INFO - Iter(train) [ 8500/13954] lr: 3.4998e-04 eta: 6:32:51 time: 4.3012 data_time: 0.0123 memory: 34962 loss: 2.7209 +2024/02/20 15:23:36 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 15:23:37 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + +2024/02/20 15:23:37 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + +2024/02/20 15:23:37 - mmengine - INFO - Saving checkpoint at 8500 iterations +2024/02/20 15:24:20 - mmengine - INFO - Iter(train) [ 8510/13954] lr: 3.4887e-04 eta: 6:32:09 time: 4.4027 data_time: 0.0936 memory: 34774 loss: 2.7530 +2024/02/20 15:25:04 - mmengine - INFO - Iter(train) [ 8520/13954] lr: 3.4777e-04 eta: 6:31:26 time: 4.3435 data_time: 0.0114 memory: 35934 loss: 2.7573 +2024/02/20 15:25:47 - mmengine - INFO - Iter(train) [ 8530/13954] lr: 3.4666e-04 eta: 6:30:42 time: 4.2961 data_time: 0.0118 memory: 34589 loss: 2.7821 +2024/02/20 15:26:30 - mmengine - INFO - Iter(train) [ 8540/13954] lr: 3.4556e-04 eta: 6:29:59 time: 4.3134 data_time: 0.0121 memory: 34774 loss: 2.7768 +2024/02/20 15:27:13 - mmengine - INFO - Iter(train) [ 8550/13954] lr: 3.4445e-04 eta: 6:29:16 time: 4.2917 data_time: 0.0128 memory: 34774 loss: 2.6906 +2024/02/20 15:27:56 - mmengine - INFO - Iter(train) [ 8560/13954] lr: 3.4335e-04 eta: 6:28:32 time: 4.3172 data_time: 0.0131 memory: 34915 loss: 2.6537 +2024/02/20 15:28:39 - mmengine - INFO - Iter(train) [ 8570/13954] lr: 3.4225e-04 eta: 6:27:49 time: 4.3274 data_time: 0.0130 memory: 34915 loss: 2.7447 +2024/02/20 15:29:22 - mmengine - INFO - Iter(train) [ 8580/13954] lr: 3.4115e-04 eta: 6:27:06 time: 4.3232 data_time: 0.0133 memory: 35149 loss: 2.6894 +2024/02/20 15:30:06 - mmengine - INFO - Iter(train) [ 8590/13954] lr: 3.4005e-04 eta: 6:26:23 time: 4.3253 data_time: 0.0127 memory: 34715 loss: 2.7326 +2024/02/20 15:30:49 - mmengine - INFO - Iter(train) [ 8600/13954] lr: 3.3895e-04 eta: 6:25:39 time: 4.3116 data_time: 0.0128 memory: 34962 loss: 2.8050 +2024/02/20 15:31:32 - mmengine - INFO - Iter(train) [ 8610/13954] lr: 3.3785e-04 eta: 6:24:56 time: 4.3149 data_time: 0.0129 memory: 34822 loss: 2.7676 +2024/02/20 15:32:15 - mmengine - INFO - Iter(train) [ 8620/13954] lr: 3.3676e-04 eta: 6:24:13 time: 4.3110 data_time: 0.0130 memory: 34727 loss: 2.7935 +2024/02/20 15:32:58 - mmengine - INFO - Iter(train) [ 8630/13954] lr: 3.3566e-04 eta: 6:23:30 time: 4.3150 data_time: 0.0128 memory: 34774 loss: 2.8207 +2024/02/20 15:33:41 - mmengine - INFO - Iter(train) [ 8640/13954] lr: 3.3456e-04 eta: 6:22:46 time: 4.3143 data_time: 0.0128 memory: 34634 loss: 2.7229 +2024/02/20 15:34:24 - mmengine - INFO - Iter(train) [ 8650/13954] lr: 3.3347e-04 eta: 6:22:03 time: 4.3097 data_time: 0.0125 memory: 34727 loss: 2.7768 +2024/02/20 15:35:08 - mmengine - INFO - Iter(train) [ 8660/13954] lr: 3.3237e-04 eta: 6:21:20 time: 4.3208 data_time: 0.0124 memory: 35757 loss: 2.8004 +2024/02/20 15:35:51 - mmengine - INFO - Iter(train) [ 8670/13954] lr: 3.3128e-04 eta: 6:20:37 time: 4.3111 data_time: 0.0125 memory: 34634 loss: 2.8273 +2024/02/20 15:36:34 - mmengine - INFO - Iter(train) [ 8680/13954] lr: 3.3019e-04 eta: 6:19:53 time: 4.3151 data_time: 0.0129 memory: 35617 loss: 2.7947 +2024/02/20 15:37:17 - mmengine - INFO - Iter(train) [ 8690/13954] lr: 3.2910e-04 eta: 6:19:10 time: 4.2953 data_time: 0.0142 memory: 34634 loss: 2.7988 +2024/02/20 15:38:00 - mmengine - INFO - Iter(train) [ 8700/13954] lr: 3.2801e-04 eta: 6:18:27 time: 4.3286 data_time: 0.0138 memory: 34727 loss: 2.7407 +2024/02/20 15:38:43 - mmengine - INFO - Iter(train) [ 8710/13954] lr: 3.2692e-04 eta: 6:17:43 time: 4.2883 data_time: 0.0120 memory: 34680 loss: 2.7141 +2024/02/20 15:39:26 - mmengine - INFO - Iter(train) [ 8720/13954] lr: 3.2583e-04 eta: 6:17:00 time: 4.2895 data_time: 0.0127 memory: 34634 loss: 2.7783 +2024/02/20 15:40:09 - mmengine - INFO - Iter(train) [ 8730/13954] lr: 3.2474e-04 eta: 6:16:17 time: 4.3046 data_time: 0.0120 memory: 34868 loss: 2.7006 +2024/02/20 15:40:52 - mmengine - INFO - Iter(train) [ 8740/13954] lr: 3.2366e-04 eta: 6:15:33 time: 4.3110 data_time: 0.0117 memory: 34962 loss: 2.8110 +2024/02/20 15:41:35 - mmengine - INFO - Iter(train) [ 8750/13954] lr: 3.2257e-04 eta: 6:14:50 time: 4.2989 data_time: 0.0129 memory: 34962 loss: 2.7392 +2024/02/20 15:42:18 - mmengine - INFO - Iter(train) [ 8760/13954] lr: 3.2149e-04 eta: 6:14:07 time: 4.3010 data_time: 0.0122 memory: 34727 loss: 2.8598 +2024/02/20 15:43:01 - mmengine - INFO - Iter(train) [ 8770/13954] lr: 3.2040e-04 eta: 6:13:23 time: 4.2880 data_time: 0.0125 memory: 34680 loss: 2.8271 +2024/02/20 15:43:44 - mmengine - INFO - Iter(train) [ 8780/13954] lr: 3.1932e-04 eta: 6:12:40 time: 4.3062 data_time: 0.0132 memory: 34589 loss: 2.7442 +2024/02/20 15:44:27 - mmengine - INFO - Iter(train) [ 8790/13954] lr: 3.1824e-04 eta: 6:11:57 time: 4.3166 data_time: 0.0135 memory: 34680 loss: 2.6615 +2024/02/20 15:45:10 - mmengine - INFO - Iter(train) [ 8800/13954] lr: 3.1716e-04 eta: 6:11:13 time: 4.2922 data_time: 0.0126 memory: 34589 loss: 2.7611 +2024/02/20 15:45:53 - mmengine - INFO - Iter(train) [ 8810/13954] lr: 3.1608e-04 eta: 6:10:30 time: 4.3142 data_time: 0.0114 memory: 34822 loss: 2.7669 +2024/02/20 15:46:36 - mmengine - INFO - Iter(train) [ 8820/13954] lr: 3.1500e-04 eta: 6:09:47 time: 4.3077 data_time: 0.0111 memory: 34774 loss: 2.7750 +2024/02/20 15:47:19 - mmengine - INFO - Iter(train) [ 8830/13954] lr: 3.1392e-04 eta: 6:09:03 time: 4.3125 data_time: 0.0113 memory: 34727 loss: 2.7265 +2024/02/20 15:48:03 - mmengine - INFO - Iter(train) [ 8840/13954] lr: 3.1285e-04 eta: 6:08:20 time: 4.3225 data_time: 0.0133 memory: 34822 loss: 2.7254 +2024/02/20 15:48:46 - mmengine - INFO - Iter(train) [ 8850/13954] lr: 3.1177e-04 eta: 6:07:37 time: 4.3266 data_time: 0.0145 memory: 36355 loss: 2.7210 +2024/02/20 15:49:29 - mmengine - INFO - Iter(train) [ 8860/13954] lr: 3.1070e-04 eta: 6:06:54 time: 4.3332 data_time: 0.0141 memory: 34634 loss: 2.7805 +2024/02/20 15:50:12 - mmengine - INFO - Iter(train) [ 8870/13954] lr: 3.0962e-04 eta: 6:06:11 time: 4.3080 data_time: 0.0140 memory: 34727 loss: 2.7502 +2024/02/20 15:50:55 - mmengine - INFO - Iter(train) [ 8880/13954] lr: 3.0855e-04 eta: 6:05:27 time: 4.3127 data_time: 0.0141 memory: 34634 loss: 2.7190 +2024/02/20 15:51:38 - mmengine - INFO - Iter(train) [ 8890/13954] lr: 3.0748e-04 eta: 6:04:44 time: 4.2826 data_time: 0.0140 memory: 34634 loss: 2.7634 +2024/02/20 15:52:21 - mmengine - INFO - Iter(train) [ 8900/13954] lr: 3.0641e-04 eta: 6:04:01 time: 4.3052 data_time: 0.0132 memory: 34822 loss: 2.7466 +2024/02/20 15:53:04 - mmengine - INFO - Iter(train) [ 8910/13954] lr: 3.0534e-04 eta: 6:03:17 time: 4.3079 data_time: 0.0138 memory: 34822 loss: 2.7078 +2024/02/20 15:53:47 - mmengine - INFO - Iter(train) [ 8920/13954] lr: 3.0427e-04 eta: 6:02:34 time: 4.2957 data_time: 0.0137 memory: 34634 loss: 2.7582 +2024/02/20 15:54:30 - mmengine - INFO - Iter(train) [ 8930/13954] lr: 3.0320e-04 eta: 6:01:51 time: 4.3033 data_time: 0.0141 memory: 34680 loss: 2.8487 +2024/02/20 15:55:13 - mmengine - INFO - Iter(train) [ 8940/13954] lr: 3.0214e-04 eta: 6:01:07 time: 4.3031 data_time: 0.0141 memory: 34634 loss: 2.7639 +2024/02/20 15:55:56 - mmengine - INFO - Iter(train) [ 8950/13954] lr: 3.0107e-04 eta: 6:00:24 time: 4.2954 data_time: 0.0141 memory: 34727 loss: 2.7631 +2024/02/20 15:56:39 - mmengine - INFO - Iter(train) [ 8960/13954] lr: 3.0001e-04 eta: 5:59:41 time: 4.2892 data_time: 0.0140 memory: 34680 loss: 2.8224 +2024/02/20 15:57:22 - mmengine - INFO - Iter(train) [ 8970/13954] lr: 2.9895e-04 eta: 5:58:57 time: 4.2965 data_time: 0.0142 memory: 34774 loss: 2.7811 +2024/02/20 15:58:05 - mmengine - INFO - Iter(train) [ 8980/13954] lr: 2.9788e-04 eta: 5:58:14 time: 4.2916 data_time: 0.0140 memory: 35009 loss: 2.8120 +2024/02/20 15:58:48 - mmengine - INFO - Iter(train) [ 8990/13954] lr: 2.9682e-04 eta: 5:57:30 time: 4.2990 data_time: 0.0140 memory: 34540 loss: 2.7309 +2024/02/20 15:59:31 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 15:59:31 - mmengine - INFO - Iter(train) [ 9000/13954] lr: 2.9576e-04 eta: 5:56:47 time: 4.3013 data_time: 0.0140 memory: 34727 loss: 2.8255 +2024/02/20 15:59:31 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 15:59:32 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore photo<|im_end|> + +2024/02/20 15:59:32 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore photo<|im_end|> + +2024/02/20 15:59:32 - mmengine - INFO - Saving checkpoint at 9000 iterations +2024/02/20 16:00:15 - mmengine - INFO - Iter(train) [ 9010/13954] lr: 2.9470e-04 eta: 5:56:04 time: 4.3903 data_time: 0.1126 memory: 34774 loss: 2.7415 +2024/02/20 16:00:58 - mmengine - INFO - Iter(train) [ 9020/13954] lr: 2.9365e-04 eta: 5:55:21 time: 4.2941 data_time: 0.0139 memory: 34589 loss: 2.7031 +2024/02/20 16:01:41 - mmengine - INFO - Iter(train) [ 9030/13954] lr: 2.9259e-04 eta: 5:54:38 time: 4.3083 data_time: 0.0140 memory: 34962 loss: 2.7471 +2024/02/20 16:02:24 - mmengine - INFO - Iter(train) [ 9040/13954] lr: 2.9153e-04 eta: 5:53:54 time: 4.3044 data_time: 0.0142 memory: 34680 loss: 2.7512 +2024/02/20 16:03:07 - mmengine - INFO - Iter(train) [ 9050/13954] lr: 2.9048e-04 eta: 5:53:11 time: 4.3116 data_time: 0.0142 memory: 34774 loss: 2.7484 +2024/02/20 16:03:50 - mmengine - INFO - Iter(train) [ 9060/13954] lr: 2.8943e-04 eta: 5:52:28 time: 4.2752 data_time: 0.0141 memory: 34774 loss: 2.7434 +2024/02/20 16:04:33 - mmengine - INFO - Iter(train) [ 9070/13954] lr: 2.8838e-04 eta: 5:51:44 time: 4.2898 data_time: 0.0142 memory: 34589 loss: 2.7959 +2024/02/20 16:05:16 - mmengine - INFO - Iter(train) [ 9080/13954] lr: 2.8732e-04 eta: 5:51:01 time: 4.2671 data_time: 0.0141 memory: 34589 loss: 2.6643 +2024/02/20 16:05:59 - mmengine - INFO - Iter(train) [ 9090/13954] lr: 2.8627e-04 eta: 5:50:17 time: 4.2987 data_time: 0.0142 memory: 35242 loss: 2.6899 +2024/02/20 16:06:42 - mmengine - INFO - Iter(train) [ 9100/13954] lr: 2.8523e-04 eta: 5:49:34 time: 4.2940 data_time: 0.0142 memory: 34868 loss: 2.7181 +2024/02/20 16:07:25 - mmengine - INFO - Iter(train) [ 9110/13954] lr: 2.8418e-04 eta: 5:48:51 time: 4.3052 data_time: 0.0141 memory: 34680 loss: 2.7205 +2024/02/20 16:08:08 - mmengine - INFO - Iter(train) [ 9120/13954] lr: 2.8313e-04 eta: 5:48:07 time: 4.3030 data_time: 0.0140 memory: 34727 loss: 2.8051 +2024/02/20 16:08:51 - mmengine - INFO - Iter(train) [ 9130/13954] lr: 2.8209e-04 eta: 5:47:24 time: 4.2951 data_time: 0.0139 memory: 34774 loss: 2.8046 +2024/02/20 16:09:34 - mmengine - INFO - Iter(train) [ 9140/13954] lr: 2.8104e-04 eta: 5:46:41 time: 4.3174 data_time: 0.0137 memory: 34634 loss: 2.7837 +2024/02/20 16:10:17 - mmengine - INFO - Iter(train) [ 9150/13954] lr: 2.8000e-04 eta: 5:45:58 time: 4.3061 data_time: 0.0139 memory: 34680 loss: 2.7025 +2024/02/20 16:11:00 - mmengine - INFO - Iter(train) [ 9160/13954] lr: 2.7896e-04 eta: 5:45:14 time: 4.2871 data_time: 0.0137 memory: 34962 loss: 2.7601 +2024/02/20 16:11:42 - mmengine - INFO - Iter(train) [ 9170/13954] lr: 2.7792e-04 eta: 5:44:31 time: 4.2789 data_time: 0.0136 memory: 34680 loss: 2.6642 +2024/02/20 16:12:25 - mmengine - INFO - Iter(train) [ 9180/13954] lr: 2.7688e-04 eta: 5:43:47 time: 4.2924 data_time: 0.0139 memory: 34589 loss: 2.7398 +2024/02/20 16:13:08 - mmengine - INFO - Iter(train) [ 9190/13954] lr: 2.7584e-04 eta: 5:43:04 time: 4.2941 data_time: 0.0139 memory: 34869 loss: 2.6899 +2024/02/20 16:13:52 - mmengine - INFO - Iter(train) [ 9200/13954] lr: 2.7481e-04 eta: 5:42:21 time: 4.3252 data_time: 0.0141 memory: 34715 loss: 2.7490 +2024/02/20 16:14:35 - mmengine - INFO - Iter(train) [ 9210/13954] lr: 2.7377e-04 eta: 5:41:38 time: 4.3080 data_time: 0.0141 memory: 34622 loss: 2.6437 +2024/02/20 16:15:18 - mmengine - INFO - Iter(train) [ 9220/13954] lr: 2.7274e-04 eta: 5:40:54 time: 4.3356 data_time: 0.0140 memory: 35570 loss: 2.7239 +2024/02/20 16:16:01 - mmengine - INFO - Iter(train) [ 9230/13954] lr: 2.7170e-04 eta: 5:40:11 time: 4.3253 data_time: 0.0141 memory: 34868 loss: 2.7996 +2024/02/20 16:16:44 - mmengine - INFO - Iter(train) [ 9240/13954] lr: 2.7067e-04 eta: 5:39:28 time: 4.3153 data_time: 0.0142 memory: 34915 loss: 2.7178 +2024/02/20 16:17:28 - mmengine - INFO - Iter(train) [ 9250/13954] lr: 2.6964e-04 eta: 5:38:45 time: 4.3139 data_time: 0.0141 memory: 34822 loss: 2.7548 +2024/02/20 16:18:10 - mmengine - INFO - Iter(train) [ 9260/13954] lr: 2.6861e-04 eta: 5:38:01 time: 4.2824 data_time: 0.0142 memory: 34589 loss: 2.6929 +2024/02/20 16:18:53 - mmengine - INFO - Iter(train) [ 9270/13954] lr: 2.6758e-04 eta: 5:37:18 time: 4.2884 data_time: 0.0141 memory: 34634 loss: 2.6667 +2024/02/20 16:19:36 - mmengine - INFO - Iter(train) [ 9280/13954] lr: 2.6656e-04 eta: 5:36:35 time: 4.3004 data_time: 0.0142 memory: 34540 loss: 2.7658 +2024/02/20 16:20:20 - mmengine - INFO - Iter(train) [ 9290/13954] lr: 2.6553e-04 eta: 5:35:52 time: 4.3389 data_time: 0.0141 memory: 34634 loss: 2.6710 +2024/02/20 16:21:03 - mmengine - INFO - Iter(train) [ 9300/13954] lr: 2.6451e-04 eta: 5:35:08 time: 4.3042 data_time: 0.0141 memory: 34634 loss: 2.7191 +2024/02/20 16:21:46 - mmengine - INFO - Iter(train) [ 9310/13954] lr: 2.6348e-04 eta: 5:34:25 time: 4.3004 data_time: 0.0142 memory: 34822 loss: 2.7647 +2024/02/20 16:22:29 - mmengine - INFO - Iter(train) [ 9320/13954] lr: 2.6246e-04 eta: 5:33:42 time: 4.2968 data_time: 0.0141 memory: 35055 loss: 2.7410 +2024/02/20 16:23:12 - mmengine - INFO - Iter(train) [ 9330/13954] lr: 2.6144e-04 eta: 5:32:58 time: 4.2941 data_time: 0.0141 memory: 34868 loss: 2.7301 +2024/02/20 16:23:55 - mmengine - INFO - Iter(train) [ 9340/13954] lr: 2.6042e-04 eta: 5:32:15 time: 4.2929 data_time: 0.0142 memory: 34540 loss: 2.7432 +2024/02/20 16:24:37 - mmengine - INFO - Iter(train) [ 9350/13954] lr: 2.5940e-04 eta: 5:31:32 time: 4.2748 data_time: 0.0141 memory: 34634 loss: 2.7481 +2024/02/20 16:25:20 - mmengine - INFO - Iter(train) [ 9360/13954] lr: 2.5839e-04 eta: 5:30:48 time: 4.3109 data_time: 0.0141 memory: 34915 loss: 2.7133 +2024/02/20 16:26:03 - mmengine - INFO - Iter(train) [ 9370/13954] lr: 2.5737e-04 eta: 5:30:05 time: 4.2984 data_time: 0.0142 memory: 34680 loss: 2.7417 +2024/02/20 16:26:47 - mmengine - INFO - Iter(train) [ 9380/13954] lr: 2.5636e-04 eta: 5:29:22 time: 4.3163 data_time: 0.0138 memory: 34680 loss: 2.6888 +2024/02/20 16:27:30 - mmengine - INFO - Iter(train) [ 9390/13954] lr: 2.5535e-04 eta: 5:28:38 time: 4.3068 data_time: 0.0136 memory: 34774 loss: 2.7785 +2024/02/20 16:28:13 - mmengine - INFO - Iter(train) [ 9400/13954] lr: 2.5433e-04 eta: 5:27:55 time: 4.3023 data_time: 0.0141 memory: 34680 loss: 2.7690 +2024/02/20 16:28:56 - mmengine - INFO - Iter(train) [ 9410/13954] lr: 2.5332e-04 eta: 5:27:12 time: 4.2980 data_time: 0.0141 memory: 35149 loss: 2.6883 +2024/02/20 16:29:38 - mmengine - INFO - Iter(train) [ 9420/13954] lr: 2.5232e-04 eta: 5:26:28 time: 4.2822 data_time: 0.0142 memory: 34634 loss: 2.6499 +2024/02/20 16:30:22 - mmengine - INFO - Iter(train) [ 9430/13954] lr: 2.5131e-04 eta: 5:25:45 time: 4.3142 data_time: 0.0141 memory: 34915 loss: 2.7555 +2024/02/20 16:31:05 - mmengine - INFO - Iter(train) [ 9440/13954] lr: 2.5030e-04 eta: 5:25:02 time: 4.3119 data_time: 0.0139 memory: 34774 loss: 2.7394 +2024/02/20 16:31:48 - mmengine - INFO - Iter(train) [ 9450/13954] lr: 2.4930e-04 eta: 5:24:19 time: 4.3220 data_time: 0.0140 memory: 34680 loss: 2.7792 +2024/02/20 16:32:31 - mmengine - INFO - Iter(train) [ 9460/13954] lr: 2.4829e-04 eta: 5:23:36 time: 4.3018 data_time: 0.0131 memory: 34680 loss: 2.7604 +2024/02/20 16:33:14 - mmengine - INFO - Iter(train) [ 9470/13954] lr: 2.4729e-04 eta: 5:22:52 time: 4.3334 data_time: 0.0127 memory: 35336 loss: 2.7382 +2024/02/20 16:33:57 - mmengine - INFO - Iter(train) [ 9480/13954] lr: 2.4629e-04 eta: 5:22:09 time: 4.3114 data_time: 0.0127 memory: 35149 loss: 2.7135 +2024/02/20 16:34:40 - mmengine - INFO - Iter(train) [ 9490/13954] lr: 2.4529e-04 eta: 5:21:26 time: 4.3052 data_time: 0.0126 memory: 34727 loss: 2.6469 +2024/02/20 16:35:24 - mmengine - INFO - Iter(train) [ 9500/13954] lr: 2.4429e-04 eta: 5:20:43 time: 4.3182 data_time: 0.0127 memory: 34634 loss: 2.7027 +2024/02/20 16:35:24 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 16:35:24 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the water<|im_end|> + +2024/02/20 16:35:24 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on it<|im_end|> + +2024/02/20 16:35:24 - mmengine - INFO - Saving checkpoint at 9500 iterations +2024/02/20 16:36:08 - mmengine - INFO - Iter(train) [ 9510/13954] lr: 2.4330e-04 eta: 5:20:00 time: 4.4055 data_time: 0.0921 memory: 34727 loss: 2.7952 +2024/02/20 16:36:51 - mmengine - INFO - Iter(train) [ 9520/13954] lr: 2.4230e-04 eta: 5:19:17 time: 4.3322 data_time: 0.0130 memory: 34680 loss: 2.6922 +2024/02/20 16:37:34 - mmengine - INFO - Iter(train) [ 9530/13954] lr: 2.4131e-04 eta: 5:18:34 time: 4.3416 data_time: 0.0131 memory: 34715 loss: 2.7434 +2024/02/20 16:38:18 - mmengine - INFO - Iter(train) [ 9540/13954] lr: 2.4032e-04 eta: 5:17:50 time: 4.3166 data_time: 0.0131 memory: 34868 loss: 2.8032 +2024/02/20 16:39:01 - mmengine - INFO - Iter(train) [ 9550/13954] lr: 2.3932e-04 eta: 5:17:07 time: 4.2979 data_time: 0.0130 memory: 34822 loss: 2.5961 +2024/02/20 16:39:44 - mmengine - INFO - Iter(train) [ 9560/13954] lr: 2.3834e-04 eta: 5:16:24 time: 4.3264 data_time: 0.0129 memory: 34915 loss: 2.7360 +2024/02/20 16:40:27 - mmengine - INFO - Iter(train) [ 9570/13954] lr: 2.3735e-04 eta: 5:15:41 time: 4.3240 data_time: 0.0131 memory: 34634 loss: 2.7598 +2024/02/20 16:41:11 - mmengine - INFO - Iter(train) [ 9580/13954] lr: 2.3636e-04 eta: 5:14:58 time: 4.3561 data_time: 0.0129 memory: 36507 loss: 2.7438 +2024/02/20 16:41:54 - mmengine - INFO - Iter(train) [ 9590/13954] lr: 2.3538e-04 eta: 5:14:14 time: 4.3178 data_time: 0.0131 memory: 34822 loss: 2.6924 +2024/02/20 16:42:37 - mmengine - INFO - Iter(train) [ 9600/13954] lr: 2.3439e-04 eta: 5:13:31 time: 4.3088 data_time: 0.0130 memory: 34774 loss: 2.6710 +2024/02/20 16:43:20 - mmengine - INFO - Iter(train) [ 9610/13954] lr: 2.3341e-04 eta: 5:12:48 time: 4.3281 data_time: 0.0130 memory: 35711 loss: 2.7356 +2024/02/20 16:44:03 - mmengine - INFO - Iter(train) [ 9620/13954] lr: 2.3243e-04 eta: 5:12:05 time: 4.2886 data_time: 0.0131 memory: 34589 loss: 2.7775 +2024/02/20 16:44:46 - mmengine - INFO - Iter(train) [ 9630/13954] lr: 2.3145e-04 eta: 5:11:21 time: 4.2999 data_time: 0.0130 memory: 34727 loss: 2.7439 +2024/02/20 16:45:29 - mmengine - INFO - Iter(train) [ 9640/13954] lr: 2.3047e-04 eta: 5:10:38 time: 4.2915 data_time: 0.0130 memory: 34634 loss: 2.8379 +2024/02/20 16:46:12 - mmengine - INFO - Iter(train) [ 9650/13954] lr: 2.2949e-04 eta: 5:09:55 time: 4.3104 data_time: 0.0132 memory: 34869 loss: 2.6478 +2024/02/20 16:46:55 - mmengine - INFO - Iter(train) [ 9660/13954] lr: 2.2852e-04 eta: 5:09:12 time: 4.3158 data_time: 0.0131 memory: 34774 loss: 2.7946 +2024/02/20 16:47:38 - mmengine - INFO - Iter(train) [ 9670/13954] lr: 2.2754e-04 eta: 5:08:28 time: 4.3072 data_time: 0.0131 memory: 34857 loss: 2.6468 +2024/02/20 16:48:21 - mmengine - INFO - Iter(train) [ 9680/13954] lr: 2.2657e-04 eta: 5:07:45 time: 4.3147 data_time: 0.0131 memory: 34668 loss: 2.7105 +2024/02/20 16:49:05 - mmengine - INFO - Iter(train) [ 9690/13954] lr: 2.2560e-04 eta: 5:07:02 time: 4.3088 data_time: 0.0130 memory: 34589 loss: 2.7254 +2024/02/20 16:49:47 - mmengine - INFO - Iter(train) [ 9700/13954] lr: 2.2463e-04 eta: 5:06:18 time: 4.2920 data_time: 0.0130 memory: 34727 loss: 2.7157 +2024/02/20 16:50:31 - mmengine - INFO - Iter(train) [ 9710/13954] lr: 2.2366e-04 eta: 5:05:35 time: 4.3047 data_time: 0.0130 memory: 35336 loss: 2.6386 +2024/02/20 16:51:14 - mmengine - INFO - Iter(train) [ 9720/13954] lr: 2.2270e-04 eta: 5:04:52 time: 4.3121 data_time: 0.0129 memory: 34868 loss: 2.7195 +2024/02/20 16:51:57 - mmengine - INFO - Iter(train) [ 9730/13954] lr: 2.2173e-04 eta: 5:04:09 time: 4.3074 data_time: 0.0129 memory: 34822 loss: 2.7913 +2024/02/20 16:52:40 - mmengine - INFO - Iter(train) [ 9740/13954] lr: 2.2077e-04 eta: 5:03:25 time: 4.2970 data_time: 0.0129 memory: 34589 loss: 2.7061 +2024/02/20 16:53:22 - mmengine - INFO - Iter(train) [ 9750/13954] lr: 2.1981e-04 eta: 5:02:42 time: 4.2720 data_time: 0.0129 memory: 34540 loss: 2.7010 +2024/02/20 16:54:05 - mmengine - INFO - Iter(train) [ 9760/13954] lr: 2.1885e-04 eta: 5:01:59 time: 4.2981 data_time: 0.0130 memory: 34668 loss: 2.6647 +2024/02/20 16:54:48 - mmengine - INFO - Iter(train) [ 9770/13954] lr: 2.1789e-04 eta: 5:01:15 time: 4.2920 data_time: 0.0131 memory: 34727 loss: 2.7897 +2024/02/20 16:55:31 - mmengine - INFO - Iter(train) [ 9780/13954] lr: 2.1693e-04 eta: 5:00:32 time: 4.2960 data_time: 0.0130 memory: 34822 loss: 2.7632 +2024/02/20 16:56:14 - mmengine - INFO - Iter(train) [ 9790/13954] lr: 2.1597e-04 eta: 4:59:49 time: 4.2923 data_time: 0.0132 memory: 34762 loss: 2.7762 +2024/02/20 16:56:57 - mmengine - INFO - Iter(train) [ 9800/13954] lr: 2.1502e-04 eta: 4:59:05 time: 4.2999 data_time: 0.0131 memory: 34634 loss: 2.7686 +2024/02/20 16:57:40 - mmengine - INFO - Iter(train) [ 9810/13954] lr: 2.1407e-04 eta: 4:58:22 time: 4.2933 data_time: 0.0131 memory: 34622 loss: 2.6701 +2024/02/20 16:58:23 - mmengine - INFO - Iter(train) [ 9820/13954] lr: 2.1312e-04 eta: 4:57:39 time: 4.3087 data_time: 0.0131 memory: 34774 loss: 2.7597 +2024/02/20 16:59:06 - mmengine - INFO - Iter(train) [ 9830/13954] lr: 2.1217e-04 eta: 4:56:56 time: 4.2986 data_time: 0.0129 memory: 34540 loss: 2.7557 +2024/02/20 16:59:49 - mmengine - INFO - Iter(train) [ 9840/13954] lr: 2.1122e-04 eta: 4:56:12 time: 4.3302 data_time: 0.0129 memory: 34822 loss: 2.7234 +2024/02/20 17:00:33 - mmengine - INFO - Iter(train) [ 9850/13954] lr: 2.1027e-04 eta: 4:55:29 time: 4.3255 data_time: 0.0129 memory: 35242 loss: 2.7176 +2024/02/20 17:01:16 - mmengine - INFO - Iter(train) [ 9860/13954] lr: 2.0933e-04 eta: 4:54:46 time: 4.3196 data_time: 0.0129 memory: 34727 loss: 2.8267 +2024/02/20 17:01:59 - mmengine - INFO - Iter(train) [ 9870/13954] lr: 2.0838e-04 eta: 4:54:03 time: 4.3022 data_time: 0.0129 memory: 34774 loss: 2.8181 +2024/02/20 17:02:42 - mmengine - INFO - Iter(train) [ 9880/13954] lr: 2.0744e-04 eta: 4:53:19 time: 4.2846 data_time: 0.0129 memory: 34762 loss: 2.7181 +2024/02/20 17:03:25 - mmengine - INFO - Iter(train) [ 9890/13954] lr: 2.0650e-04 eta: 4:52:36 time: 4.3074 data_time: 0.0130 memory: 34680 loss: 2.7401 +2024/02/20 17:04:08 - mmengine - INFO - Iter(train) [ 9900/13954] lr: 2.0556e-04 eta: 4:51:53 time: 4.3155 data_time: 0.0131 memory: 34715 loss: 2.6793 +2024/02/20 17:04:51 - mmengine - INFO - Iter(train) [ 9910/13954] lr: 2.0463e-04 eta: 4:51:10 time: 4.3109 data_time: 0.0130 memory: 35758 loss: 2.6558 +2024/02/20 17:05:34 - mmengine - INFO - Iter(train) [ 9920/13954] lr: 2.0369e-04 eta: 4:50:27 time: 4.3228 data_time: 0.0130 memory: 34715 loss: 2.8045 +2024/02/20 17:06:18 - mmengine - INFO - Iter(train) [ 9930/13954] lr: 2.0276e-04 eta: 4:49:43 time: 4.3151 data_time: 0.0131 memory: 34915 loss: 2.7243 +2024/02/20 17:07:01 - mmengine - INFO - Iter(train) [ 9940/13954] lr: 2.0182e-04 eta: 4:49:00 time: 4.3004 data_time: 0.0130 memory: 34680 loss: 2.7415 +2024/02/20 17:07:44 - mmengine - INFO - Iter(train) [ 9950/13954] lr: 2.0089e-04 eta: 4:48:17 time: 4.3064 data_time: 0.0131 memory: 34869 loss: 2.6284 +2024/02/20 17:08:27 - mmengine - INFO - Iter(train) [ 9960/13954] lr: 1.9996e-04 eta: 4:47:34 time: 4.3133 data_time: 0.0131 memory: 34727 loss: 2.7040 +2024/02/20 17:09:10 - mmengine - INFO - Iter(train) [ 9970/13954] lr: 1.9904e-04 eta: 4:46:50 time: 4.2886 data_time: 0.0131 memory: 34822 loss: 2.7708 +2024/02/20 17:09:53 - mmengine - INFO - Iter(train) [ 9980/13954] lr: 1.9811e-04 eta: 4:46:07 time: 4.3139 data_time: 0.0130 memory: 35149 loss: 2.7648 +2024/02/20 17:10:36 - mmengine - INFO - Iter(train) [ 9990/13954] lr: 1.9719e-04 eta: 4:45:24 time: 4.3329 data_time: 0.0140 memory: 35149 loss: 2.7577 +2024/02/20 17:11:19 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 17:11:19 - mmengine - INFO - Iter(train) [10000/13954] lr: 1.9626e-04 eta: 4:44:41 time: 4.3056 data_time: 0.0130 memory: 34868 loss: 2.7305 +2024/02/20 17:11:19 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 17:11:19 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat and a mountain in the background<|im_end|> + +2024/02/20 17:11:20 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat and a mountain in the background<|im_end|> + +2024/02/20 17:11:20 - mmengine - INFO - Saving checkpoint at 10000 iterations +2024/02/20 17:12:03 - mmengine - INFO - Iter(train) [10010/13954] lr: 1.9534e-04 eta: 4:43:58 time: 4.4055 data_time: 0.1055 memory: 34822 loss: 2.7593 +2024/02/20 17:12:46 - mmengine - INFO - Iter(train) [10020/13954] lr: 1.9442e-04 eta: 4:43:15 time: 4.3175 data_time: 0.0131 memory: 34540 loss: 2.7048 +2024/02/20 17:13:29 - mmengine - INFO - Iter(train) [10030/13954] lr: 1.9351e-04 eta: 4:42:31 time: 4.3014 data_time: 0.0132 memory: 34727 loss: 2.7211 +2024/02/20 17:14:12 - mmengine - INFO - Iter(train) [10040/13954] lr: 1.9259e-04 eta: 4:41:48 time: 4.2929 data_time: 0.0133 memory: 34762 loss: 2.7665 +2024/02/20 17:14:55 - mmengine - INFO - Iter(train) [10050/13954] lr: 1.9168e-04 eta: 4:41:05 time: 4.2915 data_time: 0.0133 memory: 34493 loss: 2.7509 +2024/02/20 17:15:38 - mmengine - INFO - Iter(train) [10060/13954] lr: 1.9076e-04 eta: 4:40:21 time: 4.2965 data_time: 0.0132 memory: 34762 loss: 2.7557 +2024/02/20 17:16:21 - mmengine - INFO - Iter(train) [10070/13954] lr: 1.8985e-04 eta: 4:39:38 time: 4.3047 data_time: 0.0131 memory: 34810 loss: 2.6717 +2024/02/20 17:17:04 - mmengine - INFO - Iter(train) [10080/13954] lr: 1.8894e-04 eta: 4:38:55 time: 4.3040 data_time: 0.0131 memory: 34727 loss: 2.8039 +2024/02/20 17:17:47 - mmengine - INFO - Iter(train) [10090/13954] lr: 1.8803e-04 eta: 4:38:11 time: 4.2879 data_time: 0.0132 memory: 34634 loss: 2.8270 +2024/02/20 17:18:30 - mmengine - INFO - Iter(train) [10100/13954] lr: 1.8713e-04 eta: 4:37:28 time: 4.2907 data_time: 0.0132 memory: 34727 loss: 2.6365 +2024/02/20 17:19:13 - mmengine - INFO - Iter(train) [10110/13954] lr: 1.8622e-04 eta: 4:36:45 time: 4.2970 data_time: 0.0130 memory: 34715 loss: 2.7479 +2024/02/20 17:19:56 - mmengine - INFO - Iter(train) [10120/13954] lr: 1.8532e-04 eta: 4:36:02 time: 4.3019 data_time: 0.0130 memory: 34868 loss: 2.6918 +2024/02/20 17:20:39 - mmengine - INFO - Iter(train) [10130/13954] lr: 1.8442e-04 eta: 4:35:18 time: 4.2705 data_time: 0.0129 memory: 34633 loss: 2.5802 +2024/02/20 17:21:22 - mmengine - INFO - Iter(train) [10140/13954] lr: 1.8352e-04 eta: 4:34:35 time: 4.2756 data_time: 0.0130 memory: 34857 loss: 2.7776 +2024/02/20 17:22:04 - mmengine - INFO - Iter(train) [10150/13954] lr: 1.8262e-04 eta: 4:33:52 time: 4.2889 data_time: 0.0135 memory: 34727 loss: 2.6917 +2024/02/20 17:22:47 - mmengine - INFO - Iter(train) [10160/13954] lr: 1.8173e-04 eta: 4:33:08 time: 4.2708 data_time: 0.0130 memory: 34634 loss: 2.7292 +2024/02/20 17:23:30 - mmengine - INFO - Iter(train) [10170/13954] lr: 1.8083e-04 eta: 4:32:25 time: 4.2794 data_time: 0.0133 memory: 34634 loss: 2.6832 +2024/02/20 17:24:13 - mmengine - INFO - Iter(train) [10180/13954] lr: 1.7994e-04 eta: 4:31:42 time: 4.3013 data_time: 0.0132 memory: 34680 loss: 2.7975 +2024/02/20 17:24:56 - mmengine - INFO - Iter(train) [10190/13954] lr: 1.7905e-04 eta: 4:30:58 time: 4.2893 data_time: 0.0130 memory: 34680 loss: 2.8054 +2024/02/20 17:25:39 - mmengine - INFO - Iter(train) [10200/13954] lr: 1.7816e-04 eta: 4:30:15 time: 4.2891 data_time: 0.0130 memory: 34715 loss: 2.7690 +2024/02/20 17:26:21 - mmengine - INFO - Iter(train) [10210/13954] lr: 1.7727e-04 eta: 4:29:32 time: 4.2691 data_time: 0.0131 memory: 34634 loss: 2.7670 +2024/02/20 17:27:04 - mmengine - INFO - Iter(train) [10220/13954] lr: 1.7639e-04 eta: 4:28:48 time: 4.2757 data_time: 0.0129 memory: 34821 loss: 2.7363 +2024/02/20 17:27:47 - mmengine - INFO - Iter(train) [10230/13954] lr: 1.7551e-04 eta: 4:28:05 time: 4.2876 data_time: 0.0124 memory: 35055 loss: 2.8602 +2024/02/20 17:28:30 - mmengine - INFO - Iter(train) [10240/13954] lr: 1.7462e-04 eta: 4:27:22 time: 4.2782 data_time: 0.0130 memory: 34774 loss: 2.7002 +2024/02/20 17:29:13 - mmengine - INFO - Iter(train) [10250/13954] lr: 1.7374e-04 eta: 4:26:38 time: 4.2839 data_time: 0.0138 memory: 34868 loss: 2.7030 +2024/02/20 17:29:56 - mmengine - INFO - Iter(train) [10260/13954] lr: 1.7286e-04 eta: 4:25:55 time: 4.2980 data_time: 0.0133 memory: 34634 loss: 2.7624 +2024/02/20 17:30:38 - mmengine - INFO - Iter(train) [10270/13954] lr: 1.7199e-04 eta: 4:25:12 time: 4.2798 data_time: 0.0129 memory: 34822 loss: 2.6485 +2024/02/20 17:31:21 - mmengine - INFO - Iter(train) [10280/13954] lr: 1.7111e-04 eta: 4:24:28 time: 4.2856 data_time: 0.0132 memory: 34715 loss: 2.7412 +2024/02/20 17:32:04 - mmengine - INFO - Iter(train) [10290/13954] lr: 1.7024e-04 eta: 4:23:45 time: 4.2854 data_time: 0.0133 memory: 34634 loss: 2.7393 +2024/02/20 17:32:47 - mmengine - INFO - Iter(train) [10300/13954] lr: 1.6937e-04 eta: 4:23:02 time: 4.2772 data_time: 0.0129 memory: 34774 loss: 2.8077 +2024/02/20 17:33:30 - mmengine - INFO - Iter(train) [10310/13954] lr: 1.6850e-04 eta: 4:22:18 time: 4.2911 data_time: 0.0136 memory: 34856 loss: 2.6707 +2024/02/20 17:34:13 - mmengine - INFO - Iter(train) [10320/13954] lr: 1.6763e-04 eta: 4:21:35 time: 4.3098 data_time: 0.0137 memory: 35511 loss: 2.7550 +2024/02/20 17:34:56 - mmengine - INFO - Iter(train) [10330/13954] lr: 1.6676e-04 eta: 4:20:52 time: 4.2844 data_time: 0.0151 memory: 34634 loss: 2.7495 +2024/02/20 17:35:39 - mmengine - INFO - Iter(train) [10340/13954] lr: 1.6590e-04 eta: 4:20:09 time: 4.2885 data_time: 0.0149 memory: 34727 loss: 2.7407 +2024/02/20 17:36:21 - mmengine - INFO - Iter(train) [10350/13954] lr: 1.6504e-04 eta: 4:19:25 time: 4.2847 data_time: 0.0144 memory: 34822 loss: 2.7345 +2024/02/20 17:37:05 - mmengine - INFO - Iter(train) [10360/13954] lr: 1.6418e-04 eta: 4:18:42 time: 4.3023 data_time: 0.0147 memory: 34774 loss: 2.7694 +2024/02/20 17:37:47 - mmengine - INFO - Iter(train) [10370/13954] lr: 1.6332e-04 eta: 4:17:59 time: 4.2827 data_time: 0.0142 memory: 34727 loss: 2.7331 +2024/02/20 17:38:30 - mmengine - INFO - Iter(train) [10380/13954] lr: 1.6246e-04 eta: 4:17:15 time: 4.2978 data_time: 0.0143 memory: 35055 loss: 2.7166 +2024/02/20 17:39:13 - mmengine - INFO - Iter(train) [10390/13954] lr: 1.6161e-04 eta: 4:16:32 time: 4.2916 data_time: 0.0139 memory: 34774 loss: 2.8245 +2024/02/20 17:39:56 - mmengine - INFO - Iter(train) [10400/13954] lr: 1.6075e-04 eta: 4:15:49 time: 4.2981 data_time: 0.0134 memory: 34540 loss: 2.6763 +2024/02/20 17:40:39 - mmengine - INFO - Iter(train) [10410/13954] lr: 1.5990e-04 eta: 4:15:06 time: 4.2740 data_time: 0.0135 memory: 34962 loss: 2.7430 +2024/02/20 17:41:22 - mmengine - INFO - Iter(train) [10420/13954] lr: 1.5905e-04 eta: 4:14:22 time: 4.2868 data_time: 0.0137 memory: 34680 loss: 2.7008 +2024/02/20 17:42:05 - mmengine - INFO - Iter(train) [10430/13954] lr: 1.5820e-04 eta: 4:13:39 time: 4.3144 data_time: 0.0134 memory: 34822 loss: 2.7483 +2024/02/20 17:42:48 - mmengine - INFO - Iter(train) [10440/13954] lr: 1.5736e-04 eta: 4:12:56 time: 4.3009 data_time: 0.0133 memory: 34634 loss: 2.6933 +2024/02/20 17:43:31 - mmengine - INFO - Iter(train) [10450/13954] lr: 1.5651e-04 eta: 4:12:13 time: 4.3208 data_time: 0.0132 memory: 34868 loss: 2.7521 +2024/02/20 17:44:14 - mmengine - INFO - Iter(train) [10460/13954] lr: 1.5567e-04 eta: 4:11:29 time: 4.3203 data_time: 0.0132 memory: 34634 loss: 2.6136 +2024/02/20 17:44:57 - mmengine - INFO - Iter(train) [10470/13954] lr: 1.5483e-04 eta: 4:10:46 time: 4.3078 data_time: 0.0131 memory: 35055 loss: 2.7582 +2024/02/20 17:45:41 - mmengine - INFO - Iter(train) [10480/13954] lr: 1.5399e-04 eta: 4:10:03 time: 4.3133 data_time: 0.0132 memory: 34774 loss: 2.7444 +2024/02/20 17:46:24 - mmengine - INFO - Iter(train) [10490/13954] lr: 1.5316e-04 eta: 4:09:20 time: 4.3131 data_time: 0.0126 memory: 34680 loss: 2.6961 +2024/02/20 17:47:07 - mmengine - INFO - Iter(train) [10500/13954] lr: 1.5232e-04 eta: 4:08:37 time: 4.3163 data_time: 0.0134 memory: 34680 loss: 2.7700 +2024/02/20 17:47:07 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 17:47:07 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake tahoe, california<|im_end|> + +2024/02/20 17:47:08 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake lakelake in the mountains<|im_end|> + +2024/02/20 17:47:08 - mmengine - INFO - Saving checkpoint at 10500 iterations +2024/02/20 17:47:50 - mmengine - INFO - Iter(train) [10510/13954] lr: 1.5149e-04 eta: 4:07:54 time: 4.3593 data_time: 0.0932 memory: 34589 loss: 2.7941 +2024/02/20 17:48:34 - mmengine - INFO - Iter(train) [10520/13954] lr: 1.5066e-04 eta: 4:07:10 time: 4.3083 data_time: 0.0146 memory: 34680 loss: 2.6768 +2024/02/20 17:49:17 - mmengine - INFO - Iter(train) [10530/13954] lr: 1.4983e-04 eta: 4:06:27 time: 4.3026 data_time: 0.0146 memory: 34774 loss: 2.6556 +2024/02/20 17:50:00 - mmengine - INFO - Iter(train) [10540/13954] lr: 1.4900e-04 eta: 4:05:44 time: 4.2970 data_time: 0.0146 memory: 34589 loss: 2.7296 +2024/02/20 17:50:43 - mmengine - INFO - Iter(train) [10550/13954] lr: 1.4817e-04 eta: 4:05:01 time: 4.3146 data_time: 0.0145 memory: 34868 loss: 2.6845 +2024/02/20 17:51:26 - mmengine - INFO - Iter(train) [10560/13954] lr: 1.4735e-04 eta: 4:04:17 time: 4.2835 data_time: 0.0147 memory: 34680 loss: 2.7645 +2024/02/20 17:52:09 - mmengine - INFO - Iter(train) [10570/13954] lr: 1.4653e-04 eta: 4:03:34 time: 4.3181 data_time: 0.0145 memory: 35055 loss: 2.7079 +2024/02/20 17:52:52 - mmengine - INFO - Iter(train) [10580/13954] lr: 1.4571e-04 eta: 4:02:51 time: 4.3445 data_time: 0.0143 memory: 34810 loss: 2.7889 +2024/02/20 17:53:35 - mmengine - INFO - Iter(train) [10590/13954] lr: 1.4489e-04 eta: 4:02:08 time: 4.2971 data_time: 0.0144 memory: 34822 loss: 2.7038 +2024/02/20 17:54:18 - mmengine - INFO - Iter(train) [10600/13954] lr: 1.4407e-04 eta: 4:01:25 time: 4.3096 data_time: 0.0144 memory: 34680 loss: 2.6485 +2024/02/20 17:55:01 - mmengine - INFO - Iter(train) [10610/13954] lr: 1.4326e-04 eta: 4:00:41 time: 4.3194 data_time: 0.0145 memory: 34634 loss: 2.7511 +2024/02/20 17:55:45 - mmengine - INFO - Iter(train) [10620/13954] lr: 1.4245e-04 eta: 3:59:58 time: 4.3128 data_time: 0.0144 memory: 34762 loss: 2.7420 +2024/02/20 17:56:28 - mmengine - INFO - Iter(train) [10630/13954] lr: 1.4164e-04 eta: 3:59:15 time: 4.3021 data_time: 0.0145 memory: 34727 loss: 2.7860 +2024/02/20 17:57:11 - mmengine - INFO - Iter(train) [10640/13954] lr: 1.4083e-04 eta: 3:58:32 time: 4.3071 data_time: 0.0145 memory: 34915 loss: 2.7680 +2024/02/20 17:57:54 - mmengine - INFO - Iter(train) [10650/13954] lr: 1.4002e-04 eta: 3:57:48 time: 4.3145 data_time: 0.0147 memory: 34634 loss: 2.7057 +2024/02/20 17:58:37 - mmengine - INFO - Iter(train) [10660/13954] lr: 1.3922e-04 eta: 3:57:05 time: 4.3011 data_time: 0.0145 memory: 34680 loss: 2.6448 +2024/02/20 17:59:20 - mmengine - INFO - Iter(train) [10670/13954] lr: 1.3842e-04 eta: 3:56:22 time: 4.2883 data_time: 0.0146 memory: 34680 loss: 2.7368 +2024/02/20 18:00:03 - mmengine - INFO - Iter(train) [10680/13954] lr: 1.3762e-04 eta: 3:55:39 time: 4.3151 data_time: 0.0146 memory: 34634 loss: 2.6985 +2024/02/20 18:00:46 - mmengine - INFO - Iter(train) [10690/13954] lr: 1.3682e-04 eta: 3:54:56 time: 4.3054 data_time: 0.0144 memory: 34589 loss: 2.7888 +2024/02/20 18:01:29 - mmengine - INFO - Iter(train) [10700/13954] lr: 1.3602e-04 eta: 3:54:12 time: 4.3195 data_time: 0.0145 memory: 34774 loss: 2.6234 +2024/02/20 18:02:12 - mmengine - INFO - Iter(train) [10710/13954] lr: 1.3523e-04 eta: 3:53:29 time: 4.2986 data_time: 0.0145 memory: 34950 loss: 2.5987 +2024/02/20 18:02:55 - mmengine - INFO - Iter(train) [10720/13954] lr: 1.3443e-04 eta: 3:52:46 time: 4.3054 data_time: 0.0145 memory: 34962 loss: 2.7114 +2024/02/20 18:03:38 - mmengine - INFO - Iter(train) [10730/13954] lr: 1.3364e-04 eta: 3:52:03 time: 4.2997 data_time: 0.0146 memory: 34680 loss: 2.7044 +2024/02/20 18:04:21 - mmengine - INFO - Iter(train) [10740/13954] lr: 1.3285e-04 eta: 3:51:19 time: 4.3250 data_time: 0.0146 memory: 35899 loss: 2.6238 +2024/02/20 18:05:04 - mmengine - INFO - Iter(train) [10750/13954] lr: 1.3207e-04 eta: 3:50:36 time: 4.3072 data_time: 0.0144 memory: 34680 loss: 2.5961 +2024/02/20 18:05:48 - mmengine - INFO - Iter(train) [10760/13954] lr: 1.3128e-04 eta: 3:49:53 time: 4.3205 data_time: 0.0145 memory: 34774 loss: 2.7570 +2024/02/20 18:06:31 - mmengine - INFO - Iter(train) [10770/13954] lr: 1.3050e-04 eta: 3:49:10 time: 4.2977 data_time: 0.0144 memory: 34634 loss: 2.6990 +2024/02/20 18:07:14 - mmengine - INFO - Iter(train) [10780/13954] lr: 1.2972e-04 eta: 3:48:27 time: 4.2892 data_time: 0.0146 memory: 34634 loss: 2.6385 +2024/02/20 18:07:57 - mmengine - INFO - Iter(train) [10790/13954] lr: 1.2894e-04 eta: 3:47:43 time: 4.3046 data_time: 0.0147 memory: 34822 loss: 2.6650 +2024/02/20 18:08:40 - mmengine - INFO - Iter(train) [10800/13954] lr: 1.2816e-04 eta: 3:47:00 time: 4.3006 data_time: 0.0146 memory: 34774 loss: 2.7084 +2024/02/20 18:09:23 - mmengine - INFO - Iter(train) [10810/13954] lr: 1.2739e-04 eta: 3:46:17 time: 4.3273 data_time: 0.0144 memory: 34727 loss: 2.6332 +2024/02/20 18:10:06 - mmengine - INFO - Iter(train) [10820/13954] lr: 1.2662e-04 eta: 3:45:34 time: 4.3386 data_time: 0.0147 memory: 34822 loss: 2.6543 +2024/02/20 18:10:49 - mmengine - INFO - Iter(train) [10830/13954] lr: 1.2585e-04 eta: 3:44:51 time: 4.2936 data_time: 0.0145 memory: 34868 loss: 2.6673 +2024/02/20 18:11:32 - mmengine - INFO - Iter(train) [10840/13954] lr: 1.2508e-04 eta: 3:44:07 time: 4.2963 data_time: 0.0147 memory: 34680 loss: 2.7100 +2024/02/20 18:12:15 - mmengine - INFO - Iter(train) [10850/13954] lr: 1.2431e-04 eta: 3:43:24 time: 4.2890 data_time: 0.0145 memory: 34622 loss: 2.7522 +2024/02/20 18:12:58 - mmengine - INFO - Iter(train) [10860/13954] lr: 1.2354e-04 eta: 3:42:41 time: 4.3126 data_time: 0.0144 memory: 34727 loss: 2.7160 +2024/02/20 18:13:41 - mmengine - INFO - Iter(train) [10870/13954] lr: 1.2278e-04 eta: 3:41:58 time: 4.3133 data_time: 0.0146 memory: 34680 loss: 2.6994 +2024/02/20 18:14:24 - mmengine - INFO - Iter(train) [10880/13954] lr: 1.2202e-04 eta: 3:41:14 time: 4.3092 data_time: 0.0146 memory: 34762 loss: 2.6670 +2024/02/20 18:15:08 - mmengine - INFO - Iter(train) [10890/13954] lr: 1.2126e-04 eta: 3:40:31 time: 4.3228 data_time: 0.0146 memory: 35899 loss: 2.6436 +2024/02/20 18:15:50 - mmengine - INFO - Iter(train) [10900/13954] lr: 1.2051e-04 eta: 3:39:48 time: 4.2834 data_time: 0.0147 memory: 34774 loss: 2.7096 +2024/02/20 18:16:33 - mmengine - INFO - Iter(train) [10910/13954] lr: 1.1975e-04 eta: 3:39:05 time: 4.2935 data_time: 0.0146 memory: 34727 loss: 2.7088 +2024/02/20 18:17:16 - mmengine - INFO - Iter(train) [10920/13954] lr: 1.1900e-04 eta: 3:38:21 time: 4.3089 data_time: 0.0146 memory: 34774 loss: 2.7413 +2024/02/20 18:18:00 - mmengine - INFO - Iter(train) [10930/13954] lr: 1.1825e-04 eta: 3:37:38 time: 4.3075 data_time: 0.0147 memory: 34822 loss: 2.7183 +2024/02/20 18:18:42 - mmengine - INFO - Iter(train) [10940/13954] lr: 1.1750e-04 eta: 3:36:55 time: 4.2914 data_time: 0.0143 memory: 34680 loss: 2.6654 +2024/02/20 18:19:25 - mmengine - INFO - Iter(train) [10950/13954] lr: 1.1675e-04 eta: 3:36:12 time: 4.2855 data_time: 0.0145 memory: 34774 loss: 2.7163 +2024/02/20 18:20:08 - mmengine - INFO - Iter(train) [10960/13954] lr: 1.1601e-04 eta: 3:35:29 time: 4.3023 data_time: 0.0146 memory: 35149 loss: 2.6328 +2024/02/20 18:20:51 - mmengine - INFO - Iter(train) [10970/13954] lr: 1.1527e-04 eta: 3:34:45 time: 4.3060 data_time: 0.0143 memory: 34962 loss: 2.7921 +2024/02/20 18:21:35 - mmengine - INFO - Iter(train) [10980/13954] lr: 1.1453e-04 eta: 3:34:02 time: 4.3185 data_time: 0.0145 memory: 34727 loss: 2.7491 +2024/02/20 18:22:18 - mmengine - INFO - Iter(train) [10990/13954] lr: 1.1379e-04 eta: 3:33:19 time: 4.3103 data_time: 0.0147 memory: 34589 loss: 2.6167 +2024/02/20 18:23:01 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 18:23:01 - mmengine - INFO - Iter(train) [11000/13954] lr: 1.1305e-04 eta: 3:32:36 time: 4.3176 data_time: 0.0144 memory: 34727 loss: 2.8233 +2024/02/20 18:23:01 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 18:23:01 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on it<|im_end|> + +2024/02/20 18:23:01 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on it<|im_end|> + +2024/02/20 18:23:01 - mmengine - INFO - Saving checkpoint at 11000 iterations +2024/02/20 18:23:44 - mmengine - INFO - Iter(train) [11010/13954] lr: 1.1232e-04 eta: 3:31:53 time: 4.3536 data_time: 0.0928 memory: 34589 loss: 2.6981 +2024/02/20 18:24:28 - mmengine - INFO - Iter(train) [11020/13954] lr: 1.1159e-04 eta: 3:31:09 time: 4.3146 data_time: 0.0138 memory: 35243 loss: 2.6422 +2024/02/20 18:25:11 - mmengine - INFO - Iter(train) [11030/13954] lr: 1.1086e-04 eta: 3:30:26 time: 4.3176 data_time: 0.0139 memory: 34822 loss: 2.7504 +2024/02/20 18:25:54 - mmengine - INFO - Iter(train) [11040/13954] lr: 1.1013e-04 eta: 3:29:43 time: 4.3175 data_time: 0.0139 memory: 34822 loss: 2.6799 +2024/02/20 18:26:37 - mmengine - INFO - Iter(train) [11050/13954] lr: 1.0940e-04 eta: 3:29:00 time: 4.2903 data_time: 0.0139 memory: 34680 loss: 2.7408 +2024/02/20 18:27:20 - mmengine - INFO - Iter(train) [11060/13954] lr: 1.0868e-04 eta: 3:28:17 time: 4.3064 data_time: 0.0136 memory: 34680 loss: 2.8738 +2024/02/20 18:28:03 - mmengine - INFO - Iter(train) [11070/13954] lr: 1.0796e-04 eta: 3:27:33 time: 4.3331 data_time: 0.0137 memory: 36647 loss: 2.7123 +2024/02/20 18:28:46 - mmengine - INFO - Iter(train) [11080/13954] lr: 1.0724e-04 eta: 3:26:50 time: 4.3141 data_time: 0.0139 memory: 34962 loss: 2.7602 +2024/02/20 18:29:29 - mmengine - INFO - Iter(train) [11090/13954] lr: 1.0652e-04 eta: 3:26:07 time: 4.3150 data_time: 0.0139 memory: 34810 loss: 2.7334 +2024/02/20 18:30:13 - mmengine - INFO - Iter(train) [11100/13954] lr: 1.0581e-04 eta: 3:25:24 time: 4.3045 data_time: 0.0140 memory: 34634 loss: 2.7289 +2024/02/20 18:30:56 - mmengine - INFO - Iter(train) [11110/13954] lr: 1.0510e-04 eta: 3:24:41 time: 4.3230 data_time: 0.0144 memory: 34915 loss: 2.6509 +2024/02/20 18:31:39 - mmengine - INFO - Iter(train) [11120/13954] lr: 1.0439e-04 eta: 3:23:58 time: 4.3240 data_time: 0.0147 memory: 34821 loss: 2.6110 +2024/02/20 18:32:22 - mmengine - INFO - Iter(train) [11130/13954] lr: 1.0368e-04 eta: 3:23:14 time: 4.3128 data_time: 0.0147 memory: 34680 loss: 2.7247 +2024/02/20 18:33:05 - mmengine - INFO - Iter(train) [11140/13954] lr: 1.0297e-04 eta: 3:22:31 time: 4.3116 data_time: 0.0147 memory: 34822 loss: 2.6541 +2024/02/20 18:33:48 - mmengine - INFO - Iter(train) [11150/13954] lr: 1.0227e-04 eta: 3:21:48 time: 4.2946 data_time: 0.0145 memory: 34774 loss: 2.7766 +2024/02/20 18:34:31 - mmengine - INFO - Iter(train) [11160/13954] lr: 1.0156e-04 eta: 3:21:05 time: 4.2904 data_time: 0.0146 memory: 34774 loss: 2.7340 +2024/02/20 18:35:14 - mmengine - INFO - Iter(train) [11170/13954] lr: 1.0086e-04 eta: 3:20:21 time: 4.2969 data_time: 0.0147 memory: 34762 loss: 2.7939 +2024/02/20 18:35:57 - mmengine - INFO - Iter(train) [11180/13954] lr: 1.0017e-04 eta: 3:19:38 time: 4.3124 data_time: 0.0145 memory: 34680 loss: 2.7037 +2024/02/20 18:36:40 - mmengine - INFO - Iter(train) [11190/13954] lr: 9.9470e-05 eta: 3:18:55 time: 4.3126 data_time: 0.0144 memory: 35009 loss: 2.7218 +2024/02/20 18:37:24 - mmengine - INFO - Iter(train) [11200/13954] lr: 9.8777e-05 eta: 3:18:12 time: 4.3227 data_time: 0.0144 memory: 34962 loss: 2.6908 +2024/02/20 18:38:07 - mmengine - INFO - Iter(train) [11210/13954] lr: 9.8085e-05 eta: 3:17:29 time: 4.2982 data_time: 0.0147 memory: 34774 loss: 2.7445 +2024/02/20 18:38:49 - mmengine - INFO - Iter(train) [11220/13954] lr: 9.7396e-05 eta: 3:16:45 time: 4.2904 data_time: 0.0140 memory: 34634 loss: 2.6473 +2024/02/20 18:39:33 - mmengine - INFO - Iter(train) [11230/13954] lr: 9.6709e-05 eta: 3:16:02 time: 4.3166 data_time: 0.0139 memory: 35418 loss: 2.6109 +2024/02/20 18:40:16 - mmengine - INFO - Iter(train) [11240/13954] lr: 9.6024e-05 eta: 3:15:19 time: 4.3432 data_time: 0.0142 memory: 34727 loss: 2.7650 +2024/02/20 18:40:59 - mmengine - INFO - Iter(train) [11250/13954] lr: 9.5341e-05 eta: 3:14:36 time: 4.3206 data_time: 0.0143 memory: 34810 loss: 2.6881 +2024/02/20 18:41:42 - mmengine - INFO - Iter(train) [11260/13954] lr: 9.4661e-05 eta: 3:13:53 time: 4.3133 data_time: 0.0145 memory: 34727 loss: 2.7077 +2024/02/20 18:42:25 - mmengine - INFO - Iter(train) [11270/13954] lr: 9.3983e-05 eta: 3:13:09 time: 4.3018 data_time: 0.0143 memory: 34680 loss: 2.8149 +2024/02/20 18:43:08 - mmengine - INFO - Iter(train) [11280/13954] lr: 9.3306e-05 eta: 3:12:26 time: 4.3033 data_time: 0.0144 memory: 34680 loss: 2.7404 +2024/02/20 18:43:52 - mmengine - INFO - Iter(train) [11290/13954] lr: 9.2632e-05 eta: 3:11:43 time: 4.3124 data_time: 0.0144 memory: 34915 loss: 2.7611 +2024/02/20 18:44:35 - mmengine - INFO - Iter(train) [11300/13954] lr: 9.1961e-05 eta: 3:11:00 time: 4.3290 data_time: 0.0144 memory: 35617 loss: 2.7785 +2024/02/20 18:45:18 - mmengine - INFO - Iter(train) [11310/13954] lr: 9.1291e-05 eta: 3:10:17 time: 4.3154 data_time: 0.0145 memory: 34822 loss: 2.7164 +2024/02/20 18:46:01 - mmengine - INFO - Iter(train) [11320/13954] lr: 9.0624e-05 eta: 3:09:33 time: 4.2908 data_time: 0.0146 memory: 34680 loss: 2.6953 +2024/02/20 18:46:44 - mmengine - INFO - Iter(train) [11330/13954] lr: 8.9959e-05 eta: 3:08:50 time: 4.2927 data_time: 0.0145 memory: 34962 loss: 2.7035 +2024/02/20 18:47:27 - mmengine - INFO - Iter(train) [11340/13954] lr: 8.9296e-05 eta: 3:08:07 time: 4.3036 data_time: 0.0145 memory: 34680 loss: 2.6656 +2024/02/20 18:48:10 - mmengine - INFO - Iter(train) [11350/13954] lr: 8.8635e-05 eta: 3:07:24 time: 4.3127 data_time: 0.0145 memory: 34727 loss: 2.6877 +2024/02/20 18:48:53 - mmengine - INFO - Iter(train) [11360/13954] lr: 8.7976e-05 eta: 3:06:41 time: 4.3317 data_time: 0.0141 memory: 34727 loss: 2.6990 +2024/02/20 18:49:36 - mmengine - INFO - Iter(train) [11370/13954] lr: 8.7320e-05 eta: 3:05:57 time: 4.3186 data_time: 0.0144 memory: 35009 loss: 2.7283 +2024/02/20 18:50:20 - mmengine - INFO - Iter(train) [11380/13954] lr: 8.6666e-05 eta: 3:05:14 time: 4.3039 data_time: 0.0143 memory: 35477 loss: 2.7488 +2024/02/20 18:51:03 - mmengine - INFO - Iter(train) [11390/13954] lr: 8.6014e-05 eta: 3:04:31 time: 4.3066 data_time: 0.0141 memory: 34589 loss: 2.6908 +2024/02/20 18:51:46 - mmengine - INFO - Iter(train) [11400/13954] lr: 8.5364e-05 eta: 3:03:48 time: 4.3105 data_time: 0.0137 memory: 34680 loss: 2.7776 +2024/02/20 18:52:29 - mmengine - INFO - Iter(train) [11410/13954] lr: 8.4717e-05 eta: 3:03:05 time: 4.3050 data_time: 0.0134 memory: 34680 loss: 2.6981 +2024/02/20 18:53:12 - mmengine - INFO - Iter(train) [11420/13954] lr: 8.4072e-05 eta: 3:02:21 time: 4.3220 data_time: 0.0135 memory: 35899 loss: 2.7446 +2024/02/20 18:53:55 - mmengine - INFO - Iter(train) [11430/13954] lr: 8.3429e-05 eta: 3:01:38 time: 4.2996 data_time: 0.0135 memory: 34589 loss: 2.7759 +2024/02/20 18:54:38 - mmengine - INFO - Iter(train) [11440/13954] lr: 8.2788e-05 eta: 3:00:55 time: 4.2992 data_time: 0.0135 memory: 34822 loss: 2.7558 +2024/02/20 18:55:21 - mmengine - INFO - Iter(train) [11450/13954] lr: 8.2150e-05 eta: 3:00:12 time: 4.2999 data_time: 0.0136 memory: 34680 loss: 2.6381 +2024/02/20 18:56:04 - mmengine - INFO - Iter(train) [11460/13954] lr: 8.1514e-05 eta: 2:59:29 time: 4.3122 data_time: 0.0142 memory: 35009 loss: 2.6844 +2024/02/20 18:56:47 - mmengine - INFO - Iter(train) [11470/13954] lr: 8.0880e-05 eta: 2:58:45 time: 4.3123 data_time: 0.0141 memory: 34774 loss: 2.6344 +2024/02/20 18:57:30 - mmengine - INFO - Iter(train) [11480/13954] lr: 8.0248e-05 eta: 2:58:02 time: 4.2992 data_time: 0.0141 memory: 34634 loss: 2.7531 +2024/02/20 18:58:13 - mmengine - INFO - Iter(train) [11490/13954] lr: 7.9619e-05 eta: 2:57:19 time: 4.2949 data_time: 0.0136 memory: 34903 loss: 2.7121 +2024/02/20 18:58:56 - mmengine - INFO - Iter(train) [11500/13954] lr: 7.8992e-05 eta: 2:56:36 time: 4.2866 data_time: 0.0135 memory: 34869 loss: 2.7088 +2024/02/20 18:58:56 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 18:58:56 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the shore<|im_end|> + +2024/02/20 18:58:57 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 18:58:57 - mmengine - INFO - Saving checkpoint at 11500 iterations +2024/02/20 18:59:40 - mmengine - INFO - Iter(train) [11510/13954] lr: 7.8367e-05 eta: 2:55:53 time: 4.3883 data_time: 0.0909 memory: 35230 loss: 2.7525 +2024/02/20 19:00:23 - mmengine - INFO - Iter(train) [11520/13954] lr: 7.7744e-05 eta: 2:55:10 time: 4.3237 data_time: 0.0140 memory: 34727 loss: 2.7651 +2024/02/20 19:01:06 - mmengine - INFO - Iter(train) [11530/13954] lr: 7.7124e-05 eta: 2:54:26 time: 4.3116 data_time: 0.0142 memory: 34774 loss: 2.7131 +2024/02/20 19:01:49 - mmengine - INFO - Iter(train) [11540/13954] lr: 7.6506e-05 eta: 2:53:43 time: 4.2811 data_time: 0.0145 memory: 34540 loss: 2.7161 +2024/02/20 19:02:32 - mmengine - INFO - Iter(train) [11550/13954] lr: 7.5890e-05 eta: 2:53:00 time: 4.2960 data_time: 0.0148 memory: 34727 loss: 2.6914 +2024/02/20 19:03:15 - mmengine - INFO - Iter(train) [11560/13954] lr: 7.5276e-05 eta: 2:52:17 time: 4.3004 data_time: 0.0148 memory: 34634 loss: 2.7420 +2024/02/20 19:03:58 - mmengine - INFO - Iter(train) [11570/13954] lr: 7.4665e-05 eta: 2:51:33 time: 4.3247 data_time: 0.0143 memory: 34822 loss: 2.7828 +2024/02/20 19:04:41 - mmengine - INFO - Iter(train) [11580/13954] lr: 7.4056e-05 eta: 2:50:50 time: 4.3087 data_time: 0.0140 memory: 34634 loss: 2.6991 +2024/02/20 19:05:25 - mmengine - INFO - Iter(train) [11590/13954] lr: 7.3450e-05 eta: 2:50:07 time: 4.3138 data_time: 0.0142 memory: 34634 loss: 2.7150 +2024/02/20 19:06:08 - mmengine - INFO - Iter(train) [11600/13954] lr: 7.2845e-05 eta: 2:49:24 time: 4.3110 data_time: 0.0146 memory: 34822 loss: 2.7375 +2024/02/20 19:06:51 - mmengine - INFO - Iter(train) [11610/13954] lr: 7.2243e-05 eta: 2:48:41 time: 4.3074 data_time: 0.0147 memory: 34680 loss: 2.6888 +2024/02/20 19:07:34 - mmengine - INFO - Iter(train) [11620/13954] lr: 7.1644e-05 eta: 2:47:57 time: 4.2994 data_time: 0.0144 memory: 34774 loss: 2.6642 +2024/02/20 19:08:17 - mmengine - INFO - Iter(train) [11630/13954] lr: 7.1046e-05 eta: 2:47:14 time: 4.2993 data_time: 0.0145 memory: 34856 loss: 2.7946 +2024/02/20 19:09:00 - mmengine - INFO - Iter(train) [11640/13954] lr: 7.0451e-05 eta: 2:46:31 time: 4.2972 data_time: 0.0139 memory: 34822 loss: 2.7567 +2024/02/20 19:09:43 - mmengine - INFO - Iter(train) [11650/13954] lr: 6.9858e-05 eta: 2:45:48 time: 4.3081 data_time: 0.0139 memory: 34822 loss: 2.6879 +2024/02/20 19:10:26 - mmengine - INFO - Iter(train) [11660/13954] lr: 6.9268e-05 eta: 2:45:05 time: 4.3057 data_time: 0.0136 memory: 34762 loss: 2.6695 +2024/02/20 19:11:09 - mmengine - INFO - Iter(train) [11670/13954] lr: 6.8680e-05 eta: 2:44:21 time: 4.2958 data_time: 0.0138 memory: 34774 loss: 2.7291 +2024/02/20 19:11:52 - mmengine - INFO - Iter(train) [11680/13954] lr: 6.8094e-05 eta: 2:43:38 time: 4.3011 data_time: 0.0145 memory: 34727 loss: 2.6379 +2024/02/20 19:12:35 - mmengine - INFO - Iter(train) [11690/13954] lr: 6.7510e-05 eta: 2:42:55 time: 4.3100 data_time: 0.0142 memory: 34822 loss: 2.7665 +2024/02/20 19:13:18 - mmengine - INFO - Iter(train) [11700/13954] lr: 6.6929e-05 eta: 2:42:12 time: 4.3007 data_time: 0.0141 memory: 34950 loss: 2.6946 +2024/02/20 19:14:01 - mmengine - INFO - Iter(train) [11710/13954] lr: 6.6350e-05 eta: 2:41:29 time: 4.3064 data_time: 0.0142 memory: 34727 loss: 2.7692 +2024/02/20 19:14:44 - mmengine - INFO - Iter(train) [11720/13954] lr: 6.5774e-05 eta: 2:40:45 time: 4.2966 data_time: 0.0142 memory: 34634 loss: 2.7743 +2024/02/20 19:15:27 - mmengine - INFO - Iter(train) [11730/13954] lr: 6.5200e-05 eta: 2:40:02 time: 4.2841 data_time: 0.0141 memory: 34668 loss: 2.6568 +2024/02/20 19:16:10 - mmengine - INFO - Iter(train) [11740/13954] lr: 6.4628e-05 eta: 2:39:19 time: 4.3228 data_time: 0.0143 memory: 34727 loss: 2.7092 +2024/02/20 19:16:53 - mmengine - INFO - Iter(train) [11750/13954] lr: 6.4059e-05 eta: 2:38:36 time: 4.2988 data_time: 0.0147 memory: 34774 loss: 2.7161 +2024/02/20 19:17:36 - mmengine - INFO - Iter(train) [11760/13954] lr: 6.3491e-05 eta: 2:37:53 time: 4.3017 data_time: 0.0143 memory: 34634 loss: 2.6186 +2024/02/20 19:18:19 - mmengine - INFO - Iter(train) [11770/13954] lr: 6.2927e-05 eta: 2:37:09 time: 4.3106 data_time: 0.0144 memory: 34868 loss: 2.6304 +2024/02/20 19:19:02 - mmengine - INFO - Iter(train) [11780/13954] lr: 6.2364e-05 eta: 2:36:26 time: 4.3247 data_time: 0.0145 memory: 34727 loss: 2.6827 +2024/02/20 19:19:45 - mmengine - INFO - Iter(train) [11790/13954] lr: 6.1804e-05 eta: 2:35:43 time: 4.2845 data_time: 0.0147 memory: 34962 loss: 2.7410 +2024/02/20 19:20:28 - mmengine - INFO - Iter(train) [11800/13954] lr: 6.1246e-05 eta: 2:35:00 time: 4.3234 data_time: 0.0139 memory: 35196 loss: 2.7000 +2024/02/20 19:21:11 - mmengine - INFO - Iter(train) [11810/13954] lr: 6.0691e-05 eta: 2:34:17 time: 4.2737 data_time: 0.0140 memory: 34493 loss: 2.6048 +2024/02/20 19:21:54 - mmengine - INFO - Iter(train) [11820/13954] lr: 6.0138e-05 eta: 2:33:33 time: 4.3103 data_time: 0.0142 memory: 35149 loss: 2.7682 +2024/02/20 19:22:37 - mmengine - INFO - Iter(train) [11830/13954] lr: 5.9588e-05 eta: 2:32:50 time: 4.2797 data_time: 0.0140 memory: 34869 loss: 2.7657 +2024/02/20 19:23:20 - mmengine - INFO - Iter(train) [11840/13954] lr: 5.9039e-05 eta: 2:32:07 time: 4.3041 data_time: 0.0144 memory: 35009 loss: 2.7572 +2024/02/20 19:24:03 - mmengine - INFO - Iter(train) [11850/13954] lr: 5.8494e-05 eta: 2:31:24 time: 4.2921 data_time: 0.0149 memory: 34810 loss: 2.6742 +2024/02/20 19:24:46 - mmengine - INFO - Iter(train) [11860/13954] lr: 5.7950e-05 eta: 2:30:40 time: 4.2751 data_time: 0.0149 memory: 34762 loss: 2.7066 +2024/02/20 19:25:29 - mmengine - INFO - Iter(train) [11870/13954] lr: 5.7409e-05 eta: 2:29:57 time: 4.2928 data_time: 0.0150 memory: 34727 loss: 2.7226 +2024/02/20 19:26:11 - mmengine - INFO - Iter(train) [11880/13954] lr: 5.6870e-05 eta: 2:29:14 time: 4.2798 data_time: 0.0148 memory: 34634 loss: 2.6918 +2024/02/20 19:26:54 - mmengine - INFO - Iter(train) [11890/13954] lr: 5.6334e-05 eta: 2:28:31 time: 4.2809 data_time: 0.0147 memory: 34680 loss: 2.5749 +2024/02/20 19:27:37 - mmengine - INFO - Iter(train) [11900/13954] lr: 5.5800e-05 eta: 2:27:48 time: 4.3033 data_time: 0.0149 memory: 34915 loss: 2.6901 +2024/02/20 19:28:20 - mmengine - INFO - Iter(train) [11910/13954] lr: 5.5268e-05 eta: 2:27:04 time: 4.3075 data_time: 0.0147 memory: 34810 loss: 2.8132 +2024/02/20 19:29:03 - mmengine - INFO - Iter(train) [11920/13954] lr: 5.4739e-05 eta: 2:26:21 time: 4.3056 data_time: 0.0145 memory: 34774 loss: 2.7589 +2024/02/20 19:29:47 - mmengine - INFO - Iter(train) [11930/13954] lr: 5.4213e-05 eta: 2:25:38 time: 4.3120 data_time: 0.0143 memory: 34762 loss: 2.6917 +2024/02/20 19:30:30 - mmengine - INFO - Iter(train) [11940/13954] lr: 5.3688e-05 eta: 2:24:55 time: 4.3046 data_time: 0.0141 memory: 34822 loss: 2.7765 +2024/02/20 19:31:12 - mmengine - INFO - Iter(train) [11950/13954] lr: 5.3166e-05 eta: 2:24:12 time: 4.2814 data_time: 0.0141 memory: 34727 loss: 2.7615 +2024/02/20 19:31:55 - mmengine - INFO - Iter(train) [11960/13954] lr: 5.2647e-05 eta: 2:23:28 time: 4.2919 data_time: 0.0142 memory: 34868 loss: 2.7376 +2024/02/20 19:32:38 - mmengine - INFO - Iter(train) [11970/13954] lr: 5.2130e-05 eta: 2:22:45 time: 4.3014 data_time: 0.0142 memory: 34821 loss: 2.7223 +2024/02/20 19:33:21 - mmengine - INFO - Iter(train) [11980/13954] lr: 5.1615e-05 eta: 2:22:02 time: 4.2995 data_time: 0.0142 memory: 34715 loss: 2.7163 +2024/02/20 19:34:04 - mmengine - INFO - Iter(train) [11990/13954] lr: 5.1103e-05 eta: 2:21:19 time: 4.3041 data_time: 0.0143 memory: 34822 loss: 2.7797 +2024/02/20 19:34:47 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 19:34:47 - mmengine - INFO - Iter(train) [12000/13954] lr: 5.0593e-05 eta: 2:20:36 time: 4.2943 data_time: 0.0145 memory: 35044 loss: 2.7113 +2024/02/20 19:34:47 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 19:34:48 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 19:34:48 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 19:34:48 - mmengine - INFO - Saving checkpoint at 12000 iterations +2024/02/20 19:35:31 - mmengine - INFO - Iter(train) [12010/13954] lr: 5.0085e-05 eta: 2:19:52 time: 4.3691 data_time: 0.0870 memory: 34868 loss: 2.6738 +2024/02/20 19:36:14 - mmengine - INFO - Iter(train) [12020/13954] lr: 4.9580e-05 eta: 2:19:09 time: 4.2926 data_time: 0.0145 memory: 34962 loss: 2.7290 +2024/02/20 19:36:57 - mmengine - INFO - Iter(train) [12030/13954] lr: 4.9078e-05 eta: 2:18:26 time: 4.2956 data_time: 0.0149 memory: 34589 loss: 2.7639 +2024/02/20 19:37:40 - mmengine - INFO - Iter(train) [12040/13954] lr: 4.8578e-05 eta: 2:17:43 time: 4.2887 data_time: 0.0144 memory: 34680 loss: 2.7233 +2024/02/20 19:38:23 - mmengine - INFO - Iter(train) [12050/13954] lr: 4.8080e-05 eta: 2:17:00 time: 4.2745 data_time: 0.0142 memory: 34539 loss: 2.6950 +2024/02/20 19:39:06 - mmengine - INFO - Iter(train) [12060/13954] lr: 4.7584e-05 eta: 2:16:16 time: 4.3088 data_time: 0.0142 memory: 34810 loss: 2.7674 +2024/02/20 19:39:49 - mmengine - INFO - Iter(train) [12070/13954] lr: 4.7092e-05 eta: 2:15:33 time: 4.3339 data_time: 0.0143 memory: 35430 loss: 2.7382 +2024/02/20 19:40:32 - mmengine - INFO - Iter(train) [12080/13954] lr: 4.6601e-05 eta: 2:14:50 time: 4.3080 data_time: 0.0144 memory: 34822 loss: 2.6967 +2024/02/20 19:41:15 - mmengine - INFO - Iter(train) [12090/13954] lr: 4.6113e-05 eta: 2:14:07 time: 4.2944 data_time: 0.0145 memory: 34727 loss: 2.7470 +2024/02/20 19:41:58 - mmengine - INFO - Iter(train) [12100/13954] lr: 4.5628e-05 eta: 2:13:24 time: 4.2928 data_time: 0.0147 memory: 34727 loss: 2.7624 +2024/02/20 19:42:41 - mmengine - INFO - Iter(train) [12110/13954] lr: 4.5145e-05 eta: 2:12:40 time: 4.2883 data_time: 0.0147 memory: 34822 loss: 2.7630 +2024/02/20 19:43:24 - mmengine - INFO - Iter(train) [12120/13954] lr: 4.4664e-05 eta: 2:11:57 time: 4.2810 data_time: 0.0146 memory: 34540 loss: 2.7670 +2024/02/20 19:44:06 - mmengine - INFO - Iter(train) [12130/13954] lr: 4.4186e-05 eta: 2:11:14 time: 4.2758 data_time: 0.0148 memory: 34634 loss: 2.7003 +2024/02/20 19:44:49 - mmengine - INFO - Iter(train) [12140/13954] lr: 4.3710e-05 eta: 2:10:31 time: 4.2836 data_time: 0.0143 memory: 34809 loss: 2.7080 +2024/02/20 19:45:32 - mmengine - INFO - Iter(train) [12150/13954] lr: 4.3237e-05 eta: 2:09:48 time: 4.2919 data_time: 0.0145 memory: 34634 loss: 2.7580 +2024/02/20 19:46:15 - mmengine - INFO - Iter(train) [12160/13954] lr: 4.2766e-05 eta: 2:09:04 time: 4.2952 data_time: 0.0146 memory: 34680 loss: 2.6800 +2024/02/20 19:46:58 - mmengine - INFO - Iter(train) [12170/13954] lr: 4.2298e-05 eta: 2:08:21 time: 4.3011 data_time: 0.0144 memory: 34774 loss: 2.7751 +2024/02/20 19:47:41 - mmengine - INFO - Iter(train) [12180/13954] lr: 4.1832e-05 eta: 2:07:38 time: 4.2990 data_time: 0.0137 memory: 34868 loss: 2.6731 +2024/02/20 19:48:24 - mmengine - INFO - Iter(train) [12190/13954] lr: 4.1368e-05 eta: 2:06:55 time: 4.2796 data_time: 0.0140 memory: 34680 loss: 2.6502 +2024/02/20 19:49:07 - mmengine - INFO - Iter(train) [12200/13954] lr: 4.0907e-05 eta: 2:06:11 time: 4.2863 data_time: 0.0144 memory: 34774 loss: 2.6945 +2024/02/20 19:49:50 - mmengine - INFO - Iter(train) [12210/13954] lr: 4.0449e-05 eta: 2:05:28 time: 4.2946 data_time: 0.0143 memory: 34727 loss: 2.7124 +2024/02/20 19:50:33 - mmengine - INFO - Iter(train) [12220/13954] lr: 3.9993e-05 eta: 2:04:45 time: 4.2900 data_time: 0.0145 memory: 34869 loss: 2.7502 +2024/02/20 19:51:15 - mmengine - INFO - Iter(train) [12230/13954] lr: 3.9539e-05 eta: 2:04:02 time: 4.2857 data_time: 0.0146 memory: 34962 loss: 2.6756 +2024/02/20 19:51:58 - mmengine - INFO - Iter(train) [12240/13954] lr: 3.9088e-05 eta: 2:03:19 time: 4.2967 data_time: 0.0142 memory: 34869 loss: 2.6574 +2024/02/20 19:52:41 - mmengine - INFO - Iter(train) [12250/13954] lr: 3.8640e-05 eta: 2:02:35 time: 4.2973 data_time: 0.0147 memory: 34727 loss: 2.6809 +2024/02/20 19:53:24 - mmengine - INFO - Iter(train) [12260/13954] lr: 3.8194e-05 eta: 2:01:52 time: 4.2852 data_time: 0.0143 memory: 34680 loss: 2.6705 +2024/02/20 19:54:07 - mmengine - INFO - Iter(train) [12270/13954] lr: 3.7750e-05 eta: 2:01:09 time: 4.3072 data_time: 0.0143 memory: 35242 loss: 2.7289 +2024/02/20 19:54:50 - mmengine - INFO - Iter(train) [12280/13954] lr: 3.7309e-05 eta: 2:00:26 time: 4.2917 data_time: 0.0140 memory: 34868 loss: 2.7213 +2024/02/20 19:55:33 - mmengine - INFO - Iter(train) [12290/13954] lr: 3.6870e-05 eta: 1:59:43 time: 4.3080 data_time: 0.0140 memory: 34822 loss: 2.6711 +2024/02/20 19:56:16 - mmengine - INFO - Iter(train) [12300/13954] lr: 3.6434e-05 eta: 1:58:59 time: 4.2883 data_time: 0.0141 memory: 34821 loss: 2.7443 +2024/02/20 19:56:59 - mmengine - INFO - Iter(train) [12310/13954] lr: 3.6001e-05 eta: 1:58:16 time: 4.2872 data_time: 0.0142 memory: 34634 loss: 2.7646 +2024/02/20 19:57:42 - mmengine - INFO - Iter(train) [12320/13954] lr: 3.5569e-05 eta: 1:57:33 time: 4.2892 data_time: 0.0141 memory: 34774 loss: 2.6710 +2024/02/20 19:58:25 - mmengine - INFO - Iter(train) [12330/13954] lr: 3.5141e-05 eta: 1:56:50 time: 4.2856 data_time: 0.0141 memory: 34680 loss: 2.7079 +2024/02/20 19:59:08 - mmengine - INFO - Iter(train) [12340/13954] lr: 3.4715e-05 eta: 1:56:07 time: 4.3075 data_time: 0.0140 memory: 34822 loss: 2.7597 +2024/02/20 19:59:51 - mmengine - INFO - Iter(train) [12350/13954] lr: 3.4291e-05 eta: 1:55:23 time: 4.3137 data_time: 0.0138 memory: 35102 loss: 2.7398 +2024/02/20 20:00:34 - mmengine - INFO - Iter(train) [12360/13954] lr: 3.3870e-05 eta: 1:54:40 time: 4.3056 data_time: 0.0141 memory: 34621 loss: 2.7653 +2024/02/20 20:01:17 - mmengine - INFO - Iter(train) [12370/13954] lr: 3.3451e-05 eta: 1:53:57 time: 4.3069 data_time: 0.0141 memory: 34774 loss: 2.6602 +2024/02/20 20:02:00 - mmengine - INFO - Iter(train) [12380/13954] lr: 3.3035e-05 eta: 1:53:14 time: 4.3017 data_time: 0.0143 memory: 34774 loss: 2.6803 +2024/02/20 20:02:43 - mmengine - INFO - Iter(train) [12390/13954] lr: 3.2622e-05 eta: 1:52:31 time: 4.3033 data_time: 0.0137 memory: 34634 loss: 2.7003 +2024/02/20 20:03:26 - mmengine - INFO - Iter(train) [12400/13954] lr: 3.2211e-05 eta: 1:51:48 time: 4.2815 data_time: 0.0143 memory: 34680 loss: 2.6879 +2024/02/20 20:04:09 - mmengine - INFO - Iter(train) [12410/13954] lr: 3.1802e-05 eta: 1:51:04 time: 4.2874 data_time: 0.0150 memory: 34589 loss: 2.7267 +2024/02/20 20:04:52 - mmengine - INFO - Iter(train) [12420/13954] lr: 3.1396e-05 eta: 1:50:21 time: 4.2817 data_time: 0.0152 memory: 34727 loss: 2.7331 +2024/02/20 20:05:35 - mmengine - INFO - Iter(train) [12430/13954] lr: 3.0993e-05 eta: 1:49:38 time: 4.2981 data_time: 0.0151 memory: 34774 loss: 2.7085 +2024/02/20 20:06:18 - mmengine - INFO - Iter(train) [12440/13954] lr: 3.0592e-05 eta: 1:48:55 time: 4.2992 data_time: 0.0152 memory: 34762 loss: 2.7006 +2024/02/20 20:07:00 - mmengine - INFO - Iter(train) [12450/13954] lr: 3.0193e-05 eta: 1:48:12 time: 4.2720 data_time: 0.0153 memory: 34774 loss: 2.6996 +2024/02/20 20:07:43 - mmengine - INFO - Iter(train) [12460/13954] lr: 2.9798e-05 eta: 1:47:28 time: 4.2889 data_time: 0.0131 memory: 34680 loss: 2.6430 +2024/02/20 20:08:26 - mmengine - INFO - Iter(train) [12470/13954] lr: 2.9404e-05 eta: 1:46:45 time: 4.2787 data_time: 0.0124 memory: 34774 loss: 2.6600 +2024/02/20 20:09:09 - mmengine - INFO - Iter(train) [12480/13954] lr: 2.9013e-05 eta: 1:46:02 time: 4.3101 data_time: 0.0138 memory: 34715 loss: 2.6803 +2024/02/20 20:09:52 - mmengine - INFO - Iter(train) [12490/13954] lr: 2.8625e-05 eta: 1:45:19 time: 4.2949 data_time: 0.0144 memory: 34589 loss: 2.7658 +2024/02/20 20:10:35 - mmengine - INFO - Iter(train) [12500/13954] lr: 2.8239e-05 eta: 1:44:36 time: 4.3152 data_time: 0.0149 memory: 34727 loss: 2.6965 +2024/02/20 20:10:35 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 20:10:36 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 20:10:36 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on it<|im_end|> + +2024/02/20 20:10:36 - mmengine - INFO - Saving checkpoint at 12500 iterations +2024/02/20 20:11:19 - mmengine - INFO - Iter(train) [12510/13954] lr: 2.7856e-05 eta: 1:43:53 time: 4.3750 data_time: 0.0880 memory: 34822 loss: 2.6761 +2024/02/20 20:12:02 - mmengine - INFO - Iter(train) [12520/13954] lr: 2.7475e-05 eta: 1:43:09 time: 4.3016 data_time: 0.0122 memory: 34680 loss: 2.7169 +2024/02/20 20:12:45 - mmengine - INFO - Iter(train) [12530/13954] lr: 2.7097e-05 eta: 1:42:26 time: 4.2839 data_time: 0.0137 memory: 34774 loss: 2.7554 +2024/02/20 20:13:28 - mmengine - INFO - Iter(train) [12540/13954] lr: 2.6722e-05 eta: 1:41:43 time: 4.3062 data_time: 0.0128 memory: 34715 loss: 2.6878 +2024/02/20 20:14:11 - mmengine - INFO - Iter(train) [12550/13954] lr: 2.6349e-05 eta: 1:41:00 time: 4.3080 data_time: 0.0114 memory: 34868 loss: 2.6922 +2024/02/20 20:14:54 - mmengine - INFO - Iter(train) [12560/13954] lr: 2.5978e-05 eta: 1:40:17 time: 4.2874 data_time: 0.0117 memory: 34634 loss: 2.6834 +2024/02/20 20:15:37 - mmengine - INFO - Iter(train) [12570/13954] lr: 2.5610e-05 eta: 1:39:33 time: 4.3166 data_time: 0.0120 memory: 34869 loss: 2.8021 +2024/02/20 20:16:20 - mmengine - INFO - Iter(train) [12580/13954] lr: 2.5245e-05 eta: 1:38:50 time: 4.2996 data_time: 0.0123 memory: 34762 loss: 2.6419 +2024/02/20 20:17:03 - mmengine - INFO - Iter(train) [12590/13954] lr: 2.4882e-05 eta: 1:38:07 time: 4.2975 data_time: 0.0125 memory: 34762 loss: 2.7235 +2024/02/20 20:17:46 - mmengine - INFO - Iter(train) [12600/13954] lr: 2.4522e-05 eta: 1:37:24 time: 4.3161 data_time: 0.0124 memory: 34634 loss: 2.6444 +2024/02/20 20:18:29 - mmengine - INFO - Iter(train) [12610/13954] lr: 2.4164e-05 eta: 1:36:41 time: 4.3056 data_time: 0.0121 memory: 35196 loss: 2.7103 +2024/02/20 20:19:12 - mmengine - INFO - Iter(train) [12620/13954] lr: 2.3809e-05 eta: 1:35:58 time: 4.2951 data_time: 0.0125 memory: 34774 loss: 2.6785 +2024/02/20 20:19:55 - mmengine - INFO - Iter(train) [12630/13954] lr: 2.3457e-05 eta: 1:35:14 time: 4.2930 data_time: 0.0118 memory: 34868 loss: 2.6713 +2024/02/20 20:20:38 - mmengine - INFO - Iter(train) [12640/13954] lr: 2.3107e-05 eta: 1:34:31 time: 4.3071 data_time: 0.0120 memory: 34774 loss: 2.7278 +2024/02/20 20:21:21 - mmengine - INFO - Iter(train) [12650/13954] lr: 2.2759e-05 eta: 1:33:48 time: 4.3015 data_time: 0.0121 memory: 34727 loss: 2.6955 +2024/02/20 20:22:04 - mmengine - INFO - Iter(train) [12660/13954] lr: 2.2414e-05 eta: 1:33:05 time: 4.3107 data_time: 0.0120 memory: 34868 loss: 2.6550 +2024/02/20 20:22:47 - mmengine - INFO - Iter(train) [12670/13954] lr: 2.2072e-05 eta: 1:32:22 time: 4.3078 data_time: 0.0123 memory: 34727 loss: 2.6769 +2024/02/20 20:23:30 - mmengine - INFO - Iter(train) [12680/13954] lr: 2.1732e-05 eta: 1:31:38 time: 4.3005 data_time: 0.0122 memory: 34856 loss: 2.6622 +2024/02/20 20:24:13 - mmengine - INFO - Iter(train) [12690/13954] lr: 2.1395e-05 eta: 1:30:55 time: 4.2829 data_time: 0.0120 memory: 34634 loss: 2.7728 +2024/02/20 20:24:56 - mmengine - INFO - Iter(train) [12700/13954] lr: 2.1061e-05 eta: 1:30:12 time: 4.3172 data_time: 0.0120 memory: 35196 loss: 2.7287 +2024/02/20 20:25:39 - mmengine - INFO - Iter(train) [12710/13954] lr: 2.0729e-05 eta: 1:29:29 time: 4.3038 data_time: 0.0118 memory: 34634 loss: 2.7242 +2024/02/20 20:26:22 - mmengine - INFO - Iter(train) [12720/13954] lr: 2.0399e-05 eta: 1:28:46 time: 4.3003 data_time: 0.0118 memory: 34868 loss: 2.6765 +2024/02/20 20:27:05 - mmengine - INFO - Iter(train) [12730/13954] lr: 2.0073e-05 eta: 1:28:03 time: 4.2911 data_time: 0.0121 memory: 34680 loss: 2.6506 +2024/02/20 20:27:48 - mmengine - INFO - Iter(train) [12740/13954] lr: 1.9748e-05 eta: 1:27:19 time: 4.2954 data_time: 0.0119 memory: 34868 loss: 2.7335 +2024/02/20 20:28:31 - mmengine - INFO - Iter(train) [12750/13954] lr: 1.9427e-05 eta: 1:26:36 time: 4.2944 data_time: 0.0121 memory: 34715 loss: 2.7025 +2024/02/20 20:29:14 - mmengine - INFO - Iter(train) [12760/13954] lr: 1.9108e-05 eta: 1:25:53 time: 4.3060 data_time: 0.0118 memory: 34634 loss: 2.7128 +2024/02/20 20:29:57 - mmengine - INFO - Iter(train) [12770/13954] lr: 1.8791e-05 eta: 1:25:10 time: 4.2966 data_time: 0.0122 memory: 34727 loss: 2.6740 +2024/02/20 20:30:40 - mmengine - INFO - Iter(train) [12780/13954] lr: 1.8477e-05 eta: 1:24:27 time: 4.2911 data_time: 0.0118 memory: 34810 loss: 2.7920 +2024/02/20 20:31:23 - mmengine - INFO - Iter(train) [12790/13954] lr: 1.8166e-05 eta: 1:23:43 time: 4.2950 data_time: 0.0119 memory: 34680 loss: 2.7627 +2024/02/20 20:32:06 - mmengine - INFO - Iter(train) [12800/13954] lr: 1.7858e-05 eta: 1:23:00 time: 4.2712 data_time: 0.0119 memory: 34589 loss: 2.7453 +2024/02/20 20:32:49 - mmengine - INFO - Iter(train) [12810/13954] lr: 1.7551e-05 eta: 1:22:17 time: 4.2946 data_time: 0.0121 memory: 34822 loss: 2.5928 +2024/02/20 20:33:32 - mmengine - INFO - Iter(train) [12820/13954] lr: 1.7248e-05 eta: 1:21:34 time: 4.2964 data_time: 0.0118 memory: 34727 loss: 2.7480 +2024/02/20 20:34:15 - mmengine - INFO - Iter(train) [12830/13954] lr: 1.6947e-05 eta: 1:20:51 time: 4.3140 data_time: 0.0118 memory: 34727 loss: 2.7782 +2024/02/20 20:34:58 - mmengine - INFO - Iter(train) [12840/13954] lr: 1.6649e-05 eta: 1:20:08 time: 4.3143 data_time: 0.0125 memory: 34915 loss: 2.7925 +2024/02/20 20:35:41 - mmengine - INFO - Iter(train) [12850/13954] lr: 1.6353e-05 eta: 1:19:24 time: 4.2987 data_time: 0.0122 memory: 34822 loss: 2.6567 +2024/02/20 20:36:24 - mmengine - INFO - Iter(train) [12860/13954] lr: 1.6060e-05 eta: 1:18:41 time: 4.3011 data_time: 0.0128 memory: 34821 loss: 2.7259 +2024/02/20 20:37:07 - mmengine - INFO - Iter(train) [12870/13954] lr: 1.5770e-05 eta: 1:17:58 time: 4.2959 data_time: 0.0127 memory: 34539 loss: 2.7211 +2024/02/20 20:37:50 - mmengine - INFO - Iter(train) [12880/13954] lr: 1.5482e-05 eta: 1:17:15 time: 4.3001 data_time: 0.0126 memory: 34774 loss: 2.6889 +2024/02/20 20:38:33 - mmengine - INFO - Iter(train) [12890/13954] lr: 1.5197e-05 eta: 1:16:32 time: 4.2887 data_time: 0.0129 memory: 34589 loss: 2.7121 +2024/02/20 20:39:16 - mmengine - INFO - Iter(train) [12900/13954] lr: 1.4914e-05 eta: 1:15:49 time: 4.2907 data_time: 0.0127 memory: 34540 loss: 2.6999 +2024/02/20 20:39:59 - mmengine - INFO - Iter(train) [12910/13954] lr: 1.4634e-05 eta: 1:15:05 time: 4.3179 data_time: 0.0126 memory: 34997 loss: 2.5955 +2024/02/20 20:40:42 - mmengine - INFO - Iter(train) [12920/13954] lr: 1.4357e-05 eta: 1:14:22 time: 4.2909 data_time: 0.0126 memory: 34680 loss: 2.7157 +2024/02/20 20:41:25 - mmengine - INFO - Iter(train) [12930/13954] lr: 1.4082e-05 eta: 1:13:39 time: 4.3041 data_time: 0.0126 memory: 34715 loss: 2.6748 +2024/02/20 20:42:08 - mmengine - INFO - Iter(train) [12940/13954] lr: 1.3810e-05 eta: 1:12:56 time: 4.3222 data_time: 0.0125 memory: 34903 loss: 2.7033 +2024/02/20 20:42:51 - mmengine - INFO - Iter(train) [12950/13954] lr: 1.3540e-05 eta: 1:12:13 time: 4.2735 data_time: 0.0127 memory: 34680 loss: 2.6971 +2024/02/20 20:43:34 - mmengine - INFO - Iter(train) [12960/13954] lr: 1.3273e-05 eta: 1:11:30 time: 4.3034 data_time: 0.0126 memory: 34715 loss: 2.7221 +2024/02/20 20:44:17 - mmengine - INFO - Iter(train) [12970/13954] lr: 1.3009e-05 eta: 1:10:46 time: 4.2950 data_time: 0.0127 memory: 34856 loss: 2.7395 +2024/02/20 20:45:00 - mmengine - INFO - Iter(train) [12980/13954] lr: 1.2747e-05 eta: 1:10:03 time: 4.2926 data_time: 0.0128 memory: 34540 loss: 2.7965 +2024/02/20 20:45:43 - mmengine - INFO - Iter(train) [12990/13954] lr: 1.2488e-05 eta: 1:09:20 time: 4.2851 data_time: 0.0128 memory: 34577 loss: 2.7415 +2024/02/20 20:46:26 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 20:46:26 - mmengine - INFO - Iter(train) [13000/13954] lr: 1.2232e-05 eta: 1:08:37 time: 4.3160 data_time: 0.0131 memory: 35196 loss: 2.7281 +2024/02/20 20:46:26 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 20:46:26 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 20:46:26 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 20:46:26 - mmengine - INFO - Saving checkpoint at 13000 iterations +2024/02/20 20:47:09 - mmengine - INFO - Iter(train) [13010/13954] lr: 1.1978e-05 eta: 1:07:54 time: 4.3600 data_time: 0.1024 memory: 34589 loss: 2.7114 +2024/02/20 20:47:52 - mmengine - INFO - Iter(train) [13020/13954] lr: 1.1727e-05 eta: 1:07:11 time: 4.2888 data_time: 0.0126 memory: 34856 loss: 2.6998 +2024/02/20 20:48:35 - mmengine - INFO - Iter(train) [13030/13954] lr: 1.1478e-05 eta: 1:06:27 time: 4.2742 data_time: 0.0125 memory: 34540 loss: 2.6727 +2024/02/20 20:49:18 - mmengine - INFO - Iter(train) [13040/13954] lr: 1.1232e-05 eta: 1:05:44 time: 4.3049 data_time: 0.0122 memory: 34868 loss: 2.6644 +2024/02/20 20:50:01 - mmengine - INFO - Iter(train) [13050/13954] lr: 1.0989e-05 eta: 1:05:01 time: 4.2872 data_time: 0.0119 memory: 34997 loss: 2.6612 +2024/02/20 20:50:44 - mmengine - INFO - Iter(train) [13060/13954] lr: 1.0748e-05 eta: 1:04:18 time: 4.3066 data_time: 0.0135 memory: 34727 loss: 2.7741 +2024/02/20 20:51:27 - mmengine - INFO - Iter(train) [13070/13954] lr: 1.0510e-05 eta: 1:03:35 time: 4.3030 data_time: 0.0136 memory: 34774 loss: 2.7984 +2024/02/20 20:52:10 - mmengine - INFO - Iter(train) [13080/13954] lr: 1.0275e-05 eta: 1:02:51 time: 4.3035 data_time: 0.0134 memory: 34680 loss: 2.6201 +2024/02/20 20:52:53 - mmengine - INFO - Iter(train) [13090/13954] lr: 1.0042e-05 eta: 1:02:08 time: 4.3124 data_time: 0.0135 memory: 34680 loss: 2.7209 +2024/02/20 20:53:36 - mmengine - INFO - Iter(train) [13100/13954] lr: 9.8122e-06 eta: 1:01:25 time: 4.3086 data_time: 0.0140 memory: 34774 loss: 2.6921 +2024/02/20 20:54:20 - mmengine - INFO - Iter(train) [13110/13954] lr: 9.5847e-06 eta: 1:00:42 time: 4.3370 data_time: 0.0140 memory: 35102 loss: 2.7243 +2024/02/20 20:55:03 - mmengine - INFO - Iter(train) [13120/13954] lr: 9.3599e-06 eta: 0:59:59 time: 4.2918 data_time: 0.0142 memory: 34774 loss: 2.7131 +2024/02/20 20:55:46 - mmengine - INFO - Iter(train) [13130/13954] lr: 9.1378e-06 eta: 0:59:16 time: 4.2961 data_time: 0.0141 memory: 34727 loss: 2.7192 +2024/02/20 20:56:29 - mmengine - INFO - Iter(train) [13140/13954] lr: 8.9182e-06 eta: 0:58:33 time: 4.2922 data_time: 0.0138 memory: 34680 loss: 2.7094 +2024/02/20 20:57:11 - mmengine - INFO - Iter(train) [13150/13954] lr: 8.7014e-06 eta: 0:57:49 time: 4.2940 data_time: 0.0141 memory: 34727 loss: 2.7633 +2024/02/20 20:57:54 - mmengine - INFO - Iter(train) [13160/13954] lr: 8.4871e-06 eta: 0:57:06 time: 4.2825 data_time: 0.0140 memory: 34680 loss: 2.6230 +2024/02/20 20:58:37 - mmengine - INFO - Iter(train) [13170/13954] lr: 8.2755e-06 eta: 0:56:23 time: 4.2934 data_time: 0.0138 memory: 34680 loss: 2.7744 +2024/02/20 20:59:20 - mmengine - INFO - Iter(train) [13180/13954] lr: 8.0666e-06 eta: 0:55:40 time: 4.2853 data_time: 0.0135 memory: 34822 loss: 2.7967 +2024/02/20 21:00:03 - mmengine - INFO - Iter(train) [13190/13954] lr: 7.8603e-06 eta: 0:54:57 time: 4.3059 data_time: 0.0127 memory: 35195 loss: 2.6344 +2024/02/20 21:00:46 - mmengine - INFO - Iter(train) [13200/13954] lr: 7.6567e-06 eta: 0:54:13 time: 4.2844 data_time: 0.0138 memory: 34680 loss: 2.7422 +2024/02/20 21:01:29 - mmengine - INFO - Iter(train) [13210/13954] lr: 7.4557e-06 eta: 0:53:30 time: 4.2937 data_time: 0.0140 memory: 34622 loss: 2.6671 +2024/02/20 21:02:12 - mmengine - INFO - Iter(train) [13220/13954] lr: 7.2574e-06 eta: 0:52:47 time: 4.2818 data_time: 0.0127 memory: 34822 loss: 2.6724 +2024/02/20 21:02:55 - mmengine - INFO - Iter(train) [13230/13954] lr: 7.0617e-06 eta: 0:52:04 time: 4.3172 data_time: 0.0124 memory: 34727 loss: 2.7107 +2024/02/20 21:03:38 - mmengine - INFO - Iter(train) [13240/13954] lr: 6.8687e-06 eta: 0:51:21 time: 4.2924 data_time: 0.0125 memory: 35009 loss: 2.7663 +2024/02/20 21:04:21 - mmengine - INFO - Iter(train) [13250/13954] lr: 6.6783e-06 eta: 0:50:38 time: 4.2909 data_time: 0.0124 memory: 34868 loss: 2.6126 +2024/02/20 21:05:03 - mmengine - INFO - Iter(train) [13260/13954] lr: 6.4906e-06 eta: 0:49:54 time: 4.2737 data_time: 0.0119 memory: 34774 loss: 2.7263 +2024/02/20 21:05:46 - mmengine - INFO - Iter(train) [13270/13954] lr: 6.3056e-06 eta: 0:49:11 time: 4.2859 data_time: 0.0126 memory: 34868 loss: 2.7327 +2024/02/20 21:06:29 - mmengine - INFO - Iter(train) [13280/13954] lr: 6.1232e-06 eta: 0:48:28 time: 4.2910 data_time: 0.0130 memory: 34680 loss: 2.8249 +2024/02/20 21:07:12 - mmengine - INFO - Iter(train) [13290/13954] lr: 5.9435e-06 eta: 0:47:45 time: 4.2999 data_time: 0.0129 memory: 34915 loss: 2.6014 +2024/02/20 21:07:55 - mmengine - INFO - Iter(train) [13300/13954] lr: 5.7664e-06 eta: 0:47:02 time: 4.3058 data_time: 0.0126 memory: 34774 loss: 2.7475 +2024/02/20 21:08:38 - mmengine - INFO - Iter(train) [13310/13954] lr: 5.5920e-06 eta: 0:46:19 time: 4.3081 data_time: 0.0140 memory: 34727 loss: 2.6652 +2024/02/20 21:09:21 - mmengine - INFO - Iter(train) [13320/13954] lr: 5.4203e-06 eta: 0:45:35 time: 4.3094 data_time: 0.0147 memory: 34774 loss: 2.7773 +2024/02/20 21:10:04 - mmengine - INFO - Iter(train) [13330/13954] lr: 5.2512e-06 eta: 0:44:52 time: 4.3016 data_time: 0.0140 memory: 34727 loss: 2.6491 +2024/02/20 21:10:48 - mmengine - INFO - Iter(train) [13340/13954] lr: 5.0848e-06 eta: 0:44:09 time: 4.3073 data_time: 0.0143 memory: 34868 loss: 2.6388 +2024/02/20 21:11:30 - mmengine - INFO - Iter(train) [13350/13954] lr: 4.9210e-06 eta: 0:43:26 time: 4.2794 data_time: 0.0141 memory: 34727 loss: 2.7238 +2024/02/20 21:12:13 - mmengine - INFO - Iter(train) [13360/13954] lr: 4.7600e-06 eta: 0:42:43 time: 4.2947 data_time: 0.0139 memory: 34856 loss: 2.7129 +2024/02/20 21:12:57 - mmengine - INFO - Iter(train) [13370/13954] lr: 4.6015e-06 eta: 0:42:00 time: 4.3225 data_time: 0.0140 memory: 34903 loss: 2.6506 +2024/02/20 21:13:40 - mmengine - INFO - Iter(train) [13380/13954] lr: 4.4458e-06 eta: 0:41:17 time: 4.3209 data_time: 0.0134 memory: 34774 loss: 2.7161 +2024/02/20 21:14:23 - mmengine - INFO - Iter(train) [13390/13954] lr: 4.2927e-06 eta: 0:40:33 time: 4.3114 data_time: 0.0147 memory: 34680 loss: 2.6441 +2024/02/20 21:15:06 - mmengine - INFO - Iter(train) [13400/13954] lr: 4.1423e-06 eta: 0:39:50 time: 4.3275 data_time: 0.0140 memory: 35196 loss: 2.6943 +2024/02/20 21:15:49 - mmengine - INFO - Iter(train) [13410/13954] lr: 3.9946e-06 eta: 0:39:07 time: 4.3152 data_time: 0.0143 memory: 34774 loss: 2.6762 +2024/02/20 21:16:32 - mmengine - INFO - Iter(train) [13420/13954] lr: 3.8495e-06 eta: 0:38:24 time: 4.2904 data_time: 0.0144 memory: 34822 loss: 2.7457 +2024/02/20 21:17:15 - mmengine - INFO - Iter(train) [13430/13954] lr: 3.7072e-06 eta: 0:37:41 time: 4.3142 data_time: 0.0143 memory: 34822 loss: 2.7106 +2024/02/20 21:17:58 - mmengine - INFO - Iter(train) [13440/13954] lr: 3.5674e-06 eta: 0:36:58 time: 4.2887 data_time: 0.0142 memory: 34774 loss: 2.6654 +2024/02/20 21:18:41 - mmengine - INFO - Iter(train) [13450/13954] lr: 3.4304e-06 eta: 0:36:14 time: 4.3187 data_time: 0.0140 memory: 34774 loss: 2.7035 +2024/02/20 21:19:24 - mmengine - INFO - Iter(train) [13460/13954] lr: 3.2960e-06 eta: 0:35:31 time: 4.2977 data_time: 0.0141 memory: 34868 loss: 2.6780 +2024/02/20 21:20:07 - mmengine - INFO - Iter(train) [13470/13954] lr: 3.1643e-06 eta: 0:34:48 time: 4.3061 data_time: 0.0142 memory: 34822 loss: 2.7022 +2024/02/20 21:20:50 - mmengine - INFO - Iter(train) [13480/13954] lr: 3.0353e-06 eta: 0:34:05 time: 4.3064 data_time: 0.0142 memory: 34997 loss: 2.6814 +2024/02/20 21:21:34 - mmengine - INFO - Iter(train) [13490/13954] lr: 2.9090e-06 eta: 0:33:22 time: 4.3067 data_time: 0.0144 memory: 34915 loss: 2.7351 +2024/02/20 21:22:16 - mmengine - INFO - Iter(train) [13500/13954] lr: 2.7853e-06 eta: 0:32:39 time: 4.2788 data_time: 0.0143 memory: 34540 loss: 2.6999 +2024/02/20 21:22:16 - mmengine - INFO - after_train_iter in EvaluateChatHook. +2024/02/20 21:22:17 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 21:22:17 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 21:22:17 - mmengine - INFO - Saving checkpoint at 13500 iterations +2024/02/20 21:23:00 - mmengine - INFO - Iter(train) [13510/13954] lr: 2.6644e-06 eta: 0:31:56 time: 4.4003 data_time: 0.0911 memory: 36086 loss: 2.6993 +2024/02/20 21:23:43 - mmengine - INFO - Iter(train) [13520/13954] lr: 2.5461e-06 eta: 0:31:12 time: 4.2826 data_time: 0.0144 memory: 34577 loss: 2.6778 +2024/02/20 21:24:26 - mmengine - INFO - Iter(train) [13530/13954] lr: 2.4304e-06 eta: 0:30:29 time: 4.2968 data_time: 0.0143 memory: 34634 loss: 2.7449 +2024/02/20 21:25:09 - mmengine - INFO - Iter(train) [13540/13954] lr: 2.3175e-06 eta: 0:29:46 time: 4.3034 data_time: 0.0143 memory: 34774 loss: 2.6542 +2024/02/20 21:25:52 - mmengine - INFO - Iter(train) [13550/13954] lr: 2.2072e-06 eta: 0:29:03 time: 4.2823 data_time: 0.0141 memory: 34622 loss: 2.7760 +2024/02/20 21:26:35 - mmengine - INFO - Iter(train) [13560/13954] lr: 2.0997e-06 eta: 0:28:20 time: 4.3254 data_time: 0.0143 memory: 36086 loss: 2.6822 +2024/02/20 21:27:18 - mmengine - INFO - Iter(train) [13570/13954] lr: 1.9948e-06 eta: 0:27:37 time: 4.2922 data_time: 0.0141 memory: 34868 loss: 2.7055 +2024/02/20 21:28:01 - mmengine - INFO - Iter(train) [13580/13954] lr: 1.8926e-06 eta: 0:26:53 time: 4.3012 data_time: 0.0141 memory: 34634 loss: 2.7634 +2024/02/20 21:28:44 - mmengine - INFO - Iter(train) [13590/13954] lr: 1.7930e-06 eta: 0:26:10 time: 4.2969 data_time: 0.0142 memory: 34668 loss: 2.7634 +2024/02/20 21:29:27 - mmengine - INFO - Iter(train) [13600/13954] lr: 1.6962e-06 eta: 0:25:27 time: 4.2863 data_time: 0.0142 memory: 34680 loss: 2.6463 +2024/02/20 21:30:10 - mmengine - INFO - Iter(train) [13610/13954] lr: 1.6020e-06 eta: 0:24:44 time: 4.2720 data_time: 0.0141 memory: 34668 loss: 2.7645 +2024/02/20 21:30:53 - mmengine - INFO - Iter(train) [13620/13954] lr: 1.5105e-06 eta: 0:24:01 time: 4.3012 data_time: 0.0145 memory: 34822 loss: 2.7326 +2024/02/20 21:31:36 - mmengine - INFO - Iter(train) [13630/13954] lr: 1.4217e-06 eta: 0:23:18 time: 4.3025 data_time: 0.0143 memory: 34774 loss: 2.7044 +2024/02/20 21:32:19 - mmengine - INFO - Iter(train) [13640/13954] lr: 1.3356e-06 eta: 0:22:34 time: 4.3066 data_time: 0.0143 memory: 34821 loss: 2.6807 +2024/02/20 21:33:02 - mmengine - INFO - Iter(train) [13650/13954] lr: 1.2522e-06 eta: 0:21:51 time: 4.2967 data_time: 0.0142 memory: 34680 loss: 2.7149 +2024/02/20 21:33:45 - mmengine - INFO - Iter(train) [13660/13954] lr: 1.1715e-06 eta: 0:21:08 time: 4.3143 data_time: 0.0141 memory: 34680 loss: 2.6415 +2024/02/20 21:34:28 - mmengine - INFO - Iter(train) [13670/13954] lr: 1.0934e-06 eta: 0:20:25 time: 4.2848 data_time: 0.0134 memory: 34634 loss: 2.7166 +2024/02/20 21:35:11 - mmengine - INFO - Iter(train) [13680/13954] lr: 1.0181e-06 eta: 0:19:42 time: 4.2970 data_time: 0.0130 memory: 34681 loss: 2.7073 +2024/02/20 21:35:54 - mmengine - INFO - Iter(train) [13690/13954] lr: 9.4540e-07 eta: 0:18:59 time: 4.3280 data_time: 0.0124 memory: 34915 loss: 2.6852 +2024/02/20 21:36:37 - mmengine - INFO - Iter(train) [13700/13954] lr: 8.7541e-07 eta: 0:18:16 time: 4.3254 data_time: 0.0129 memory: 34822 loss: 2.6896 +2024/02/20 21:37:20 - mmengine - INFO - Iter(train) [13710/13954] lr: 8.0812e-07 eta: 0:17:32 time: 4.3072 data_time: 0.0128 memory: 34727 loss: 2.7459 +2024/02/20 21:38:03 - mmengine - INFO - Iter(train) [13720/13954] lr: 7.4351e-07 eta: 0:16:49 time: 4.3015 data_time: 0.0128 memory: 34634 loss: 2.6521 +2024/02/20 21:38:47 - mmengine - INFO - Iter(train) [13730/13954] lr: 6.8159e-07 eta: 0:16:06 time: 4.3112 data_time: 0.0142 memory: 34822 loss: 2.7479 +2024/02/20 21:39:30 - mmengine - INFO - Iter(train) [13740/13954] lr: 6.2237e-07 eta: 0:15:23 time: 4.3095 data_time: 0.0141 memory: 34680 loss: 2.7127 +2024/02/20 21:40:13 - mmengine - INFO - Iter(train) [13750/13954] lr: 5.6583e-07 eta: 0:14:40 time: 4.3005 data_time: 0.0137 memory: 34822 loss: 2.8003 +2024/02/20 21:40:56 - mmengine - INFO - Iter(train) [13760/13954] lr: 5.1198e-07 eta: 0:13:57 time: 4.3125 data_time: 0.0134 memory: 35664 loss: 2.6601 +2024/02/20 21:41:39 - mmengine - INFO - Iter(train) [13770/13954] lr: 4.6082e-07 eta: 0:13:13 time: 4.2816 data_time: 0.0132 memory: 34774 loss: 2.7529 +2024/02/20 21:42:22 - mmengine - INFO - Iter(train) [13780/13954] lr: 4.1236e-07 eta: 0:12:30 time: 4.3091 data_time: 0.0142 memory: 34634 loss: 2.7155 +2024/02/20 21:43:04 - mmengine - INFO - Iter(train) [13790/13954] lr: 3.6658e-07 eta: 0:11:47 time: 4.2808 data_time: 0.0141 memory: 34589 loss: 2.8013 +2024/02/20 21:43:47 - mmengine - INFO - Iter(train) [13800/13954] lr: 3.2350e-07 eta: 0:11:04 time: 4.2903 data_time: 0.0130 memory: 34634 loss: 2.6991 +2024/02/20 21:44:31 - mmengine - INFO - Iter(train) [13810/13954] lr: 2.8311e-07 eta: 0:10:21 time: 4.3161 data_time: 0.0130 memory: 34915 loss: 2.7851 +2024/02/20 21:45:14 - mmengine - INFO - Iter(train) [13820/13954] lr: 2.4541e-07 eta: 0:09:38 time: 4.3008 data_time: 0.0133 memory: 34774 loss: 2.5998 +2024/02/20 21:45:56 - mmengine - INFO - Iter(train) [13830/13954] lr: 2.1040e-07 eta: 0:08:55 time: 4.2873 data_time: 0.0129 memory: 34774 loss: 2.5961 +2024/02/20 21:46:39 - mmengine - INFO - Iter(train) [13840/13954] lr: 1.7809e-07 eta: 0:08:11 time: 4.2842 data_time: 0.0123 memory: 34634 loss: 2.7554 +2024/02/20 21:47:22 - mmengine - INFO - Iter(train) [13850/13954] lr: 1.4846e-07 eta: 0:07:28 time: 4.3153 data_time: 0.0122 memory: 34727 loss: 2.6034 +2024/02/20 21:48:05 - mmengine - INFO - Iter(train) [13860/13954] lr: 1.2153e-07 eta: 0:06:45 time: 4.3015 data_time: 0.0121 memory: 34540 loss: 2.6593 +2024/02/20 21:48:48 - mmengine - INFO - Iter(train) [13870/13954] lr: 9.7293e-08 eta: 0:06:02 time: 4.2937 data_time: 0.0123 memory: 34856 loss: 2.6845 +2024/02/20 21:49:32 - mmengine - INFO - Iter(train) [13880/13954] lr: 7.5748e-08 eta: 0:05:19 time: 4.3396 data_time: 0.0123 memory: 34727 loss: 2.6669 +2024/02/20 21:50:15 - mmengine - INFO - Iter(train) [13890/13954] lr: 5.6895e-08 eta: 0:04:36 time: 4.3102 data_time: 0.0121 memory: 34869 loss: 2.6778 +2024/02/20 21:50:58 - mmengine - INFO - Iter(train) [13900/13954] lr: 4.0736e-08 eta: 0:03:53 time: 4.3269 data_time: 0.0121 memory: 34715 loss: 2.6861 +2024/02/20 21:51:41 - mmengine - INFO - Iter(train) [13910/13954] lr: 2.7270e-08 eta: 0:03:09 time: 4.3121 data_time: 0.0121 memory: 34589 loss: 2.7282 +2024/02/20 21:52:24 - mmengine - INFO - Iter(train) [13920/13954] lr: 1.6497e-08 eta: 0:02:26 time: 4.2916 data_time: 0.0122 memory: 34680 loss: 2.7127 +2024/02/20 21:53:07 - mmengine - INFO - Iter(train) [13930/13954] lr: 8.4166e-09 eta: 0:01:43 time: 4.2979 data_time: 0.0122 memory: 34540 loss: 2.6147 +2024/02/20 21:53:50 - mmengine - INFO - Iter(train) [13940/13954] lr: 3.0300e-09 eta: 0:01:00 time: 4.3029 data_time: 0.0121 memory: 34727 loss: 2.7822 +2024/02/20 21:54:33 - mmengine - INFO - Iter(train) [13950/13954] lr: 3.3667e-10 eta: 0:00:17 time: 4.2898 data_time: 0.0127 memory: 34680 loss: 2.7388 +2024/02/20 21:54:47 - mmengine - INFO - Exp name: llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy_20240220_050613 +2024/02/20 21:54:47 - mmengine - INFO - Saving checkpoint at 13954 iterations +2024/02/20 21:54:47 - mmengine - INFO - after_train in EvaluateChatHook. +2024/02/20 21:54:47 - mmengine - INFO - Sample output: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + +2024/02/20 21:54:48 - mmengine - INFO - Sample output: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10499.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb99fde6fccdd482376dd19e69583156b75e8870 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake tahoe, california<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake lakelake in the mountains<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10999.txt new file mode 100644 index 0000000000000000000000000000000000000000..372f24ad3fe5e18d789df4a25add9152856ba238 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_10999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on it<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11499.txt new file mode 100644 index 0000000000000000000000000000000000000000..ccd1759a41f8c20d7fcf390972b099fd43ebcec9 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the shore<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11999.txt new file mode 100644 index 0000000000000000000000000000000000000000..e92faf9b72202a357401b4234d54f6109cf3c6cf --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_11999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12499.txt new file mode 100644 index 0000000000000000000000000000000000000000..338eb8375e85c224b547089925a525e5af0d3110 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12999.txt new file mode 100644 index 0000000000000000000000000000000000000000..e92faf9b72202a357401b4234d54f6109cf3c6cf --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_12999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_13499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_13499.txt new file mode 100644 index 0000000000000000000000000000000000000000..e92faf9b72202a357401b4234d54f6109cf3c6cf --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_13499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1499.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac2615ddad6be1eeead1a0244a267e9c1d6dbd3a --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a view of the water<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a view of the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1999.txt new file mode 100644 index 0000000000000000000000000000000000000000..f87278dfe299462699f12cf3977b9ffddf18a880 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_1999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a wooden pier on a lake in the mountains<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a wooden pier with a boat on the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2499.txt new file mode 100644 index 0000000000000000000000000000000000000000..4ac7fbaa265bb1b924efc324555189096a0ef104 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock in the lake with a boat on the shore<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2999.txt new file mode 100644 index 0000000000000000000000000000000000000000..acee8c2efd6ad4ba911ce8bcaac527768d1d22ac --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_2999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3499.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b81fee497043027b97fd80ad18559dfbeeb2a65 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock in the background<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a boat dock in the background<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3999.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c2db46d752e2a27ddd95ade21c6818b2cc57418 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_3999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock with a boat and a lake in the background<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock with a boat on the water and a bridge over it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4499.txt new file mode 100644 index 0000000000000000000000000000000000000000..3cd37dbe94b209f3e2582a3b457b4ca8baf5b3a7 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a pier on a lake with a boat on the pier<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_499.txt new file mode 100644 index 0000000000000000000000000000000000000000..e594e7aa2e38674c6e262f9b7074331f8d08e6ae --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a wooden bridge in the mountains with a view of the lake<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a wooden bridge in the mountains with a view of the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4999.txt new file mode 100644 index 0000000000000000000000000000000000000000..355822bc82dbb18f21d293202c311a53fcbe66fb --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_4999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat and a boat house<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5499.txt new file mode 100644 index 0000000000000000000000000000000000000000..a04eb0dc7dd9a74af99d1febf74a6ef08143d0af --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock and pier on a lake with a boat<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock with a boat on the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5999.txt new file mode 100644 index 0000000000000000000000000000000000000000..e78eee2867275dfd93581aa6ee9e0e53d8ec444e --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_5999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on the water<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6499.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e9d2da49f3de98b9ddf1930165bbc28c990285e --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake with a boat on the shore<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6999.txt new file mode 100644 index 0000000000000000000000000000000000000000..1f4e9490a26957153dc1e0fd74e63c2827056093 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_6999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7499.txt new file mode 100644 index 0000000000000000000000000000000000000000..779a93c3b872634f1aab084175323ec487fcee7b --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat and a dock<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7999.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ee8fc981dabfcc5559cca59c9ae3c560fc8ca5e --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_7999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8499.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b58d234567f5ac0f4b1fe3591c23ec3fd1b448d --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock on a lake with a boat on the shore<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8999.txt new file mode 100644 index 0000000000000000000000000000000000000000..dd734477ebd5ce8c2452cee335707dd93c6a995c --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_8999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore photo<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat on the shore photo<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9499.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9499.txt new file mode 100644 index 0000000000000000000000000000000000000000..fc49532c4c35eb38c60fff35b46263274f65fae0 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9499.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at the lake with a boat on the water<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at a lake with a boat on it<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_999.txt new file mode 100644 index 0000000000000000000000000000000000000000..8e666b980539f03eed95076c4eabdfd7d6ae9a8c --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with a mountain range in the background<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a bridge over a lake in the mountains with the sun setting over the water<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9999.txt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9999.txt new file mode 100644 index 0000000000000000000000000000000000000000..82f2fb54c53c85743a9bb0cd7d79e45792a3b572 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/eval_outputs_iter_9999.txt @@ -0,0 +1,16 @@ +Eval output 1: +<|im_start|>user + +请描述一下这张照片<|im_end|> +<|im_start|>assistant +a dock at lake with a boat and a mountain in the background<|im_end|> + + +Eval output 2: +<|im_start|>user + +Please describe this picture<|im_end|> +<|im_start|>assistant +a dock at lake with a boat and a mountain in the background<|im_end|> + + diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/events.out.tfevents.1708405574.b447a125019d.77998.0 b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/events.out.tfevents.1708405574.b447a125019d.77998.0 new file mode 100644 index 0000000000000000000000000000000000000000..10d91f6765fd9f92670f197e6b77ddd4693514d5 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/20240220_050613/vis_data/events.out.tfevents.1708405574.b447a125019d.77998.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:863398f4d30a127fd2f17932e0255bfa5bfa92e3d4c6d0ff923551593d2ceb9e +size 374431 diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d61e148e79cc2b2fb2ed8f54e0c95bd07e76972 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0232b6c3371d3eea84f49847dbe95ea31d06a597ac58be51fb665d91b4af35b +size 75549943 diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/mp_rank_00_model_states.pt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a840d6dc5b66680298dbcf60f69db53c0f31168a --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13500.pth/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49a35652d7c91ea73e7e30eb753801f822f4e6f59b12406dbbcbd2dd6fa419a4 +size 13577563 diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8511279215e0544d4779dfe768ff8cddd396bc74 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853fcd7c7865d95e9016b23b703eb1ff1a80448f63342b28c2163b55b243d8f4 +size 75549943 diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/mp_rank_00_model_states.pt b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0681dbebad2119a88cddff8bc3b84e02606bb0ef --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/iter_13954.pth/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:680d780055ad0f155550e80fc19719369865df98d9faa5ab5c4badbf205a902d +size 13610395 diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/last_checkpoint b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/last_checkpoint new file mode 100644 index 0000000000000000000000000000000000000000..fa55a3e1fa3697182d3a5f276a6175cc0ae3f227 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/last_checkpoint @@ -0,0 +1 @@ +/workspace/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain_copy/iter_13954.pth \ No newline at end of file diff --git a/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/zero_to_fp32.py b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/zero_to_fp32.py new file mode 100755 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/work_dirs/llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu1_pretrain/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)