Training in progress, step 400

axolotl_config.yaml

@@ -13,7 +13,9 @@ datasets:
     no_input_format: '{instruction}'
     system_format: '{system}'
     system_prompt: ''
-eval_steps: 20
+device_map: auto
+eval_sample_packing: false
+eval_steps: 200
 flash_attention: true
 gpu_memory_limit: 80GiB
 group_by_length: true
@@ -23,17 +25,19 @@ learning_rate: 0.0002
 logging_steps: 10
 lr_scheduler: cosine
 max_steps: 2500
-micro_batch_size: 2
+micro_batch_size: 4
 model_type: AutoModelForCausalLM
 num_epochs: 100
 optimizer: adamw_bnb_8bit
 output_dir: /root/.sn56/axolotl/outputs/SmolLM-360M
 pad_to_sequence_len: true
 resize_token_embeddings_to_32x: false
-save_steps: 40
+sample_packing: false
+save_steps: 400
 save_total_limit: 1
 sequence_len: 2048
 tokenizer_type: GPT2TokenizerFast
+torch_dtype: bf16
 trust_remote_code: true
 val_set_size: 0.1
 wandb_entity: ''

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:373e5049b16a04ba5b60f23f70dbec74642cccc842345b60c717c3089d6eea92
+oid sha256:19465df8c765c100b1ba5242ef17b58f9b637c549a87ac2747076fa42c39e0a6
 size 723674912

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ccb5d11ed0b2e61fe01472d61a2a7bde594d06ab5bfd5fb60139b845efe088a
-size 6712
+oid sha256:112b4c8284645a698146b4282cd59099e5d2d309ac0c12020a3a74886e780a25
+size 6840