Training in progress, step 200, checkpoint

last-checkpoint/config.json

@@ -1,31 +1,28 @@
 {
-  "_name_or_path": "trl-internal-testing/tiny-random-LlamaForCausalLM",
+  "_name_or_path": "unsloth/OpenHermes-2.5-Mistral-7B",
   "architectures": [
-    "LlamaForCausalLM"
+    "MistralForCausalLM"
   ],
-  "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 0,
-  "eos_token_id": 2,
-  "head_dim": 4,
+  "bos_token_id": 1,
+  "eos_token_id": 32000,
+  "head_dim": 128,
   "hidden_act": "silu",
-  "hidden_size": 16,
+  "hidden_size": 4096,
   "initializer_range": 0.02,
-  "intermediate_size": 64,
-  "max_position_embeddings": 2048,
-  "mlp_bias": false,
-  "model_type": "llama",
-  "num_attention_heads": 4,
-  "num_hidden_layers": 2,
-  "num_key_value_heads": 4,
-  "pad_token_id": -1,
-  "pretraining_tp": 1,
-  "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.48.1",
   "use_cache": false,
-  "vocab_size": 32000
+  "vocab_size": 32002
 }

last-checkpoint/generation_config.json

@@ -1,8 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
+  "bos_token_id": 1,
   "do_sample": true,
-  "eos_token_id": 1,
-  "pad_token_id": 2,
+  "eos_token_id": 32000,
   "transformers_version": "4.48.1"
 }

last-checkpoint/model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7868cab33fad23d3511b8ea224cf1dbea7722902a8867937737b71ccacc2059
+oid sha256:410afacf2f348a926671e6ccc3caa4629e74e907f868e42ecb93d6f5d05ee853
 size 4943178720

last-checkpoint/model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:378ce005606bc9c097868eef32fada737076ab88cbbb362c2ba481fb98b05de1
+oid sha256:f52bef3c6ffa1c00376a6dfa73ddef315bc86f0dd6c2057a47be4e61e1fb7c3d
 size 4999819336

last-checkpoint/model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15ff3e698d027f621311484ef4aedcc8082231840d43b6ce7616843621290ca6
+oid sha256:0278078bf87ec279f8a814ac678eb563b3b6410abe702bdf7049de5062e7cd9e
 size 4540532728

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c54a820f98fa07f24b707a5b13ad41e602ccc916761791610cd651856882df16
-size 2162798
+oid sha256:b12933aa23a0c560e75cf26a59069e1e7ada485e87376d133ef8818dfabd9bd9
+size 14710155092

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4651e32e118f1ea1a8e26dfbbe64298593e12e6a71bcd36cb77f04f86d3f86d
+oid sha256:7f20d155a14d34cd1bb6d04e5de90f3224906e1758821edd752a8f1a9085a2db
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f0c1eba909fbb51daca773a25c075f182b4096aff21c9b4ff19dbada2080ac99
+oid sha256:ad18e870a176ca75a54f6620f83de92bdfd5a91302744d90bff8e5feae2fe0c5
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28adb9f06e220aefdc723ea4380a84d42b8bfb87cc53ce65859d55ce1876f51c
+oid sha256:9009a52be47b75834407dc5e146ed5360e6f23a35bff27bab34ef6fb47df1661
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:423c49ed521f6986d20d8b29112b383f4b0f3f2e228084ef82c2ad7dcd5d1de8
+oid sha256:7fb494a08fd57e7c6f63f06826c872164986e81b271996be0496671f713bdcc3
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a60c7d771c1fd156acee762fba03c724cb41829a3f71df370ecd1d20b134982
+oid sha256:9cd11d413bc67bf01de9a1a006e9e7655be307353028b25f5b3c299e5b6b7a44
 size 1064

last-checkpoint/special_tokens_map.json

@@ -7,7 +7,7 @@
     "single_word": false
   },
   "eos_token": {
-    "content": "</s>",
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

last-checkpoint/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d606f46a8aa6f29f0a0abdec7c3ffddefc9f9bfe26919532d209a0a850e25029
-size 3619013
+oid sha256:04222cd76979c181cd3f72c3bf6982fe2a09d9f4b8f23d82902efde18f1d0668
+size 3506125

last-checkpoint/tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
-size 499723
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

last-checkpoint/tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
-  "add_prefix_space": null,
+  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -26,17 +26,38 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32000": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
+  "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
+  "eos_token": "<|im_end|>",
   "extra_special_tokens": {},
   "legacy": true,
-  "model_max_length": 2048,
+  "model_max_length": 32768,
   "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
+  "trust_remote_code": false,
   "unk_token": "<unk>",
   "use_default_system_prompt": true,
   "use_fast": true

last-checkpoint/trainer_state.json

@@ -1,96 +1,174 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 16.666666666666668,
+  "epoch": 1.9047619047619047,
   "eval_steps": 200,
-  "global_step": 100,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.16666666666666666,
-      "eval_loss": 10.376375198364258,
-      "eval_runtime": 2.3455,
-      "eval_samples_per_second": 639.941,
-      "eval_steps_per_second": 5.116,
+      "epoch": 0.009523809523809525,
+      "eval_loss": 1.873080849647522,
+      "eval_runtime": 27.7442,
+      "eval_samples_per_second": 54.101,
+      "eval_steps_per_second": 6.776,
       "step": 1
     },
     {
-      "epoch": 1.6666666666666665,
-      "grad_norm": 0.09375,
-      "learning_rate": 0.00019863613034027224,
-      "loss": 10.3756,
+      "epoch": 0.09523809523809523,
+      "grad_norm": 32.75,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 2.6922,
       "step": 10
     },
     {
-      "epoch": 3.3333333333333335,
-      "grad_norm": 0.12353515625,
-      "learning_rate": 0.0001879473751206489,
-      "loss": 10.3632,
+      "epoch": 0.19047619047619047,
+      "grad_norm": 628.0,
+      "learning_rate": 0.00013333333333333334,
+      "loss": 3.47,
       "step": 20
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 0.2236328125,
-      "learning_rate": 0.00016772815716257412,
-      "loss": 10.3433,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 141.0,
+      "learning_rate": 0.0002,
+      "loss": 9.2162,
       "step": 30
     },
     {
-      "epoch": 6.666666666666667,
-      "grad_norm": 0.3203125,
-      "learning_rate": 0.00014016954246529696,
-      "loss": 10.3073,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 30.625,
+      "learning_rate": 0.00019984815164333163,
+      "loss": 8.4222,
       "step": 40
     },
     {
-      "epoch": 8.333333333333334,
-      "grad_norm": 0.330078125,
-      "learning_rate": 0.00010825793454723325,
-      "loss": 10.2602,
+      "epoch": 0.47619047619047616,
+      "grad_norm": 44.75,
+      "learning_rate": 0.00019939306773179497,
+      "loss": 8.4529,
       "step": 50
     },
     {
-      "epoch": 10.0,
-      "grad_norm": 0.326171875,
-      "learning_rate": 7.54514512859201e-05,
-      "loss": 10.2203,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 29.375,
+      "learning_rate": 0.00019863613034027224,
+      "loss": 8.0506,
       "step": 60
     },
     {
-      "epoch": 11.666666666666666,
-      "grad_norm": 0.326171875,
-      "learning_rate": 4.530518418775733e-05,
-      "loss": 10.1945,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 25.375,
+      "learning_rate": 0.00019757963826274357,
+      "loss": 8.6481,
       "step": 70
     },
     {
-      "epoch": 13.333333333333334,
-      "grad_norm": 0.328125,
-      "learning_rate": 2.1085949060360654e-05,
-      "loss": 10.1812,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 26.75,
+      "learning_rate": 0.00019622680003092503,
+      "loss": 7.9906,
       "step": 80
     },
     {
-      "epoch": 15.0,
-      "grad_norm": 0.328125,
-      "learning_rate": 5.418275829936537e-06,
-      "loss": 10.1773,
+      "epoch": 0.8571428571428571,
+      "grad_norm": 33.25,
+      "learning_rate": 0.00019458172417006347,
+      "loss": 8.4843,
       "step": 90
     },
     {
-      "epoch": 16.666666666666668,
-      "grad_norm": 0.328125,
-      "learning_rate": 0.0,
-      "loss": 10.1767,
+      "epoch": 0.9523809523809523,
+      "grad_norm": 23.5,
+      "learning_rate": 0.00019264940672148018,
+      "loss": 7.4575,
       "step": 100
+    },
+    {
+      "epoch": 1.0476190476190477,
+      "grad_norm": 16.25,
+      "learning_rate": 0.00019043571606975777,
+      "loss": 7.4497,
+      "step": 110
+    },
+    {
+      "epoch": 1.1428571428571428,
+      "grad_norm": 19.75,
+      "learning_rate": 0.0001879473751206489,
+      "loss": 7.3829,
+      "step": 120
+    },
+    {
+      "epoch": 1.2380952380952381,
+      "grad_norm": 13.6875,
+      "learning_rate": 0.00018519194088383273,
+      "loss": 7.3748,
+      "step": 130
+    },
+    {
+      "epoch": 1.3333333333333333,
+      "grad_norm": 25.0,
+      "learning_rate": 0.0001821777815225245,
+      "loss": 7.3801,
+      "step": 140
+    },
+    {
+      "epoch": 1.4285714285714286,
+      "grad_norm": 13.0625,
+      "learning_rate": 0.00017891405093963938,
+      "loss": 7.2868,
+      "step": 150
+    },
+    {
+      "epoch": 1.5238095238095237,
+      "grad_norm": 52.0,
+      "learning_rate": 0.00017541066097768963,
+      "loss": 7.2058,
+      "step": 160
+    },
+    {
+      "epoch": 1.619047619047619,
+      "grad_norm": 98.5,
+      "learning_rate": 0.00017167825131684513,
+      "loss": 7.2541,
+      "step": 170
+    },
+    {
+      "epoch": 1.7142857142857144,
+      "grad_norm": 10.8125,
+      "learning_rate": 0.00016772815716257412,
+      "loss": 7.1795,
+      "step": 180
+    },
+    {
+      "epoch": 1.8095238095238095,
+      "grad_norm": 18.125,
+      "learning_rate": 0.00016357237482099684,
+      "loss": 7.0485,
+      "step": 190
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "grad_norm": 9.4375,
+      "learning_rate": 0.00015922352526649803,
+      "loss": 6.9094,
+      "step": 200
+    },
+    {
+      "epoch": 1.9047619047619047,
+      "eval_loss": 6.993889808654785,
+      "eval_runtime": 27.036,
+      "eval_samples_per_second": 55.519,
+      "eval_steps_per_second": 6.954,
+      "step": 200
     }
   ],
   "logging_steps": 10,
-  "max_steps": 100,
+  "max_steps": 600,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 17,
+  "num_train_epochs": 6,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -99,13 +177,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 81831709900800.0,
-  "train_batch_size": 32,
+  "total_flos": 1.3980142170800128e+17,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11cf48786efac37806223cc8882d4253d84ff8c3599dc92c9fda0e12bc8a651f
-size 6840
+oid sha256:ef3eb8b3b2a8acd873a0906e2cee7771ee216bb2230a5edc2d984d58604ec971
+size 6776