End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.8505885711638258e-05,
-    "train_runtime": 6580.358,
-    "train_samples": 6,
-    "train_samples_per_second": 0.006,
-    "train_steps_per_second": 0.002
+    "train_loss": 6.198883056640625e-07,
+    "train_runtime": 1226.5225,
+    "train_samples": 5,
+    "train_samples_per_second": 0.033,
+    "train_steps_per_second": 0.008
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf61dd9734743863a69b2b540d36bd9b51536bf59f1333e13c0e52dd6e45c6bd
+oid sha256:b17adb89ed44f71d4a01f77b6fd841a58ed9f3e4a0f1dc8880e91944f7b3b71d
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.8505885711638258e-05,
-    "train_runtime": 6580.358,
-    "train_samples": 6,
-    "train_samples_per_second": 0.006,
-    "train_steps_per_second": 0.002
+    "train_loss": 6.198883056640625e-07,
+    "train_runtime": 1226.5225,
+    "train_samples": 5,
+    "train_samples_per_second": 0.033,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.3333333333333335,
+  "epoch": 3.4,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -10,114 +10,114 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 17.25,
-      "epoch": 0.6666666666666666,
-      "grad_norm": 48.05388641357422,
+      "completion_length": 176.875,
+      "epoch": 0.8,
+      "grad_norm": 7.9261298179626465,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 0.6133153163827956,
-      "reward_std": 0.7537651844322681,
+      "loss": 0.0,
+      "reward": 0.06529645575210452,
+      "reward_std": 0.031127130147069693,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.0705653433687985,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.06529645575210452,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.042750000953674316,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 2
     },
     {
-      "completion_length": 45.25,
-      "epoch": 1.3333333333333333,
-      "grad_norm": 78.80774688720703,
-      "kl": 0.020999361149733886,
+      "completion_length": 175.83333333333334,
+      "epoch": 1.4,
+      "grad_norm": 7.617096424102783,
+      "kl": 0.00035043573977115255,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 0.09825393743813038,
-      "reward_std": 0.02789358670997899,
+      "reward": 0.19371196130911508,
+      "reward_std": 0.08724394316474597,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08175393752753735,
+      "rewards/question_recreation_reward_func": 0.19371196130911508,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.016499999910593033,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 4
     },
     {
-      "completion_length": 42.625,
+      "completion_length": 217.66666666666666,
       "epoch": 2.0,
-      "grad_norm": 0.004175778012722731,
-      "kl": 0.049324687279295176,
+      "grad_norm": 5.332751750946045,
+      "kl": 0.000615184020716697,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 0.9253544360399246,
-      "reward_std": 0.40504125878214836,
+      "reward": 0.02679351655145486,
+      "reward_std": 0.009128622089823088,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.05735448165796697,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.02679351655145486,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11800000071525574,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 6
     },
     {
-      "completion_length": 17.75,
-      "epoch": 2.6666666666666665,
-      "grad_norm": 0.003430261043831706,
-      "kl": 0.04985775472596288,
+      "completion_length": 189.75,
+      "epoch": 2.8,
+      "grad_norm": 6.812089920043945,
+      "kl": 0.0007357565191341564,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0,
-      "reward": 0.6951797232031822,
-      "reward_std": 0.020487656816840172,
+      "reward": 0.06324641685932875,
+      "reward_std": 0.013577945763245225,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.05430475785396993,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.06324641685932875,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14087500050663948,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 8
     },
     {
-      "completion_length": 20.0,
-      "epoch": 3.3333333333333335,
-      "grad_norm": 55.28184127807617,
-      "kl": 0.02525496296584606,
+      "completion_length": 148.83333333333334,
+      "epoch": 3.4,
+      "grad_norm": 19.7174129486084,
+      "kl": 0.0018994022199573617,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 0.39077210798859596,
-      "reward_std": 0.3560393115039915,
+      "reward": 0.0978486401339372,
+      "reward_std": 0.02896895833934347,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.10927211865782738,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.0978486401339372,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03150000050663948,
+      "rewards/xmlcount_reward_func": 0.0,
       "step": 10
     },
     {
-      "epoch": 3.3333333333333335,
+      "epoch": 3.4,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 2.8505885711638258e-05,
-      "train_runtime": 6580.358,
-      "train_samples_per_second": 0.006,
-      "train_steps_per_second": 0.002
+      "train_loss": 6.198883056640625e-07,
+      "train_runtime": 1226.5225,
+      "train_samples_per_second": 0.033,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 5,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24a80b9404f4fffaedc1b8ca509e6ed1757684c5d9bfe58301d5f689c832b365
+oid sha256:6a5811d4022ee43bf43a3d7e557e467d606190ea87357fe97015c06855109fba
 size 5880