End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 2857.3366,
-    "train_samples": 5,
-    "train_samples_per_second": 0.014,
-    "train_steps_per_second": 0.003
+    "train_loss": 2.8505885711638258e-05,
+    "train_runtime": 6580.358,
+    "train_samples": 6,
+    "train_samples_per_second": 0.006,
+    "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03fa5b779ed82abaf9249cc992e5c3699de6a2f1e27f02f3d9b927ff68515e2d
+oid sha256:cf61dd9734743863a69b2b540d36bd9b51536bf59f1333e13c0e52dd6e45c6bd
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 2857.3366,
-    "train_samples": 5,
-    "train_samples_per_second": 0.014,
-    "train_steps_per_second": 0.003
+    "train_loss": 2.8505885711638258e-05,
+    "train_runtime": 6580.358,
+    "train_samples": 6,
+    "train_samples_per_second": 0.006,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.4,
+  "epoch": 3.3333333333333335,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -10,114 +10,114 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 4.0,
-      "epoch": 0.8,
-      "grad_norm": 0.0,
+      "completion_length": 17.25,
+      "epoch": 0.6666666666666666,
+      "grad_norm": 48.05388641357422,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.448614865541458,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.917499989271164,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": -0.0,
+      "reward": 0.6133153163827956,
+      "reward_std": 0.7537651844322681,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.031114899553358555,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.0705653433687985,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.042750000953674316,
       "step": 2
     },
     {
-      "completion_length": 4.0,
-      "epoch": 1.4,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completion_length": 45.25,
+      "epoch": 1.3333333333333333,
+      "grad_norm": 78.80774688720703,
+      "kl": 0.020999361149733886,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 4.009881416956584,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.9186666409174602,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 0.09825393743813038,
+      "reward_std": 0.02789358670997899,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09121479839086533,
+      "rewards/question_recreation_reward_func": 0.08175393752753735,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.016499999910593033,
       "step": 4
     },
     {
-      "completion_length": 4.0,
+      "completion_length": 42.625,
       "epoch": 2.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 0.004175778012722731,
+      "kl": 0.049324687279295176,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 1.9338454802831013,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.9179999828338623,
+      "reward": 0.9253544360399246,
+      "reward_std": 0.40504125878214836,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.015845506141583126,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.05735448165796697,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11800000071525574,
       "step": 6
     },
     {
-      "completion_length": 4.0,
-      "epoch": 2.8,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completion_length": 17.75,
+      "epoch": 2.6666666666666665,
+      "grad_norm": 0.003430261043831706,
+      "kl": 0.04985775472596288,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0,
-      "reward": 2.454974263906479,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.9184999763965607,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.6951797232031822,
+      "reward_std": 0.020487656816840172,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.036474294029176235,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.05430475785396993,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14087500050663948,
       "step": 8
     },
     {
-      "completion_length": 4.0,
-      "epoch": 3.4,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completion_length": 20.0,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 55.28184127807617,
+      "kl": 0.02525496296584606,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 2.618675430615743,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func": 1.9166666666666667,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 0.39077210798859596,
+      "reward_std": 0.3560393115039915,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.035342139502366386,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.10927211865782738,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.03150000050663948,
       "step": 10
     },
     {
-      "epoch": 3.4,
+      "epoch": 3.3333333333333335,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 2857.3366,
-      "train_samples_per_second": 0.014,
-      "train_steps_per_second": 0.003
+      "train_loss": 2.8505885711638258e-05,
+      "train_runtime": 6580.358,
+      "train_samples_per_second": 0.006,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 4,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {