End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.7833811044693e-06,
-    "train_runtime": 37094.8126,
+    "train_loss": 6.447264695452759e-06,
+    "train_runtime": 10252.1281,
     "train_samples": 17,
-    "train_samples_per_second": 0.004,
-    "train_steps_per_second": 0.0
+    "train_samples_per_second": 0.016,
+    "train_steps_per_second": 0.001
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6014feacd4b8e91c47e7a8a14b6aea876da8cf3ec09af3db813a37d3251fdc88
+oid sha256:a1d1bfde27d5e6be495333e581102a6579a06b7e6a052a1a55513f1ec93bcc2f
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.7833811044693e-06,
-    "train_runtime": 37094.8126,
+    "train_loss": 6.447264695452759e-06,
+    "train_runtime": 10252.1281,
     "train_samples": 17,
-    "train_samples_per_second": 0.004,
-    "train_steps_per_second": 0.0
+    "train_samples_per_second": 0.016,
+    "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -10,108 +10,108 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 179.03125,
+      "completion_length": 129.0625,
       "epoch": 0.9411764705882353,
-      "grad_norm": 4.956028461456299,
+      "grad_norm": 5.772158622741699,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.5409406707622111,
-      "reward_std": 0.19756870809942484,
+      "reward": 0.591631580144167,
+      "reward_std": 0.2950996853032848,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.49097191635519266,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.500725319609046,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0499687489354983,
+      "rewards/xmlcount_reward_func": 0.02840625075623393,
       "step": 2
     },
     {
-      "completion_length": 185.33333333333334,
+      "completion_length": 129.88888888888889,
       "epoch": 1.4705882352941178,
-      "grad_norm": 5.115660667419434,
-      "kl": 0.006464091026120716,
+      "grad_norm": 4.538054943084717,
+      "kl": 0.003908961649156279,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 0.27550574412776363,
-      "reward_std": 0.26185208906988716,
+      "reward": 0.5140680852863524,
+      "reward_std": 0.10897246843928264,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.38950574004815686,
+      "rewards/question_recreation_reward_func": 0.49312362840606105,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.11399999385078748,
+      "rewards/xmlcount_reward_func": 0.020944445704420406,
       "step": 4
     },
     {
-      "completion_length": 131.33333333333334,
+      "completion_length": 138.77777777777777,
       "epoch": 2.0,
-      "grad_norm": 35.98667907714844,
-      "kl": 0.024488080085979566,
+      "grad_norm": 5.477197647094727,
+      "kl": 0.008041509905726545,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 0.3252610233094957,
-      "reward_std": 0.3340902552008629,
+      "reward": 0.7325452516476313,
+      "reward_std": 0.1888044167103039,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3654276885920101,
+      "rewards/final_correctness_reward_func": 0.2222222222222222,
+      "rewards/question_recreation_reward_func": 0.5095452732510037,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04016666528251436,
+      "rewards/xmlcount_reward_func": 0.0007777793022493521,
       "step": 6
     },
     {
-      "completion_length": 133.5,
+      "completion_length": 154.5,
       "epoch": 2.9411764705882355,
-      "grad_norm": 41.26161193847656,
-      "kl": 0.010965528272208758,
+      "grad_norm": 4.317551136016846,
+      "kl": 0.016037766326917335,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0,
-      "reward": 0.3778454198036343,
-      "reward_std": 0.31115904528996907,
+      "reward": 0.5921503761783242,
+      "reward_std": 0.3880798064637929,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3857204255182296,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5306503740139306,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.007874999195337296,
+      "rewards/xmlcount_reward_func": -0.0635000029578805,
       "step": 8
     },
     {
-      "completion_length": 139.5,
+      "completion_length": 145.83333333333334,
       "epoch": 3.4705882352941178,
-      "grad_norm": 6.064628601074219,
-      "kl": 0.009947262238711119,
+      "grad_norm": 5.180707931518555,
+      "kl": 0.016869528788245387,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 0.46963657945808435,
-      "reward_std": 0.19559346791356802,
+      "reward": 0.6629479693041908,
+      "reward_std": 0.28609387824932736,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4411365854450398,
+      "rewards/question_recreation_reward_func": 0.567670188844204,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.028499999368149374,
+      "rewards/xmlcount_reward_func": 0.09527777466509077,
       "step": 10
     },
     {
       "epoch": 3.4705882352941178,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 6.7833811044693e-06,
-      "train_runtime": 37094.8126,
-      "train_samples_per_second": 0.004,
-      "train_steps_per_second": 0.0
+      "train_loss": 6.447264695452759e-06,
+      "train_runtime": 10252.1281,
+      "train_samples_per_second": 0.016,
+      "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 2,