End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.000559220383183856,
-    "train_runtime": 298.5282,
+    "train_loss": 1.2790784239768983e-06,
+    "train_runtime": 291.7059,
     "train_samples": 28,
-    "train_samples_per_second": 1.072,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 1.097,
+    "train_steps_per_second": 0.069
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:252331b062976ee5fb162128f35adb49e50d322e72007dce42b860cd560acb0a
+oid sha256:82c63423942fcaeb942c0ce36a4449a4784b461993310bf83a55c652504123a2
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.000559220383183856,
-    "train_runtime": 298.5282,
+    "train_loss": 1.2790784239768983e-06,
+    "train_runtime": 291.7059,
     "train_samples": 28,
-    "train_samples_per_second": 1.072,
-    "train_steps_per_second": 0.067
+    "train_samples_per_second": 1.097,
+    "train_steps_per_second": 0.069
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 354.5625,
+      "completion_length": 326.59375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.754081726074219,
+      "grad_norm": 4.934439182281494,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 1.6411238303408027,
-      "reward_std": 1.7271968586137518,
-      "rewards/concensus_correctness_reward_func": 0.7330624982714653,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": 0.0,
+      "reward": 0.9843985685729422,
+      "reward_std": 0.7370969671756029,
+      "rewards/concensus_correctness_reward_func": -0.02437499910593033,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4333425478544086,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.39177357318112627,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16221874672919512,
+      "rewards/xmlcount_reward_func": 0.3670000056736171,
       "step": 2
     },
     {
-      "completion_length": 394.1666666666667,
+      "completion_length": 371.0,
       "epoch": 1.0,
-      "grad_norm": 103.15747833251953,
-      "kl": 0.0011694440690916963,
+      "grad_norm": 3.6815781593322754,
+      "kl": 0.0011949266287653397,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.6417979632193844,
-      "reward_std": 2.4711355743929744,
-      "rewards/concensus_correctness_reward_func": 0.8632499997814497,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.5001778639853,
+      "reward_std": 0.7827915487190088,
+      "rewards/concensus_correctness_reward_func": 0.03458333263794581,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.468506284058094,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4009695214529832,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06004166603088379,
+      "rewards/xmlcount_reward_func": 0.5646250148614248,
       "step": 4
     },
     {
-      "completion_length": 326.84375,
+      "completion_length": 359.96875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 8.310850143432617,
-      "kl": 0.0020925263124809135,
+      "grad_norm": 4.556028842926025,
+      "kl": 0.0010278360459778924,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.914471060037613,
-      "reward_std": 1.8766977214254439,
-      "rewards/concensus_correctness_reward_func": 0.7248125001788139,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.1807272167643532,
+      "reward_std": 0.653216159567819,
+      "rewards/concensus_correctness_reward_func": 0.06068750098347664,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5054710638942197,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.47382097283843905,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30918750260025263,
+      "rewards/xmlcount_reward_func": 0.45871875062584877,
       "step": 6
     },
     {
-      "completion_length": 380.4166666666667,
+      "completion_length": 439.2916666666667,
       "epoch": 2.0,
-      "grad_norm": 3.1144468784332275,
-      "kl": 0.0017934188605674233,
+      "grad_norm": 2.2755231857299805,
+      "kl": 0.0013096223750229303,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.0309538878500462,
-      "reward_std": 0.5571006955578923,
+      "reward": 0.7530886133511862,
+      "reward_std": 0.9865690590813756,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.45816220218936604,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.37442193475241464,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.40612500347197056,
+      "rewards/xmlcount_reward_func": 0.295333335796992,
       "step": 8
     },
     {
-      "completion_length": 435.21875,
+      "completion_length": 370.40625,
       "epoch": 2.571428571428571,
-      "grad_norm": 4.002050876617432,
-      "kl": 5.514499381351925,
+      "grad_norm": 6.62834358215332,
+      "kl": 0.0012265997065696865,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0055,
-      "reward": 1.3926817565225065,
-      "reward_std": 1.464526791125536,
-      "rewards/concensus_correctness_reward_func": 0.2616249993443489,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0,
+      "reward": 0.8175993217155337,
+      "reward_std": 1.0472558625042439,
+      "rewards/concensus_correctness_reward_func": 0.05756250023841858,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.3692754968069494,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4674430672894232,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1367812603712082,
+      "rewards/xmlcount_reward_func": 0.16759375017136335,
       "step": 10
     },
     {
-      "completion_length": 434.7916666666667,
+      "completion_length": 356.4583333333333,
       "epoch": 3.0,
-      "grad_norm": 3.287328004837036,
-      "kl": 0.02375750773353502,
+      "grad_norm": 3.410081148147583,
+      "kl": 0.001667177789689352,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.7725401210288206,
-      "reward_std": 0.4789987488960226,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.0313266267379124,
+      "reward_std": 0.6612533178801338,
+      "rewards/concensus_correctness_reward_func": 0.0790833334128062,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.469706774999698,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.572909953383108,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2195000077287356,
+      "rewards/xmlcount_reward_func": 0.37933334335684776,
       "step": 12
     },
     {
-      "completion_length": 441.1875,
+      "completion_length": 401.03125,
       "epoch": 3.571428571428571,
-      "grad_norm": 4.873816013336182,
-      "kl": 0.0029075223501422442,
+      "grad_norm": 6.091869354248047,
+      "kl": 0.0018705126858549193,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.9816373744979501,
-      "reward_std": 0.9603902782546356,
-      "rewards/concensus_correctness_reward_func": 0.10537499934434891,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.7606599755818024,
+      "reward_std": 0.9554330813698471,
+      "rewards/concensus_correctness_reward_func": 0.06806249916553497,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.39413736655842513,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5348162406007759,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2946250010281801,
+      "rewards/xmlcount_reward_func": 0.09528125822544098,
       "step": 14
     },
     {
-      "completion_length": 280.5416666666667,
+      "completion_length": 407.9166666666667,
       "epoch": 4.0,
-      "grad_norm": 2.644231081008911,
-      "kl": 0.027301095115641754,
+      "grad_norm": 2.649594306945801,
+      "kl": 0.001604035545218115,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.207851281234374,
-      "reward_std": 2.0342274831297495,
-      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
+      "reward": 0.6916571507851282,
+      "reward_std": 1.01990426153255,
+      "rewards/concensus_correctness_reward_func": 0.024666666984558105,
       "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.4336429964751005,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4062821501865983,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3575416690049072,
+      "rewards/xmlcount_reward_func": 0.17737499624490738,
       "step": 16
     },
     {
-      "completion_length": 525.1875,
+      "completion_length": 412.125,
       "epoch": 4.571428571428571,
-      "grad_norm": 4.063370704650879,
-      "kl": 0.007224528286315035,
+      "grad_norm": 119.07859802246094,
+      "kl": 0.0031047585580381565,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.3370077246800065,
-      "reward_std": 2.354903077452036,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "reward": 0.8601471782312728,
+      "reward_std": 0.6142369294539094,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.43528897408396006,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.464647185173817,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.33921874687075615,
+      "rewards/xmlcount_reward_func": 0.2705000061541796,
       "step": 18
     },
     {
-      "completion_length": 374.1666666666667,
+      "completion_length": 346.3333333333333,
       "epoch": 5.0,
-      "grad_norm": 3.4866976737976074,
-      "kl": 0.033373012770122536,
+      "grad_norm": 4.018592834472656,
+      "kl": 0.0016965343481084953,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.4424385850628216,
-      "reward_std": 0.6604198403656483,
-      "rewards/concensus_correctness_reward_func": 0.07608333230018616,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.0170780109862487,
+      "reward_std": 0.9925470755745968,
+      "rewards/concensus_correctness_reward_func": -0.0019999990860621133,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5623552203178406,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.32816133989642066,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3040000107139349,
+      "rewards/xmlcount_reward_func": 0.35758334149916965,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.000559220383183856,
-      "train_runtime": 298.5282,
-      "train_samples_per_second": 1.072,
-      "train_steps_per_second": 0.067
+      "train_loss": 1.2790784239768983e-06,
+      "train_runtime": 291.7059,
+      "train_samples_per_second": 1.097,
+      "train_steps_per_second": 0.069
     }
   ],
   "logging_steps": 2,