End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.4807249903678895e-06,
-    "train_runtime": 271.9171,
+    "train_loss": 3.5451927260510274e-06,
+    "train_runtime": 236.747,
     "train_samples": 28,
-    "train_samples_per_second": 1.177,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 1.352,
+    "train_steps_per_second": 0.084
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9a73c35df4b109be4d3130b17b9902433ed7ef90dce3ce84ccaf1fb72f3455d
+oid sha256:d129835df8e16a2c93fea8490570318f8d0fc2999cd62f6af2285d80c9eb18b0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.4807249903678895e-06,
-    "train_runtime": 271.9171,
+    "train_loss": 3.5451927260510274e-06,
+    "train_runtime": 236.747,
     "train_samples": 28,
-    "train_samples_per_second": 1.177,
-    "train_steps_per_second": 0.074
+    "train_samples_per_second": 1.352,
+    "train_steps_per_second": 0.084
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 443.8125,
+      "completion_length": 305.0625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.0108537673950195,
+      "grad_norm": 5.660951614379883,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.9394766306504607,
-      "reward_std": 0.8769232220947742,
-      "rewards/concensus_correctness_reward_func": 0.05974999815225601,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": -0.0,
+      "reward": 2.460386883933097,
+      "reward_std": 1.7337693277513608,
+      "rewards/concensus_correctness_reward_func": 0.8088750019669533,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5109141170978546,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6287306472659111,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24381250934675336,
+      "rewards/xmlcount_reward_func": 0.7102812547236681,
       "step": 2
     },
     {
-      "completion_length": 395.875,
+      "completion_length": 221.66666666666666,
       "epoch": 1.0,
-      "grad_norm": 2.882847309112549,
-      "kl": 0.0020494584023253992,
+      "grad_norm": 3.8874945640563965,
+      "kl": 0.001395704142244843,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.441322589914004,
-      "reward_std": 1.6230067620053887,
-      "rewards/concensus_correctness_reward_func": 0.22516666849454245,
+      "reward": 2.2181622932354608,
+      "reward_std": 1.2326010285566251,
+      "rewards/concensus_correctness_reward_func": 0.25625000645716983,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.5356976023564736,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5083289686590433,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34712500870227814,
+      "rewards/xmlcount_reward_func": 0.9535833448171616,
       "step": 4
     },
     {
-      "completion_length": 404.46875,
+      "completion_length": 254.3125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 7.562240123748779,
-      "kl": 0.00213110827098717,
+      "grad_norm": 5.26646089553833,
+      "kl": 0.002352659485040931,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.0263560209423304,
-      "reward_std": 0.7502539354027249,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.6883456259965897,
+      "reward_std": 1.0170315988361835,
+      "rewards/concensus_correctness_reward_func": 0.20918750576674938,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5181372761726379,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5529393274337053,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5082187606021762,
+      "rewards/xmlcount_reward_func": 0.5512187629938126,
       "step": 6
     },
     {
-      "completion_length": 327.2083333333333,
+      "completion_length": 347.5833333333333,
       "epoch": 2.0,
-      "grad_norm": 4.4285454750061035,
-      "kl": 0.0033311487350147218,
+      "grad_norm": 3.270358085632324,
+      "kl": 0.004270745276395853,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.0281017074982324,
-      "reward_std": 1.0761653492227197,
-      "rewards/concensus_correctness_reward_func": 0.2510000020265579,
+      "reward": 2.2331515848636627,
+      "reward_std": 0.9642113272566348,
+      "rewards/concensus_correctness_reward_func": 0.32608333230018616,
       "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7451016729076704,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.6685264905293783,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6986666594942411,
+      "rewards/xmlcount_reward_func": 0.7385416726271311,
       "step": 8
     },
     {
-      "completion_length": 327.6875,
+      "completion_length": 288.1875,
       "epoch": 2.571428571428571,
-      "grad_norm": 6.3724188804626465,
-      "kl": 0.003951843835238833,
+      "grad_norm": 5.7600579261779785,
+      "kl": 0.005326771060936153,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.3906152062118053,
-      "reward_std": 0.765568673145026,
-      "rewards/concensus_correctness_reward_func": 0.05974999815225601,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.6227679140865803,
+      "reward_std": 0.7090006899961736,
+      "rewards/concensus_correctness_reward_func": 0.12793750315904617,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5555214579217136,
+      "rewards/question_recreation_reward_func": 0.5139241893775761,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7128437627106905,
+      "rewards/xmlcount_reward_func": 0.793406255543232,
       "step": 10
     },
     {
-      "completion_length": 276.6666666666667,
+      "completion_length": 282.5,
       "epoch": 3.0,
-      "grad_norm": 3.9053845405578613,
-      "kl": 0.005441825041392197,
+      "grad_norm": 3.537179708480835,
+      "kl": 0.005162405354591708,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.5927054782708485,
-      "reward_std": 0.9333043030152718,
-      "rewards/concensus_correctness_reward_func": 0.09050000210603078,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 3.3730034033457437,
+      "reward_std": 1.6889308356524755,
+      "rewards/concensus_correctness_reward_func": 1.1676666637261708,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5969137859841188,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.7677534197767576,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7386249999205271,
+      "rewards/xmlcount_reward_func": 0.7709166618684927,
       "step": 12
     },
     {
-      "completion_length": 323.71875,
+      "completion_length": 286.3125,
       "epoch": 3.571428571428571,
-      "grad_norm": 13.971353530883789,
-      "kl": 0.005141458539583255,
+      "grad_norm": 5.225093364715576,
+      "kl": 0.004380538783152588,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.2191743962466717,
-      "reward_std": 1.4723687917576171,
-      "rewards/concensus_correctness_reward_func": 0.4583125039935112,
+      "reward": 2.611244770232588,
+      "reward_std": 1.1959627732721856,
+      "rewards/concensus_correctness_reward_func": 0.5465000029653311,
       "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5449556156527251,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6421822616248392,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7159062540158629,
+      "rewards/xmlcount_reward_func": 0.7350625019753352,
       "step": 14
     },
     {
-      "completion_length": 337.0416666666667,
+      "completion_length": 263.7916666666667,
       "epoch": 4.0,
-      "grad_norm": 4.71185827255249,
-      "kl": 0.00529346401647975,
+      "grad_norm": 3.689728260040283,
+      "kl": 0.005115592464183767,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0,
-      "reward": 1.49052993953228,
-      "reward_std": 0.5134295720296601,
-      "rewards/concensus_correctness_reward_func": 0.04050000011920929,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "loss": -0.0,
+      "reward": 1.660034865140915,
+      "reward_std": 0.5135989048867486,
+      "rewards/concensus_correctness_reward_func": 0.16166667143503824,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5593632707993189,
+      "rewards/question_recreation_reward_func": 0.5029515164593855,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8073333352804184,
+      "rewards/xmlcount_reward_func": 0.8287500118215879,
       "step": 16
     },
     {
-      "completion_length": 340.125,
+      "completion_length": 287.0625,
       "epoch": 4.571428571428571,
-      "grad_norm": 6.314624309539795,
-      "kl": 0.0055000996217131615,
+      "grad_norm": 7.379291534423828,
+      "kl": 0.007929646439151838,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.9279198572039604,
-      "reward_std": 0.8200628219055943,
-      "rewards/concensus_correctness_reward_func": 0.18043750151991844,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.4730680622160435,
+      "reward_std": 0.974102801643312,
+      "rewards/concensus_correctness_reward_func": 0.37162500619888306,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7055448405444622,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.640130530227907,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7919375095516443,
+      "rewards/xmlcount_reward_func": 0.7738125212490559,
       "step": 18
     },
     {
-      "completion_length": 290.6666666666667,
+      "completion_length": 379.375,
       "epoch": 5.0,
-      "grad_norm": 3.5206193923950195,
-      "kl": 0.006180711983082195,
+      "grad_norm": 3.7960731983184814,
+      "kl": 0.004161193520606806,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.6220816473166149,
-      "reward_std": 0.6310094334185123,
-      "rewards/concensus_correctness_reward_func": 0.1162499984105428,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.4008877476056416,
+      "reward_std": 1.2709657562275727,
+      "rewards/concensus_correctness_reward_func": 0.09125000238418579,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6283732987940311,
+      "rewards/question_recreation_reward_func": 0.49847108746568364,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7941250105698904,
+      "rewards/xmlcount_reward_func": 0.6445000022649765,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.4807249903678895e-06,
-      "train_runtime": 271.9171,
-      "train_samples_per_second": 1.177,
-      "train_steps_per_second": 0.074
+      "train_loss": 3.5451927260510274e-06,
+      "train_runtime": 236.747,
+      "train_samples_per_second": 1.352,
+      "train_steps_per_second": 0.084
     }
   ],
   "logging_steps": 2,