End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.08790596319595352,
-    "train_runtime": 130.1751,
+    "train_loss": 31.168473226716742,
+    "train_runtime": 124.1651,
     "train_samples": 160,
-    "train_samples_per_second": 2.458,
-    "train_steps_per_second": 0.154
+    "train_samples_per_second": 2.577,
+    "train_steps_per_second": 0.161
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a528644cfa75593fff6b0dd1ed62424763667da1a2166a1b63a3aa3d17022df5
+oid sha256:dea62cda083e75adf7071eedd2309b167e36ea0af1770a9a3b0a23d87e1fd412
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.08790596319595352,
-    "train_runtime": 130.1751,
+    "train_loss": 31.168473226716742,
+    "train_runtime": 124.1651,
     "train_samples": 160,
-    "train_samples_per_second": 2.458,
-    "train_steps_per_second": 0.154
+    "train_samples_per_second": 2.577,
+    "train_steps_per_second": 0.161
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 205.78125,
+      "completion_length": 189.46875,
       "epoch": 0.1,
-      "grad_norm": 327.27691650390625,
+      "grad_norm": 72.04657745361328,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": -0.0,
-      "reward": 3.911146618425846,
-      "reward_std": 1.487842702306807,
-      "rewards/concensus_correctness_reward_func": 1.129874998703599,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 3.999774469062686,
+      "reward_std": 2.164257466851268,
+      "rewards/concensus_correctness_reward_func": 1.5554375015199184,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.610677899006987,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7209931435063481,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 0.8112187534570694,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6295937476679683,
       "step": 2
     },
     {
-      "completion_length": 187.96875,
+      "completion_length": 132.5625,
       "epoch": 0.2,
-      "grad_norm": 358.25482177734375,
-      "kl": 6.240644539677305,
+      "grad_norm": 10451.45703125,
+      "kl": 253.42494644969702,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0062,
-      "reward": 5.06112876534462,
-      "reward_std": 1.1203965046443045,
-      "rewards/concensus_correctness_reward_func": 1.3846249803900719,
+      "loss": 0.2534,
+      "reward": 5.015479948371649,
+      "reward_std": 0.6629242373019224,
+      "rewards/concensus_correctness_reward_func": 1.5227500051259995,
       "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8568788375705481,
+      "rewards/question_recreation_reward_func": 0.8574799399357289,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.9446249939501286,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.8539999984204769,
       "step": 4
     },
     {
-      "completion_length": 191.25,
+      "completion_length": 143.75,
       "epoch": 0.3,
-      "grad_norm": 514.3082885742188,
-      "kl": 28.115042565390468,
+      "grad_norm": 97.95394897460938,
+      "kl": 96.7897967826575,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0281,
-      "reward": 5.191891148686409,
-      "reward_std": 1.147927716931008,
-      "rewards/concensus_correctness_reward_func": 1.2651249915361404,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0968,
+      "reward": 5.306805005297065,
+      "reward_std": 0.6057099380996078,
+      "rewards/concensus_correctness_reward_func": 1.5708125047385693,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9241099432110786,
+      "rewards/question_recreation_reward_func": 0.8909612875431776,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 1.0651562474668026,
+      "rewards/strict_format_reward_func": 0.109375,
+      "rewards/xmlcount_reward_func": 0.9856562428176403,
       "step": 6
     },
     {
-      "completion_length": 149.78125,
+      "completion_length": 163.25,
       "epoch": 0.4,
-      "grad_norm": 228.35906982421875,
-      "kl": 29.26655377820134,
+      "grad_norm": 118.51630401611328,
+      "kl": 10.452200602274388,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0293,
-      "reward": 5.6574289463460445,
-      "reward_std": 0.6735454360023141,
-      "rewards/concensus_correctness_reward_func": 1.7846249928697944,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0105,
+      "reward": 5.741433784365654,
+      "reward_std": 0.5551880679558963,
+      "rewards/concensus_correctness_reward_func": 1.730749998241663,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8681163862347603,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8832775540649891,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.8953125048428774,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.0024062488228083,
       "step": 8
     },
     {
-      "completion_length": 144.65625,
+      "completion_length": 133.96875,
       "epoch": 0.5,
-      "grad_norm": 1554.95703125,
-      "kl": 138.00590826105326,
+      "grad_norm": 311.2564392089844,
+      "kl": 6.50762258892064,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.138,
-      "reward": 6.434723109006882,
-      "reward_std": 0.6814799948479049,
-      "rewards/concensus_correctness_reward_func": 2.1148749962449074,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0065,
+      "reward": 5.870141640305519,
+      "reward_std": 1.113852635025978,
+      "rewards/concensus_correctness_reward_func": 1.815562479197979,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.9212230481207371,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8371417056769133,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.9923750087618828,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.0299374982714653,
       "step": 10
     },
     {
-      "completion_length": 143.96875,
+      "completion_length": 145.78125,
       "epoch": 0.6,
-      "grad_norm": 66.57030487060547,
-      "kl": 513.9867404703982,
+      "grad_norm": 77.84561157226562,
+      "kl": 310228.7304797147,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.514,
-      "reward": 5.189173512160778,
-      "reward_std": 0.2735713243018836,
-      "rewards/concensus_correctness_reward_func": 1.5454999953508377,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 310.2288,
+      "reward": 4.60096238553524,
+      "reward_std": 0.7561608459218405,
+      "rewards/concensus_correctness_reward_func": 1.2766249924898148,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8041734099388123,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.8684623129665852,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.96450000628829,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.9246250009164214,
       "step": 12
     },
     {
-      "completion_length": 143.15625,
+      "completion_length": 136.3125,
       "epoch": 0.7,
-      "grad_norm": 6868.37451171875,
-      "kl": 85.21862078900449,
+      "grad_norm": 33276.56640625,
+      "kl": 892.4592962227762,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0852,
-      "reward": 5.831615820527077,
-      "reward_std": 0.8292150638299063,
-      "rewards/concensus_correctness_reward_func": 1.7690624948590994,
+      "loss": 0.8925,
+      "reward": 5.898394346237183,
+      "reward_std": 0.5051253397541586,
+      "rewards/concensus_correctness_reward_func": 1.8743750005960464,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8815220082178712,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8720193216577172,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.0560312494635582,
+      "rewards/strict_format_reward_func": 0.109375,
+      "rewards/xmlcount_reward_func": 0.9801249913871288,
       "step": 14
     },
     {
-      "completion_length": 145.9375,
+      "completion_length": 150.3125,
       "epoch": 0.8,
-      "grad_norm": 68.38047790527344,
-      "kl": 1.4743171306326985,
+      "grad_norm": 237.27255249023438,
+      "kl": 170.10233331704512,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0015,
-      "reward": 5.784710422158241,
-      "reward_std": 0.5091803311370313,
-      "rewards/concensus_correctness_reward_func": 1.720812514424324,
+      "loss": 0.1701,
+      "reward": 5.7240906953811646,
+      "reward_std": 0.7569041545502841,
+      "rewards/concensus_correctness_reward_func": 1.7523125000298023,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8722417000681162,
+      "rewards/question_recreation_reward_func": 0.9259657934308052,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.05103125795722,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 0.9833124950528145,
       "step": 16
     },
     {
-      "completion_length": 135.375,
+      "completion_length": 146.5625,
       "epoch": 0.9,
-      "grad_norm": 91.15985870361328,
-      "kl": 6.782652285182849,
+      "grad_norm": 62.39609909057617,
+      "kl": 21.542335913982242,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0068,
-      "reward": 5.582313135266304,
-      "reward_std": 0.5564112643623957,
-      "rewards/concensus_correctness_reward_func": 1.644749991595745,
+      "loss": 0.0215,
+      "reward": 5.123600989580154,
+      "reward_std": 1.120502723613754,
+      "rewards/concensus_correctness_reward_func": 1.5977499894797802,
       "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8648755885660648,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8194135017693043,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.9789375066757202,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.8158124927431345,
       "step": 18
     },
     {
-      "completion_length": 129.25,
+      "completion_length": 149.75,
       "epoch": 1.0,
-      "grad_norm": 1769.4683837890625,
-      "kl": 69.95284202182665,
+      "grad_norm": 64.91241455078125,
+      "kl": 4.645937927765772,
       "learning_rate": 0.0,
-      "loss": 0.07,
-      "reward": 5.859360247850418,
-      "reward_std": 0.7594537734985352,
-      "rewards/concensus_correctness_reward_func": 1.8381249979138374,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0046,
+      "reward": 5.778949812054634,
+      "reward_std": 0.8444689979951363,
+      "rewards/concensus_correctness_reward_func": 1.751749999821186,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8620164804160595,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8938873894512653,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 1.0029687527567148,
+      "rewards/xmlcount_reward_func": 0.9770625047385693,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.08790596319595352,
-      "train_runtime": 130.1751,
-      "train_samples_per_second": 2.458,
-      "train_steps_per_second": 0.154
+      "train_loss": 31.168473226716742,
+      "train_runtime": 124.1651,
+      "train_samples_per_second": 2.577,
+      "train_steps_per_second": 0.161
     }
   ],
   "logging_steps": 2,