End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7698.1475028006225,
-    "train_runtime": 1123.2771,
+    "train_loss": 0.9904466477659298,
+    "train_runtime": 713.0205,
     "train_samples": 28,
-    "train_samples_per_second": 0.285,
-    "train_steps_per_second": 0.018
+    "train_samples_per_second": 0.449,
+    "train_steps_per_second": 0.028
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b53af726974c1bebe56e88984edf47c787039630fba808881558305c68d2415b
+oid sha256:bdf5ef409ec024790f9ffe4e6f6c1eeae37538c094fdb2202dc7a648cee4ad43
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7698.1475028006225,
-    "train_runtime": 1123.2771,
+    "train_loss": 0.9904466477659298,
+    "train_runtime": 713.0205,
     "train_samples": 28,
-    "train_samples_per_second": 0.285,
-    "train_steps_per_second": 0.018
+    "train_samples_per_second": 0.449,
+    "train_steps_per_second": 0.028
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 240.125,
+      "completion_length": 247.21875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 25.637950897216797,
+      "grad_norm": 6.694652557373047,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 2.974940799176693,
-      "reward_std": 0.786946261767298,
-      "rewards/concensus_correctness_reward_func": 0.585562489926815,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 3.429838713258505,
+      "reward_std": 0.7307246705458965,
+      "rewards/concensus_correctness_reward_func": 0.713249996304512,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7403783169575036,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6806199923157692,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 0.7583750085905194,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 0.7859687637537718,
       "step": 2
     },
     {
-      "completion_length": 149.16666666666666,
+      "completion_length": 125.83333333333333,
       "epoch": 1.0,
-      "grad_norm": 0.6037622690200806,
-      "kl": 0.09583777639394005,
+      "grad_norm": 16.098657608032227,
+      "kl": 0.10447756666690111,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0001,
-      "reward": 6.496234893798828,
-      "reward_std": 0.06761332352956136,
-      "rewards/concensus_correctness_reward_func": 1.857999970515569,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 5.87607725461324,
+      "reward_std": 0.4451955556869507,
+      "rewards/concensus_correctness_reward_func": 1.698999971151352,
+      "rewards/consensus_reward_func": 1.5833333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9611515551805496,
+      "rewards/question_recreation_reward_func": 0.964202269911766,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4375,
-      "rewards/xmlcount_reward_func": 1.2395833333333333,
+      "rewards/strict_format_reward_func": 0.3958333333333333,
+      "rewards/xmlcount_reward_func": 1.2337083319822948,
       "step": 4
     },
     {
-      "completion_length": 132.0,
+      "completion_length": 135.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 3106784768.0,
-      "kl": 76366540.49848042,
+      "grad_norm": 256478.375,
+      "kl": 8980.802092778962,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 76366.5391,
-      "reward": 6.02274464070797,
-      "reward_std": 0.5367404207354411,
-      "rewards/concensus_correctness_reward_func": 1.7007499895989895,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 8.9808,
+      "reward": 5.556592166423798,
+      "reward_std": 0.5139178396202624,
+      "rewards/concensus_correctness_reward_func": 1.4853749834001064,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9430884085595608,
+      "rewards/question_recreation_reward_func": 0.8682797141373158,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.23828125,
+      "rewards/xmlcount_reward_func": 1.2498124986886978,
       "step": 6
     },
     {
-      "completion_length": 154.33333333333334,
+      "completion_length": 145.66666666666666,
       "epoch": 2.0,
-      "grad_norm": 10.594367980957031,
-      "kl": 0.6017608487357696,
+      "grad_norm": 15.734909057617188,
+      "kl": 12.738230961724184,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0005,
-      "reward": 5.644766867160797,
-      "reward_std": 0.29860954036121257,
-      "rewards/concensus_correctness_reward_func": 1.4102499683698018,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0096,
+      "reward": 5.932813902695973,
+      "reward_std": 0.21585626168719804,
+      "rewards/concensus_correctness_reward_func": 1.4209999740123749,
+      "rewards/consensus_reward_func": 1.8333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.942850261926651,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.963147242863973,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.2083333333333333,
+      "rewards/strict_format_reward_func": 0.3958333333333333,
+      "rewards/xmlcount_reward_func": 1.2361666758855183,
       "step": 8
     },
     {
-      "completion_length": 161.3125,
+      "completion_length": 151.0,
       "epoch": 2.571428571428571,
-      "grad_norm": 14.723726272583008,
-      "kl": 0.267008597496897,
+      "grad_norm": 27.214447021484375,
+      "kl": 4.368775271810591,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0003,
-      "reward": 5.799619108438492,
-      "reward_std": 0.32607479006401263,
-      "rewards/concensus_correctness_reward_func": 1.5179999768733978,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0044,
+      "reward": 5.704393148422241,
+      "reward_std": 0.6130428695760202,
+      "rewards/concensus_correctness_reward_func": 1.5178749822080135,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.95302539691329,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.8489869562909007,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.453125,
-      "rewards/xmlcount_reward_func": 1.1879687495529652,
+      "rewards/strict_format_reward_func": 0.421875,
+      "rewards/xmlcount_reward_func": 1.228156253695488,
       "step": 10
     },
     {
-      "completion_length": 125.95833333333333,
+      "completion_length": 126.29166666666667,
       "epoch": 3.0,
-      "grad_norm": 8.453343391418457,
-      "kl": 811236.5537435077,
+      "grad_norm": 20.617341995239258,
+      "kl": 0.49174783130486804,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 608.4273,
-      "reward": 6.105852663516998,
-      "reward_std": 0.0559066707889239,
-      "rewards/concensus_correctness_reward_func": 1.6154999832312267,
-      "rewards/consensus_reward_func": 1.8333333333333333,
+      "loss": 0.0004,
+      "reward": 5.857549011707306,
+      "reward_std": 0.021132514579221606,
+      "rewards/concensus_correctness_reward_func": 1.4578333099683125,
+      "rewards/consensus_reward_func": 1.6666666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.956727663675944,
+      "rewards/question_recreation_reward_func": 0.9830490450064341,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.4583333333333333,
-      "rewards/xmlcount_reward_func": 1.2419583400090535,
+      "rewards/strict_format_reward_func": 0.5,
+      "rewards/xmlcount_reward_func": 1.25,
       "step": 12
     },
     {
-      "completion_length": 152.3125,
+      "completion_length": 151.6875,
       "epoch": 3.571428571428571,
-      "grad_norm": 13.161581039428711,
-      "kl": 0.8139539376134053,
+      "grad_norm": 42.12635040283203,
+      "kl": 2.118111965013668,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0008,
-      "reward": 5.977140709757805,
-      "reward_std": 0.3884289844427258,
-      "rewards/concensus_correctness_reward_func": 1.5888749845325947,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0021,
+      "reward": 5.8429388999938965,
+      "reward_std": 0.42098340444499627,
+      "rewards/concensus_correctness_reward_func": 1.5896249823272228,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9791407212615013,
+      "rewards/question_recreation_reward_func": 0.9300950989127159,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.2060000076889992,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.2294687554240227,
       "step": 14
     },
     {
-      "completion_length": 138.125,
+      "completion_length": 147.45833333333334,
       "epoch": 4.0,
-      "grad_norm": 12.443143844604492,
-      "kl": 1.8625677494953077,
+      "grad_norm": 17.08171844482422,
+      "kl": 16.20084721719225,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0014,
-      "reward": 5.712031563123067,
-      "reward_std": 0.665348711113135,
-      "rewards/concensus_correctness_reward_func": 1.6014999747276306,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0122,
+      "reward": 5.538925488789876,
+      "reward_std": 0.7823425438255072,
+      "rewards/concensus_correctness_reward_func": 1.5709999750057857,
+      "rewards/consensus_reward_func": 1.5833333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.870864987373352,
+      "rewards/question_recreation_reward_func": 0.8435088787227869,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1979999939600627,
+      "rewards/xmlcount_reward_func": 1.1660833358764648,
       "step": 16
     },
     {
-      "completion_length": 134.8125,
+      "completion_length": 126.90625,
       "epoch": 4.571428571428571,
-      "grad_norm": 25.22212028503418,
-      "kl": 4.916931964689866,
+      "grad_norm": 41542.9296875,
+      "kl": 894.772123008268,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0049,
-      "reward": 5.7459776699543,
-      "reward_std": 0.5427248467458412,
-      "rewards/concensus_correctness_reward_func": 1.4767499789595604,
+      "loss": 0.8948,
+      "reward": 5.760149195790291,
+      "reward_std": 0.5905774181082961,
+      "rewards/concensus_correctness_reward_func": 1.4777499809861183,
       "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9261964000761509,
+      "rewards/question_recreation_reward_func": 0.9594617262482643,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.421875,
-      "rewards/xmlcount_reward_func": 1.2336562499403954,
+      "rewards/xmlcount_reward_func": 1.2135625034570694,
       "step": 18
     },
     {
-      "completion_length": 142.91666666666666,
+      "completion_length": 140.375,
       "epoch": 5.0,
-      "grad_norm": 72.73345184326172,
-      "kl": 8667.639827588573,
+      "grad_norm": 12.598287582397461,
+      "kl": 0.34355814941227436,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 6.5007,
-      "reward": 5.585010310014089,
-      "reward_std": 0.5861762501299381,
-      "rewards/concensus_correctness_reward_func": 1.538833310206731,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0003,
+      "reward": 5.7935870885849,
+      "reward_std": 0.2697240054840222,
+      "rewards/concensus_correctness_reward_func": 1.5494166413942974,
+      "rewards/consensus_reward_func": 1.5833333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9628436168034872,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.9316704521576563,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3958333333333333,
-      "rewards/xmlcount_reward_func": 1.1875,
+      "rewards/strict_format_reward_func": 0.4166666666666667,
+      "rewards/xmlcount_reward_func": 1.2291666666666667,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7698.1475028006225,
-      "train_runtime": 1123.2771,
-      "train_samples_per_second": 0.285,
-      "train_steps_per_second": 0.018
+      "train_loss": 0.9904466477659298,
+      "train_runtime": 713.0205,
+      "train_samples_per_second": 0.449,
+      "train_steps_per_second": 0.028
     }
   ],
   "logging_steps": 2,