End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.3861788213253023e-05,
-    "train_runtime": 339.6079,
+    "train_loss": 3.586895763874054e-06,
+    "train_runtime": 310.4043,
     "train_samples": 28,
-    "train_samples_per_second": 0.942,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 1.031,
+    "train_steps_per_second": 0.064
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce538056736b82c3b43ceffd7952ae75d9bba36906e7528073814ebb993f51b4
+oid sha256:4fe197a9ef5ada02edf66c70899bfe3daf339e7b90b75fc4eaa0b7cd37633684
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.3861788213253023e-05,
-    "train_runtime": 339.6079,
+    "train_loss": 3.586895763874054e-06,
+    "train_runtime": 310.4043,
     "train_samples": 28,
-    "train_samples_per_second": 0.942,
-    "train_steps_per_second": 0.059
+    "train_samples_per_second": 1.031,
+    "train_steps_per_second": 0.064
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 407.6875,
+      "completion_length": 435.4375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 25.53277587890625,
+      "grad_norm": 3.8439865112304688,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 1.0433049155399203,
-      "reward_std": 0.782902276609093,
-      "rewards/concensus_correctness_reward_func": -0.008625000715255737,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": -0.0,
+      "reward": 0.6914588066283613,
+      "reward_std": 0.9236088133766316,
+      "rewards/concensus_correctness_reward_func": 0.05974999815225601,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5767111429013312,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3949900634761434,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.35021876357495785,
+      "rewards/xmlcount_reward_func": 0.11171875428408384,
       "step": 2
     },
     {
-      "completion_length": 407.75,
+      "completion_length": 333.1666666666667,
       "epoch": 1.0,
-      "grad_norm": 2.0310778617858887,
-      "kl": 0.0009430798551572176,
+      "grad_norm": 4.06553316116333,
+      "kl": 0.0011363048397470266,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.4023114492495854,
-      "reward_std": 1.0993938581862797,
-      "rewards/concensus_correctness_reward_func": 0.10033333549896876,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.343650996685028,
+      "reward_std": 0.9110305064047376,
+      "rewards/concensus_correctness_reward_func": 0.03800000001986822,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.48901975868890685,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.4201510362327099,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3962916728730003,
+      "rewards/xmlcount_reward_func": 0.6354999976853529,
       "step": 4
     },
     {
-      "completion_length": 521.28125,
+      "completion_length": 349.0,
       "epoch": 1.5714285714285714,
-      "grad_norm": 4.210171222686768,
-      "kl": 0.0017607526024221443,
+      "grad_norm": 5.977542400360107,
+      "kl": 0.002151765987946419,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.712698183953762,
-      "reward_std": 1.150176638038829,
-      "rewards/concensus_correctness_reward_func": 0.13343749940395355,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.060482904314995,
+      "reward_std": 2.559483935125172,
+      "rewards/concensus_correctness_reward_func": 0.8796250000596046,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5881356769241393,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.43748294562101364,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5536250025033951,
+      "rewards/xmlcount_reward_func": 0.1808750033378601,
       "step": 6
     },
     {
-      "completion_length": 451.6666666666667,
+      "completion_length": 349.0,
       "epoch": 2.0,
-      "grad_norm": 2.436331033706665,
-      "kl": 0.0020083066289468357,
+      "grad_norm": 3.4864773750305176,
+      "kl": 0.002737251779763028,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.2294512589772542,
-      "reward_std": 1.7083880168696244,
-      "rewards/concensus_correctness_reward_func": 0.1574166715145111,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.5876267155011494,
+      "reward_std": 0.7928874848100046,
+      "rewards/concensus_correctness_reward_func": 0.03283333328242103,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5759512403359016,
+      "rewards/question_recreation_reward_func": 0.4747934063586096,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07941667238871257,
+      "rewards/xmlcount_reward_func": 0.49666666115323704,
       "step": 8
     },
     {
-      "completion_length": 402.4375,
+      "completion_length": 409.59375,
       "epoch": 2.571428571428571,
-      "grad_norm": 5.234063148498535,
-      "kl": 0.004025009140605107,
+      "grad_norm": 4.636322021484375,
+      "kl": 0.006173002642754,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.241635442711413,
-      "reward_std": 0.8916195398196578,
-      "rewards/concensus_correctness_reward_func": 0.11999999731779099,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.8668802455067635,
+      "reward_std": 1.6170106320641935,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.37804169388255104,
+      "rewards/question_recreation_reward_func": 0.46563018672168255,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4310937486588955,
+      "rewards/xmlcount_reward_func": 0.5262500150129199,
       "step": 10
     },
     {
-      "completion_length": 351.9166666666667,
+      "completion_length": 369.25,
       "epoch": 3.0,
-      "grad_norm": 58.81644821166992,
-      "kl": 0.24880207097157836,
+      "grad_norm": 4.06023645401001,
+      "kl": 0.004249369774091368,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0002,
-      "reward": 1.6927041411399841,
-      "reward_std": 0.9551689277092615,
-      "rewards/concensus_correctness_reward_func": 0.1510833352804184,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": 0.0,
+      "reward": 1.6554374198118846,
+      "reward_std": 0.6809886172413826,
+      "rewards/concensus_correctness_reward_func": 0.07999999821186066,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5061208084225655,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.6594790716965994,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5355000030249357,
+      "rewards/xmlcount_reward_func": 0.6659583300352097,
       "step": 12
     },
     {
-      "completion_length": 568.90625,
+      "completion_length": 345.34375,
       "epoch": 3.571428571428571,
-      "grad_norm": 3.6482553482055664,
-      "kl": 0.004224516262183897,
+      "grad_norm": 4.189980506896973,
+      "kl": 0.006222577969310805,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.4312433823943138,
-      "reward_std": 0.5516666639596224,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.3473324589431286,
+      "reward_std": 0.679174258490093,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6232121395878494,
+      "rewards/question_recreation_reward_func": 0.5202699541114271,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49553125631064177,
+      "rewards/xmlcount_reward_func": 0.5067499973811209,
       "step": 14
     },
     {
-      "completion_length": 384.8333333333333,
+      "completion_length": 369.7083333333333,
       "epoch": 4.0,
-      "grad_norm": 2.1138391494750977,
-      "kl": 0.008626916348778954,
+      "grad_norm": 3.0566606521606445,
+      "kl": 0.006038210412953049,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0923200373848279,
-      "reward_std": 0.7976163200413188,
-      "rewards/concensus_correctness_reward_func": 0.028333333631356556,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.7356221427520115,
+      "reward_std": 0.8265478493024906,
+      "rewards/concensus_correctness_reward_func": 0.11958333399767677,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.5855700361231962,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.511413821640114,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.31174999475479126,
+      "rewards/xmlcount_reward_func": 0.6046250065167745,
       "step": 16
     },
     {
-      "completion_length": 419.6875,
+      "completion_length": 389.375,
       "epoch": 4.571428571428571,
-      "grad_norm": 4.1763129234313965,
-      "kl": 0.006283669870754238,
+      "grad_norm": 16.731468200683594,
+      "kl": 0.006534892905619927,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.5983755812048912,
-      "reward_std": 1.4708348184358329,
-      "rewards/concensus_correctness_reward_func": 0.19931250251829624,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.3630528301000595,
+      "reward_std": 0.9979011313989758,
+      "rewards/concensus_correctness_reward_func": 0.11162499710917473,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.45675060944631696,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5975528075359762,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3173124846071005,
+      "rewards/xmlcount_reward_func": 0.4663750035688281,
       "step": 18
     },
     {
-      "completion_length": 411.1666666666667,
+      "completion_length": 407.8333333333333,
       "epoch": 5.0,
-      "grad_norm": 2.9116275310516357,
-      "kl": 0.03663837438216433,
+      "grad_norm": 5.084082126617432,
+      "kl": 0.005891711577229823,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.9936327536900839,
-      "reward_std": 0.6301505618418256,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8431620399157206,
+      "reward_std": 1.1959138717502356,
+      "rewards/concensus_correctness_reward_func": 0.037499999006589256,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6164244307825962,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.37187035474926233,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.37720833842953044,
+      "rewards/xmlcount_reward_func": 0.017124998072783153,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.3861788213253023e-05,
-      "train_runtime": 339.6079,
-      "train_samples_per_second": 0.942,
-      "train_steps_per_second": 0.059
+      "train_loss": 3.586895763874054e-06,
+      "train_runtime": 310.4043,
+      "train_samples_per_second": 1.031,
+      "train_steps_per_second": 0.064
     }
   ],
   "logging_steps": 2,