End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bc8d08991822457b2888707e0845cd50d1b9f296b881e0618678389058df4dc
+oid sha256:fe9b5b384ab43f623a6696c232f2cf26a8d463c960836cc88cc9f30f463544e1
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.770155662001344e-07,
-    "train_runtime": 1641.4504,
+    "train_loss": 8.527416355263995e-07,
+    "train_runtime": 1647.6492,
     "train_samples": 14,
-    "train_samples_per_second": 0.195,
+    "train_samples_per_second": 0.194,
     "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.770155662001344e-07,
-    "train_runtime": 1641.4504,
+    "train_loss": 8.527416355263995e-07,
+    "train_runtime": 1647.6492,
     "train_samples": 14,
-    "train_samples_per_second": 0.195,
+    "train_samples_per_second": 0.194,
     "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 253.09375,
+      "completion_length": 294.3125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.8441521525382996,
-      "kl": 0.001027968966809567,
+      "grad_norm": 0.6074583530426025,
+      "kl": 0.0008409952279180288,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 2.445363014936447,
-      "reward_std": 1.554322987794876,
-      "rewards/concensus_correctness_reward_func": 0.5899375192821026,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 1.133454579859972,
+      "reward_std": 0.6814509630203247,
+      "rewards/concensus_correctness_reward_func": 0.04206249862909317,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.36276928428560495,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.42214206606149673,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5551562518812716,
+      "rewards/xmlcount_reward_func": 0.41924999561160803,
       "step": 2
     },
     {
-      "completion_length": 265.3333333333333,
+      "completion_length": 353.125,
       "epoch": 1.0,
-      "grad_norm": 0.5612057447433472,
-      "kl": 0.0008629107614979148,
+      "grad_norm": 47.75371551513672,
+      "kl": 0.0022826601731746146,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.595840906103452,
-      "reward_std": 2.7292892833550773,
-      "rewards/concensus_correctness_reward_func": 0.875,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.965188905596733,
+      "reward_std": 2.484490007162094,
+      "rewards/concensus_correctness_reward_func": 0.8333333333333334,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.5107991347710291,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5360221962134043,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.293374997874101,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.22083332824210325,
       "step": 4
     },
     {
-      "completion_length": 258.4375,
+      "completion_length": 322.65625,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.6174992322921753,
-      "kl": 0.0008592173544457182,
+      "grad_norm": 0.5006768703460693,
+      "kl": 0.0008434747505816631,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 4.440339490771294,
-      "reward_std": 3.852556239813566,
-      "rewards/concensus_correctness_reward_func": 2.0627499986439943,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 1.9395091831684113,
+      "reward_std": 1.1901352517306805,
+      "rewards/concensus_correctness_reward_func": 0.31156249600462615,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.4768081372603774,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.47363421507179737,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5257812514901161,
+      "rewards/xmlcount_reward_func": 0.38868751376867294,
       "step": 6
     },
     {
-      "completion_length": 304.5833333333333,
+      "completion_length": 304.9166666666667,
       "epoch": 2.0,
-      "grad_norm": 0.3486388325691223,
-      "kl": 0.0008832794167877486,
+      "grad_norm": 0.315962016582489,
+      "kl": 0.0009037207928486168,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.232958128054937,
-      "reward_std": 2.3463094731171927,
-      "rewards/concensus_correctness_reward_func": 0.8541666666666666,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 0.922406812508901,
+      "reward_std": 0.7466032604376475,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.6007080723841985,
+      "rewards/question_recreation_reward_func": 0.501031776269277,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.528083344300588,
+      "rewards/xmlcount_reward_func": 0.33804167062044144,
       "step": 8
     },
     {
-      "completion_length": 271.3125,
+      "completion_length": 299.6875,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.5388278961181641,
-      "kl": 0.0008147476255544461,
+      "grad_norm": 0.6998950242996216,
+      "kl": 0.0009004111489048228,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.523550663143396,
-      "reward_std": 3.4894793033599854,
-      "rewards/concensus_correctness_reward_func": 2.5368750002235174,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.027998574078083,
+      "reward_std": 2.434359833598137,
+      "rewards/concensus_correctness_reward_func": 0.6409374997019768,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5827383045107126,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.45209239795804024,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5133124943822622,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3724687579087913,
       "step": 10
     },
     {
-      "completion_length": 277.2916666666667,
+      "completion_length": 278.1666666666667,
       "epoch": 3.0,
-      "grad_norm": 0.32562169432640076,
-      "kl": 0.0008330401809265217,
+      "grad_norm": 0.37337160110473633,
+      "kl": 0.0008745406521484256,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.457584659258525,
-      "reward_std": 2.9557254314422607,
-      "rewards/concensus_correctness_reward_func": 2.7333333492279053,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 1.5453739960988362,
+      "reward_std": 1.1052613481879234,
+      "rewards/concensus_correctness_reward_func": 0.07408333414544661,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.38312624643246335,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.43824899196624756,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2577916731437047,
+      "rewards/strict_format_reward_func": 0.041666666666666664,
+      "rewards/xmlcount_reward_func": 0.40804166346788406,
       "step": 12
     },
     {
-      "completion_length": 312.40625,
+      "completion_length": 298.40625,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.5490025281906128,
-      "kl": 0.0008507163001922891,
+      "grad_norm": 0.7391191124916077,
+      "kl": 0.0008627813731436618,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.4363817051053047,
-      "reward_std": 1.9251035377383232,
-      "rewards/concensus_correctness_reward_func": 0.7594999903813004,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 3.311558946967125,
+      "reward_std": 4.124635513871908,
+      "rewards/concensus_correctness_reward_func": 1.4486249685287476,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.583256684243679,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6010276973247528,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5936249978840351,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3712812513113022,
       "step": 14
     },
     {
-      "completion_length": 263.9166666666667,
+      "completion_length": 347.25,
       "epoch": 4.0,
-      "grad_norm": 0.5361287593841553,
-      "kl": 0.0008900455140974373,
+      "grad_norm": 0.3265421986579895,
+      "kl": 0.0008276354152864466,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 3.834973076979319,
-      "reward_std": 3.299295981725057,
-      "rewards/concensus_correctness_reward_func": 1.73666666696469,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 1.2531405289967854,
+      "reward_std": 0.898355116446813,
+      "rewards/concensus_correctness_reward_func": 0.058666667280097805,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.2683481238782406,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4269738793373108,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4966250037153562,
+      "rewards/xmlcount_reward_func": 0.35083333899577457,
       "step": 16
     },
     {
-      "completion_length": 288.84375,
+      "completion_length": 297.3125,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.77579265832901,
-      "kl": 0.0009945300407707691,
+      "grad_norm": 0.4362623989582062,
+      "kl": 0.0008101245621219277,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.240542829036713,
-      "reward_std": 3.941610235720873,
-      "rewards/concensus_correctness_reward_func": 2.0078749991953373,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.52060616761446,
+      "reward_std": 2.1808464396744967,
+      "rewards/concensus_correctness_reward_func": 1.2743749991059303,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5649179890751839,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.49613741412758827,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5271250046789646,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3438437443692237,
       "step": 18
     },
     {
-      "completion_length": 248.5,
+      "completion_length": 309.7916666666667,
       "epoch": 5.0,
-      "grad_norm": 0.27806469798088074,
-      "kl": 0.0008993402977163593,
+      "grad_norm": 0.25287431478500366,
+      "kl": 0.0008112282569830617,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.3960214108228683,
-      "reward_std": 2.7024554796516895,
-      "rewards/concensus_correctness_reward_func": 0.9303333337108294,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 0.8035016556580862,
+      "reward_std": 1.0816570321718852,
+      "rewards/concensus_correctness_reward_func": 0.01824999942133824,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.4027714629968007,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.31116832544406253,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.29208332921067875,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2240833342075348,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.770155662001344e-07,
-      "train_runtime": 1641.4504,
-      "train_samples_per_second": 0.195,
+      "train_loss": 8.527416355263995e-07,
+      "train_runtime": 1647.6492,
+      "train_samples_per_second": 0.194,
       "train_steps_per_second": 0.012
     }
   ],