End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c961420335e1c924b9cf652210b9e740ef814366585a51c5f199e55c91f3870
+oid sha256:7bc8d08991822457b2888707e0845cd50d1b9f296b881e0618678389058df4dc
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.587115760543383e-07,
-    "train_runtime": 1705.0529,
+    "train_loss": 7.770155662001344e-07,
+    "train_runtime": 1641.4504,
     "train_samples": 14,
-    "train_samples_per_second": 0.188,
+    "train_samples_per_second": 0.195,
     "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.587115760543383e-07,
-    "train_runtime": 1705.0529,
+    "train_loss": 7.770155662001344e-07,
+    "train_runtime": 1641.4504,
     "train_samples": 14,
-    "train_samples_per_second": 0.188,
+    "train_samples_per_second": 0.195,
     "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 347.9375,
+      "completion_length": 253.09375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.5356497168540955,
-      "kl": 0.000872969270858448,
+      "grad_norm": 0.8441521525382996,
+      "kl": 0.001027968966809567,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.167648907750845,
-      "reward_std": 1.1919554769992828,
-      "rewards/concensus_correctness_reward_func": 0.12287499941885471,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 2.445363014936447,
+      "reward_std": 1.554322987794876,
+      "rewards/concensus_correctness_reward_func": 0.5899375192821026,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5234926212579012,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.36276928428560495,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2087812628597021,
+      "rewards/xmlcount_reward_func": 0.5551562518812716,
       "step": 2
     },
     {
-      "completion_length": 235.58333333333334,
+      "completion_length": 265.3333333333333,
       "epoch": 1.0,
-      "grad_norm": 0.37739747762680054,
-      "kl": 0.0007977873416772733,
+      "grad_norm": 0.5612057447433472,
+      "kl": 0.0008629107614979148,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 11.468140810728073,
-      "reward_std": 7.96479082852602,
-      "rewards/concensus_correctness_reward_func": 8.333333333333334,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "reward": 2.595840906103452,
+      "reward_std": 2.7292892833550773,
+      "rewards/concensus_correctness_reward_func": 0.875,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.566015842060248,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.5107991347710291,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4854583491881688,
+      "rewards/xmlcount_reward_func": 0.293374997874101,
       "step": 4
     },
     {
-      "completion_length": 303.59375,
+      "completion_length": 258.4375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.6155076026916504,
-      "kl": 0.0009255852637579665,
+      "grad_norm": 0.6174992322921753,
+      "kl": 0.0008592173544457182,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.4901900067925453,
-      "reward_std": 2.660517193377018,
-      "rewards/concensus_correctness_reward_func": 1.9703124985098839,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 4.440339490771294,
+      "reward_std": 3.852556239813566,
+      "rewards/concensus_correctness_reward_func": 2.0627499986439943,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.43297142535448074,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.4768081372603774,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3837812524288893,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5257812514901161,
       "step": 6
     },
     {
-      "completion_length": 284.3333333333333,
+      "completion_length": 304.5833333333333,
       "epoch": 2.0,
-      "grad_norm": 0.33494114875793457,
-      "kl": 0.0008494727759777257,
+      "grad_norm": 0.3486388325691223,
+      "kl": 0.0008832794167877486,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 6.63089241584142,
-      "reward_std": 6.649104615052541,
-      "rewards/concensus_correctness_reward_func": 4.177083333333333,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 2.232958128054937,
+      "reward_std": 2.3463094731171927,
+      "rewards/concensus_correctness_reward_func": 0.8541666666666666,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.47814221680164337,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.6007080723841985,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4756666806836923,
+      "rewards/xmlcount_reward_func": 0.528083344300588,
       "step": 8
     },
     {
-      "completion_length": 296.71875,
+      "completion_length": 271.3125,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.6769649982452393,
-      "kl": 0.00095233663887484,
+      "grad_norm": 0.5388278961181641,
+      "kl": 0.0008147476255544461,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.9268490821123123,
-      "reward_std": 3.860744908452034,
-      "rewards/concensus_correctness_reward_func": 1.9869375247508287,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 4.523550663143396,
+      "reward_std": 3.4894793033599854,
+      "rewards/concensus_correctness_reward_func": 2.5368750002235174,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5795054230839014,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5827383045107126,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3447812532540411,
+      "rewards/xmlcount_reward_func": 0.5133124943822622,
       "step": 10
     },
     {
-      "completion_length": 272.3333333333333,
+      "completion_length": 277.2916666666667,
       "epoch": 3.0,
-      "grad_norm": 0.36784347891807556,
-      "kl": 0.0010108925537982334,
+      "grad_norm": 0.32562169432640076,
+      "kl": 0.0008330401809265217,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 4.578291356563568,
-      "reward_std": 3.1783340821663537,
-      "rewards/concensus_correctness_reward_func": 2.742249995470047,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 4.457584659258525,
+      "reward_std": 2.9557254314422607,
+      "rewards/concensus_correctness_reward_func": 2.7333333492279053,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.39724966883659363,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.38312624643246335,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.5846250007549921,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2577916731437047,
       "step": 12
     },
     {
-      "completion_length": 335.53125,
+      "completion_length": 312.40625,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.46538424491882324,
-      "kl": 0.0007929047133075073,
+      "grad_norm": 0.5490025281906128,
+      "kl": 0.0008507163001922891,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 7.037881502881646,
-      "reward_std": 6.554215002804995,
-      "rewards/concensus_correctness_reward_func": 5.057874999940395,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 2.4363817051053047,
+      "reward_std": 1.9251035377383232,
+      "rewards/concensus_correctness_reward_func": 0.7594999903813004,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5687252599745989,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.583256684243679,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16128124482929707,
+      "rewards/xmlcount_reward_func": 0.5936249978840351,
       "step": 14
     },
     {
-      "completion_length": 267.2916666666667,
+      "completion_length": 263.9166666666667,
       "epoch": 4.0,
-      "grad_norm": 0.366767555475235,
-      "kl": 0.0009911478749321152,
+      "grad_norm": 0.5361287593841553,
+      "kl": 0.0008900455140974373,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 3.2440696557362876,
-      "reward_std": 3.304353336493174,
-      "rewards/concensus_correctness_reward_func": 1.7609166651964188,
+      "reward": 3.834973076979319,
+      "reward_std": 3.299295981725057,
+      "rewards/concensus_correctness_reward_func": 1.73666666696469,
       "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.3033614456653595,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.2683481238782406,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5131250148018202,
+      "rewards/xmlcount_reward_func": 0.4966250037153562,
       "step": 16
     },
     {
-      "completion_length": 321.5,
+      "completion_length": 288.84375,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.6150040626525879,
-      "kl": 0.000730342733731959,
+      "grad_norm": 0.77579265832901,
+      "kl": 0.0009945300407707691,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.889002770185471,
-      "reward_std": 3.9090433791279793,
-      "rewards/concensus_correctness_reward_func": 2.761312510818243,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 4.240542829036713,
+      "reward_std": 3.941610235720873,
+      "rewards/concensus_correctness_reward_func": 2.0078749991953373,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.5080652236938477,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5649179890751839,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24462500400841236,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5271250046789646,
       "step": 18
     },
     {
-      "completion_length": 279.125,
+      "completion_length": 248.5,
       "epoch": 5.0,
-      "grad_norm": 0.4653594195842743,
-      "kl": 0.0008820409517890463,
+      "grad_norm": 0.27806469798088074,
+      "kl": 0.0008993402977163593,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 5.272506097952525,
-      "reward_std": 5.008866022030513,
-      "rewards/concensus_correctness_reward_func": 3.3964999988675117,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 2.3960214108228683,
+      "reward_std": 2.7024554796516895,
+      "rewards/concensus_correctness_reward_func": 0.9303333337108294,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.35333968202273053,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.4027714629968007,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.3351666610687971,
+      "rewards/xmlcount_reward_func": 0.29208332921067875,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.587115760543383e-07,
-      "train_runtime": 1705.0529,
-      "train_samples_per_second": 0.188,
+      "train_loss": 7.770155662001344e-07,
+      "train_runtime": 1641.4504,
+      "train_samples_per_second": 0.195,
       "train_steps_per_second": 0.012
     }
   ],