End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bef461733c9bf37484c2069176747480fe26c92ba92cfddafda4ead2f5e72ed
+oid sha256:204c882b54379fa8abce3df2950329e1d4eeccea90480384a4401dc31626349f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.355144987286622e-07,
-    "train_runtime": 621.2835,
+    "train_loss": 6.219378576588497e-07,
+    "train_runtime": 634.3838,
     "train_samples": 11,
-    "train_samples_per_second": 0.515,
+    "train_samples_per_second": 0.504,
     "train_steps_per_second": 0.032
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.355144987286622e-07,
-    "train_runtime": 621.2835,
+    "train_loss": 6.219378576588497e-07,
+    "train_runtime": 634.3838,
     "train_samples": 11,
-    "train_samples_per_second": 0.515,
+    "train_samples_per_second": 0.504,
     "train_steps_per_second": 0.032
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 407.5,
+      "completion_length": 466.9375,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.5940650701522827,
-      "kl": 0.0007805308450770099,
+      "grad_norm": 0.4590955376625061,
+      "kl": 0.0006995353105594404,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.089016579091549,
-      "reward_std": 1.0153970010578632,
-      "rewards/concensus_correctness_reward_func": 0.008999999612569809,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.0615508595947176,
+      "reward_std": 1.0327275833114982,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43173531256616116,
+      "rewards/question_recreation_reward_func": 0.5619883388280869,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3826562473550439,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.14018749445676804,
       "step": 2
     },
     {
-      "completion_length": 438.57142857142856,
+      "completion_length": 527.7857142857143,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.528326153755188,
-      "kl": 0.0007703895846913968,
+      "grad_norm": 0.3488618731498718,
+      "kl": 0.0006287240789138845,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.8055335581302643,
-      "reward_std": 0.7478888928890228,
-      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.063449433871678,
+      "reward_std": 1.1212820240429469,
+      "rewards/concensus_correctness_reward_func": 0.009285713945116316,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.35178354595388683,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.26569944407258717,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30196428298950195,
+      "rewards/xmlcount_reward_func": 0.41346428649766104,
       "step": 4
     },
     {
-      "completion_length": 402.2857142857143,
+      "completion_length": 484.60714285714283,
       "epoch": 2.0,
-      "grad_norm": 0.3969787061214447,
-      "kl": 0.0007904843743225294,
+      "grad_norm": 0.3382681906223297,
+      "kl": 0.0006566861369979701,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.0549773488725935,
-      "reward_std": 0.95536869764328,
-      "rewards/concensus_correctness_reward_func": 0.03142857125827244,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.45333464443683624,
+      "reward_std": 0.9945533658776965,
+      "rewards/concensus_correctness_reward_func": 0.018928570406777517,
+      "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.502727325473513,
+      "rewards/question_recreation_reward_func": 0.44633465153830393,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30653572188956396,
+      "rewards/xmlcount_reward_func": -0.1547857118504388,
       "step": 6
     },
     {
-      "completion_length": 449.4375,
+      "completion_length": 515.65625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.4158034920692444,
-      "kl": 0.0006508094738819636,
+      "grad_norm": 0.38815051317214966,
+      "kl": 0.000585481491725659,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.7934029828757048,
-      "reward_std": 1.0302056893706322,
-      "rewards/concensus_correctness_reward_func": 0.0044999998062849045,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.9084305539727211,
+      "reward_std": 1.0081006679683924,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.481402987614274,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.02625000011175871,
+      "rewards/question_recreation_reward_func": 0.3833993040025234,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15003126417286694,
       "step": 8
     },
     {
-      "completion_length": 547.4285714285714,
+      "completion_length": 466.39285714285717,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.7092744708061218,
-      "kl": 0.0007078879072131323,
+      "grad_norm": 0.5429649353027344,
+      "kl": 0.0007274033975720938,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.6819984870297569,
-      "reward_std": 1.1465651563235693,
-      "rewards/concensus_correctness_reward_func": -0.016357143010411943,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 0.8720454105309078,
+      "reward_std": 0.9343861000878471,
+      "rewards/concensus_correctness_reward_func": 0.03785714081355503,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4367127673966544,
+      "rewards/question_recreation_reward_func": 0.43379540528569904,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11878570701394763,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.09682144224643707,
       "step": 10
     },
     {
-      "completion_length": 526.7857142857143,
+      "completion_length": 495.42857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.25688636302948,
-      "kl": 0.0006741188657802663,
+      "grad_norm": 0.2401263564825058,
+      "kl": 0.0006068038305134646,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9941186138561794,
-      "reward_std": 0.8434865730149406,
-      "rewards/concensus_correctness_reward_func": -0.05057142887796674,
+      "reward": 0.8263501333338874,
+      "reward_std": 1.0283640282494682,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5689757402454104,
+      "rewards/question_recreation_reward_func": 0.39677869847842623,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11857144641024726,
+      "rewards/xmlcount_reward_func": 0.0724285798413413,
       "step": 12
     },
     {
-      "completion_length": 546.96875,
+      "completion_length": 440.5625,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.35739797353744507,
-      "kl": 0.0007387452606053557,
+      "grad_norm": 1.4231760501861572,
+      "kl": 0.0007035199232632294,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.4755905121564865,
-      "reward_std": 0.9978351555764675,
-      "rewards/concensus_correctness_reward_func": 0.07062499970197678,
+      "reward": 1.0565725192427635,
+      "reward_std": 1.2860574126243591,
+      "rewards/concensus_correctness_reward_func": 0.01656249910593033,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5299967229366302,
+      "rewards/question_recreation_reward_func": 0.37782250344753265,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.4687187559902668,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27156251296401024,
       "step": 14
     },
     {
-      "completion_length": 474.5,
+      "completion_length": 540.5357142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.5380047559738159,
-      "kl": 0.0007827934750821441,
+      "grad_norm": 1.7067158222198486,
+      "kl": 0.000782927913990404,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.1178060003689356,
-      "reward_std": 0.9631217760699136,
-      "rewards/concensus_correctness_reward_func": -0.007428571581840515,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.7602636175496238,
+      "reward_std": 1.1363796251160758,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5305917114019394,
+      "rewards/question_recreation_reward_func": 0.43315647542476654,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.23750000806259258,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.02353573271206447,
       "step": 16
     },
     {
-      "completion_length": 377.2857142857143,
+      "completion_length": 508.39285714285717,
       "epoch": 6.0,
-      "grad_norm": 0.6083970665931702,
-      "kl": 0.0009674839176503676,
+      "grad_norm": 0.33809056878089905,
+      "kl": 0.0006033167924865016,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.7046442798205785,
-      "reward_std": 0.9649685876710075,
-      "rewards/concensus_correctness_reward_func": 0.015428570764405387,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.1679890028067998,
+      "reward_std": 1.0983782325472151,
+      "rewards/concensus_correctness_reward_func": 0.018928570406777517,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43110853007861544,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.4114890034709658,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.5795357120888573,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1661428383418492,
       "step": 18
     },
     {
-      "completion_length": 503.5625,
+      "completion_length": 471.4375,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.5003705620765686,
-      "kl": 0.0009899184442474507,
+      "grad_norm": 0.43950626254081726,
+      "kl": 0.0006802630596212111,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.9532081484794617,
-      "reward_std": 1.5441483110189438,
-      "rewards/concensus_correctness_reward_func": 0.005374999716877937,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.1322370530106127,
+      "reward_std": 0.9111562184989452,
+      "rewards/concensus_correctness_reward_func": 0.015625,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3971456475555897,
+      "rewards/question_recreation_reward_func": 0.45245583169162273,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.0975625105202198,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16415624879300594,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.355144987286622e-07,
-      "train_runtime": 621.2835,
-      "train_samples_per_second": 0.515,
+      "train_loss": 6.219378576588497e-07,
+      "train_runtime": 634.3838,
+      "train_samples_per_second": 0.504,
       "train_steps_per_second": 0.032
     }
   ],