End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ed03b6b96169cae6f7225e3914577de0c8cae6b006dd24bdd0023d4642e1cac
+oid sha256:e6ead104683349173622bcec18b97442ffa6574ba6972b040e52dde52adf6e5f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.093154235666589e-07,
-    "train_runtime": 591.1687,
+    "train_loss": 7.850331087411178e-07,
+    "train_runtime": 601.4811,
     "train_samples": 11,
-    "train_samples_per_second": 0.541,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 0.532,
+    "train_steps_per_second": 0.033
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.093154235666589e-07,
-    "train_runtime": 591.1687,
+    "train_loss": 7.850331087411178e-07,
+    "train_runtime": 601.4811,
     "train_samples": 11,
-    "train_samples_per_second": 0.541,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 0.532,
+    "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 468.03125,
+      "completion_length": 440.21875,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.4582962989807129,
-      "kl": 0.0008591811565565877,
+      "grad_norm": 0.4294714033603668,
+      "kl": 0.0008740375633351505,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.9028365463018417,
-      "reward_std": 1.1918021105229855,
-      "rewards/concensus_correctness_reward_func": 0.05624999850988388,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.9038631860166788,
+      "reward_std": 0.8780690524727106,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4146490623243153,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5466131996363401,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.10381251480430365,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16975000221282244,
       "step": 2
     },
     {
-      "completion_length": 518.5357142857143,
+      "completion_length": 504.07142857142856,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.3954562246799469,
-      "kl": 0.0008725193204424743,
+      "grad_norm": 0.3360004723072052,
+      "kl": 0.0007394546825837876,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.194850172315325,
-      "reward_std": 1.0669879913330078,
-      "rewards/concensus_correctness_reward_func": 0.017857142857142856,
+      "reward": 1.0961955828326089,
+      "reward_std": 1.0172933425222124,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5166715937001365,
+      "rewards/question_recreation_reward_func": 0.43351701008422033,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30317857222897665,
+      "rewards/xmlcount_reward_func": 0.3055357166699001,
       "step": 4
     },
     {
-      "completion_length": 385.75,
+      "completion_length": 476.17857142857144,
       "epoch": 2.0,
-      "grad_norm": 0.42865145206451416,
-      "kl": 0.0010318544643398906,
+      "grad_norm": 0.3323291540145874,
+      "kl": 0.000823132585667606,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.21840481034347,
-      "reward_std": 0.6876192965677806,
-      "rewards/concensus_correctness_reward_func": 0.003642857074737549,
+      "reward": 1.0995356323463576,
+      "reward_std": 0.7275045003209796,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.45765478909015656,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.5147499101502555,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.5249642963920321,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.29907142903123585,
       "step": 6
     },
     {
-      "completion_length": 411.5,
+      "completion_length": 461.375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.5174846649169922,
-      "kl": 0.0009470830264035612,
+      "grad_norm": 0.5940535664558411,
+      "kl": 0.0008499226314597763,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.0020326934754848,
-      "reward_std": 1.2967891693115234,
-      "rewards/concensus_correctness_reward_func": 0.06262499839067459,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 0.6256613619625568,
+      "reward_std": 1.1299266181886196,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4319077325053513,
+      "rewards/question_recreation_reward_func": 0.4355988558381796,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.03875000216066837,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.002562491223216057,
       "step": 8
     },
     {
-      "completion_length": 456.35714285714283,
+      "completion_length": 484.5357142857143,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.34678906202316284,
-      "kl": 0.000698891597234511,
+      "grad_norm": 0.7707741260528564,
+      "kl": 0.00088938106949042,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.9917895368167332,
-      "reward_std": 1.1331866468702043,
-      "rewards/concensus_correctness_reward_func": 0.021499999931880405,
+      "reward": 1.2423403646264757,
+      "reward_std": 0.5533896961382457,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.4156466616051538,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5163760696138654,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34035714183534893,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.5652499965259007,
       "step": 10
     },
     {
-      "completion_length": 377.85714285714283,
+      "completion_length": 381.7857142857143,
       "epoch": 4.0,
-      "grad_norm": 0.38536587357521057,
-      "kl": 0.0010365713221420134,
+      "grad_norm": 0.4201972782611847,
+      "kl": 0.0008284871998642172,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.0362207075314862,
-      "reward_std": 0.5890559788261142,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.727727758032935,
+      "reward_std": 0.8987090247017997,
+      "rewards/concensus_correctness_reward_func": 0.04642856972558158,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3699350229331425,
+      "rewards/question_recreation_reward_func": 0.45708488140787396,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.43414284927504404,
+      "rewards/xmlcount_reward_func": 0.13492857558386667,
       "step": 12
     },
     {
-      "completion_length": 429.90625,
+      "completion_length": 496.53125,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.6163267493247986,
-      "kl": 0.0008207274950109422,
+      "grad_norm": 0.5337567925453186,
+      "kl": 0.0008140378195093945,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.9250219464302063,
-      "reward_std": 0.9558937475085258,
-      "rewards/concensus_correctness_reward_func": 0.0182499997317791,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.8857628256082535,
+      "reward_std": 0.7104290407150984,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.47452196246013045,
+      "rewards/question_recreation_reward_func": 0.4623878486454487,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.1666250042617321,
+      "rewards/xmlcount_reward_func": 0.40775001229485497,
       "step": 14
     },
     {
-      "completion_length": 455.4642857142857,
+      "completion_length": 493.2142857142857,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.5645512938499451,
-      "kl": 0.0011287129933147558,
+      "grad_norm": 0.41676825284957886,
+      "kl": 0.0008308267736408327,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5952527565615517,
-      "reward_std": 0.8575102069548198,
-      "rewards/concensus_correctness_reward_func": 0.010428571275302343,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "reward": 0.7405556184904916,
+      "reward_std": 1.0715518082891191,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.34239561749356134,
+      "rewards/question_recreation_reward_func": 0.398269921541214,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.15314286202192307,
+      "rewards/xmlcount_reward_func": 0.03871428008590426,
       "step": 16
     },
     {
-      "completion_length": 557.3571428571429,
+      "completion_length": 434.85714285714283,
       "epoch": 6.0,
-      "grad_norm": 0.2774151861667633,
-      "kl": 0.0006982476791433458,
+      "grad_norm": 0.3682529628276825,
+      "kl": 0.0007807948111024286,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.8293260633945465,
-      "reward_std": 0.8850810527801514,
-      "rewards/concensus_correctness_reward_func": 0.03571428571428571,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 0.9870162712676185,
+      "reward_std": 0.9266383754355567,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.45307605607169016,
+      "rewards/question_recreation_reward_func": 0.49662342028958456,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05482143642646926,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.258250002350126,
       "step": 18
     },
     {
-      "completion_length": 527.5,
+      "completion_length": 485.90625,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.3615259826183319,
-      "kl": 0.0007502013068005908,
+      "grad_norm": 0.376955509185791,
+      "kl": 0.0009621086792321876,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8531642556190491,
-      "reward_std": 1.1323423981666565,
-      "rewards/concensus_correctness_reward_func": 0.015625,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.7075213864445686,
+      "reward_std": 0.8086372166872025,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.48825799115002155,
+      "rewards/question_recreation_reward_func": 0.3648651074618101,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.04134374111890793,
+      "rewards/xmlcount_reward_func": 0.20203125104308128,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.093154235666589e-07,
-      "train_runtime": 591.1687,
-      "train_samples_per_second": 0.541,
-      "train_steps_per_second": 0.034
+      "train_loss": 7.850331087411178e-07,
+      "train_runtime": 601.4811,
+      "train_samples_per_second": 0.532,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 2,