End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.91517847776413e-06,
-    "train_runtime": 908.6981,
+    "train_loss": 0.0003779519018280553,
+    "train_runtime": 451.5947,
     "train_samples": 28,
-    "train_samples_per_second": 0.352,
-    "train_steps_per_second": 0.022
+    "train_samples_per_second": 0.709,
+    "train_steps_per_second": 0.044
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07f2da95991ca40ebcf242d99bb14d4ae8154e93a9f594db928feadde214a9a8
+oid sha256:8bc12d47cf95b49469731a28ec0c60145e1b3b42bb18a756e6e00316a7eaa361
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.91517847776413e-06,
-    "train_runtime": 908.6981,
+    "train_loss": 0.0003779519018280553,
+    "train_runtime": 451.5947,
     "train_samples": 28,
-    "train_samples_per_second": 0.352,
-    "train_steps_per_second": 0.022
+    "train_samples_per_second": 0.709,
+    "train_steps_per_second": 0.044
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 476.78125,
+      "completion_length": 344.625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 4.431652069091797,
+      "grad_norm": 10.772970199584961,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.92695583216846,
-      "reward_std": 2.384034690912813,
-      "rewards/concensus_correctness_reward_func": 0.8203750029206276,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.24003540235571563,
+      "reward_std": 0.3615740498935338,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4601745998952538,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.25100415269844234,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.208906244719401,
+      "rewards/xmlcount_reward_func": -0.010968748014420271,
       "step": 2
     },
     {
-      "completion_length": 375.1666666666667,
+      "completion_length": 361.7083333333333,
       "epoch": 1.0,
-      "grad_norm": 2.8096444606781006,
-      "kl": 0.0017536189673895326,
+      "grad_norm": 42.27848434448242,
+      "kl": 0.0015548948867945,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.7996605038642883,
-      "reward_std": 1.0895458900680144,
-      "rewards/concensus_correctness_reward_func": 0.5238333344459534,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 0.4717333244625479,
+      "reward_std": 0.5787315891357139,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.6694520985086759,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2576499928254634,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7730416692793369,
+      "rewards/xmlcount_reward_func": 0.13075000412451723,
       "step": 4
     },
     {
-      "completion_length": 412.5625,
+      "completion_length": 383.4375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 5.2593913078308105,
-      "kl": 0.005835396033944562,
+      "grad_norm": 6.3906779289245605,
+      "kl": 0.003872946799674537,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.567254636436701,
-      "reward_std": 1.7708478854037821,
-      "rewards/concensus_correctness_reward_func": 0.9221875034272671,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.4794421146507375,
+      "reward_std": 0.27617324620950967,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6375983776524663,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.30297336814692244,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.44496875908225775,
+      "rewards/xmlcount_reward_func": 0.11396875674836338,
       "step": 6
     },
     {
-      "completion_length": 377.2916666666667,
+      "completion_length": 388.7916666666667,
       "epoch": 2.0,
-      "grad_norm": 5.22662353515625,
-      "kl": 0.005144034415328254,
+      "grad_norm": 10.911130905151367,
+      "kl": 0.0028913425194332376,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.130537753303846,
-      "reward_std": 1.0754900506387155,
-      "rewards/concensus_correctness_reward_func": 0.24241666992505392,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.46779738351081807,
+      "reward_std": 0.56250952805082,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7983710691332817,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2721307026222348,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8397500018278757,
+      "rewards/xmlcount_reward_func": 0.11233333125710487,
       "step": 8
     },
     {
-      "completion_length": 394.3125,
+      "completion_length": 355.5,
       "epoch": 2.571428571428571,
-      "grad_norm": 5.33180570602417,
-      "kl": 0.011582882769289427,
+      "grad_norm": 9.093505859375,
+      "kl": 0.00975232083874289,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.6330229602754116,
-      "reward_std": 1.1605141687905416,
-      "rewards/concensus_correctness_reward_func": 0.22556250169873238,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.4602516919840127,
+      "reward_std": 0.4189709427018897,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4206479690037668,
+      "rewards/question_recreation_reward_func": 0.2312829417642206,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6118124946951866,
+      "rewards/xmlcount_reward_func": 0.16646874835714698,
       "step": 10
     },
     {
-      "completion_length": 436.5,
+      "completion_length": 291.4166666666667,
       "epoch": 3.0,
-      "grad_norm": 3.7397239208221436,
-      "kl": 0.010172933999759456,
+      "grad_norm": 66.55801391601562,
+      "kl": 0.18349976455404735,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0,
-      "reward": 2.538457383712133,
-      "reward_std": 1.7962782899170027,
-      "rewards/concensus_correctness_reward_func": 0.5396666675806046,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "loss": 0.0001,
+      "reward": 0.32576117835318047,
+      "reward_std": 0.2934178128295268,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7018323391675949,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.24851117640112838,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6302916631102562,
+      "rewards/xmlcount_reward_func": 0.05641666303078333,
       "step": 12
     },
     {
-      "completion_length": 447.15625,
+      "completion_length": 343.09375,
       "epoch": 3.571428571428571,
-      "grad_norm": 5.77060079574585,
-      "kl": 0.012620764726307243,
+      "grad_norm": 17.57956886291504,
+      "kl": 0.0372060123263509,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.3586921766400337,
-      "reward_std": 1.5907898396253586,
-      "rewards/concensus_correctness_reward_func": 0.33725000359117985,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 0.22849576943553984,
+      "reward_std": 0.7314633782953024,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6683796690776944,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.22037077555432916,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6030625160783529,
+      "rewards/xmlcount_reward_func": -0.06218750216066837,
       "step": 14
     },
     {
-      "completion_length": 291.7916666666667,
+      "completion_length": 383.7083333333333,
       "epoch": 4.0,
-      "grad_norm": 3.3413925170898438,
-      "kl": 0.01326390371347467,
+      "grad_norm": 19.69514274597168,
+      "kl": 3.7607118334659995,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0,
-      "reward": 2.5506533284982047,
-      "reward_std": 1.0989755101812382,
-      "rewards/concensus_correctness_reward_func": 0.4469166696071625,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "loss": 0.0028,
+      "reward": 0.08723691183452804,
+      "reward_std": 0.3238961525882284,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.6814449317753315,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.15461191271121302,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8389583478371302,
+      "rewards/xmlcount_reward_func": -0.15070833389957747,
       "step": 16
     },
     {
-      "completion_length": 450.40625,
+      "completion_length": 402.25,
       "epoch": 4.571428571428571,
-      "grad_norm": 7.903269290924072,
-      "kl": 0.015724379918538034,
+      "grad_norm": 15.42269515991211,
+      "kl": 0.0024876317584130447,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.0777935050427914,
-      "reward_std": 0.9112980275676819,
-      "rewards/concensus_correctness_reward_func": 0.18800000473856926,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.46647889574524015,
+      "reward_std": 0.3916395944033866,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7580434791743755,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3051351552712731,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5067500052973628,
+      "rewards/xmlcount_reward_func": 0.09884374821558595,
       "step": 18
     },
     {
-      "completion_length": 492.0,
+      "completion_length": 278.6666666666667,
       "epoch": 5.0,
-      "grad_norm": 2.7399392127990723,
-      "kl": 0.012714986107312143,
+      "grad_norm": 5.755941867828369,
+      "kl": 1.0229452109100141,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0,
-      "reward": 2.185473953684171,
-      "reward_std": 0.9812121223658323,
-      "rewards/concensus_correctness_reward_func": 0.4572500040133794,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "loss": 0.0008,
+      "reward": 0.49779408425092697,
+      "reward_std": 0.30871247686445713,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.5613906369544566,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3205857429032524,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5835000028212866,
+      "rewards/xmlcount_reward_func": 0.17720833172400793,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.91517847776413e-06,
-      "train_runtime": 908.6981,
-      "train_samples_per_second": 0.352,
-      "train_steps_per_second": 0.022
+      "train_loss": 0.0003779519018280553,
+      "train_runtime": 451.5947,
+      "train_samples_per_second": 0.709,
+      "train_steps_per_second": 0.044
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a936cf436fe0b9289b612d4a6d52e2c0909642defc99aa2cca6536bc5a4a81b8
+oid sha256:b8a740c114889c9cfd9d6df66d8762679a5440a165be9704bbcc18da98ff08fd
 size 5880