End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.580423071980476e-06,
-    "train_runtime": 1193.0864,
-    "train_samples": 35,
-    "train_samples_per_second": 0.268,
-    "train_steps_per_second": 0.017
+    "train_loss": 2.0149204829067457e-06,
+    "train_runtime": 695.2878,
+    "train_samples": 28,
+    "train_samples_per_second": 0.46,
+    "train_steps_per_second": 0.029
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e6077ebf9d6a04dfc98a6496fcc303cdfe8f7da66c0db6b6a6ac32c6d06ded5
+oid sha256:ff39378e0867f960f4f98ec7e376a770bda5e3dda468bec99a9ef7f189298d7a
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.580423071980476e-06,
-    "train_runtime": 1193.0864,
-    "train_samples": 35,
-    "train_samples_per_second": 0.268,
-    "train_steps_per_second": 0.017
+    "train_loss": 2.0149204829067457e-06,
+    "train_runtime": 695.2878,
+    "train_samples": 28,
+    "train_samples_per_second": 0.46,
+    "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 4.0,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 363.4375,
-      "epoch": 0.45714285714285713,
-      "grad_norm": 5.619543552398682,
+      "completion_length": 440.6875,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 9.569999694824219,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.773698499193415,
-      "reward_std": 0.48795315153256524,
+      "loss": -0.0,
+      "reward": 0.3243499989621341,
+      "reward_std": 0.3059519662056118,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.2940422515384853,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2623187464196235,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.22965624928474426,
+      "rewards/xmlcount_reward_func": 0.062031252309679985,
       "step": 2
     },
     {
-      "completion_length": 364.09375,
-      "epoch": 0.9142857142857143,
-      "grad_norm": 33.36140060424805,
-      "kl": 0.001602776515937876,
+      "completion_length": 452.625,
+      "epoch": 1.0,
+      "grad_norm": 4.007540225982666,
+      "kl": 0.0011334384956474726,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.3017667849198915,
-      "reward_std": 0.3853252926201094,
+      "reward": 0.39658889764298993,
+      "reward_std": 0.40815691797373194,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.29989178088726476,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.25904723287870485,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06062499713152647,
+      "rewards/xmlcount_reward_func": 0.054208334535360336,
       "step": 4
     },
     {
-      "completion_length": 337.1363636363636,
-      "epoch": 1.2285714285714286,
-      "grad_norm": 8.144990921020508,
-      "kl": 0.0023854352290403435,
+      "completion_length": 414.09375,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 7.70716667175293,
+      "kl": 0.0032120771502377465,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.8732337211682037,
-      "reward_std": 0.41643684526736086,
+      "reward": 0.1853979651350528,
+      "reward_std": 0.43380695390987967,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.18181818181818182,
-      "rewards/question_recreation_reward_func": 0.36664281463758513,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.22086670284625143,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.32477272335778584,
+      "rewards/xmlcount_reward_func": -0.035468751564621925,
       "step": 6
     },
     {
-      "completion_length": 422.09375,
-      "epoch": 1.6857142857142857,
-      "grad_norm": 14.14414119720459,
-      "kl": 0.0017187029407068621,
+      "completion_length": 430.0416666666667,
+      "epoch": 2.0,
+      "grad_norm": 2.9629688262939453,
+      "kl": 0.003562211131793447,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.44808060699142516,
-      "reward_std": 0.606506590207573,
+      "reward": -0.13463043607771397,
+      "reward_std": 0.953704769781325,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.29564311308786273,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2258278999167184,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.027437500655651093,
+      "rewards/xmlcount_reward_func": -0.36045834111670655,
       "step": 8
     },
     {
-      "completion_length": 333.59090909090907,
-      "epoch": 2.0,
-      "grad_norm": 2.4536659717559814,
-      "kl": 0.02611667228947309,
+      "completion_length": 435.5,
+      "epoch": 2.571428571428571,
+      "grad_norm": 4.483899116516113,
+      "kl": 0.0037658300061593764,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.08433498152193021,
-      "reward_std": 0.7974421469934896,
+      "reward": 0.2881736643612385,
+      "reward_std": 0.3216674964496633,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.09090909090909091,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.14501678670587187,
+      "rewards/question_recreation_reward_func": 0.20839241141220555,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1515908972783522,
+      "rewards/xmlcount_reward_func": 0.07978125056251884,
       "step": 10
     },
     {
-      "completion_length": 449.21875,
-      "epoch": 2.4571428571428573,
-      "grad_norm": 12.012519836425781,
-      "kl": 0.002608031500130892,
+      "completion_length": 413.9166666666667,
+      "epoch": 3.0,
+      "grad_norm": 4.878198623657227,
+      "kl": 0.00158434373831066,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.7090451065450907,
-      "reward_std": 0.4962189652142115,
+      "reward": 0.22179130464792252,
+      "reward_std": 0.5298341307789087,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.34976384742185473,
+      "rewards/question_recreation_reward_func": 0.35283296182751656,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.21865624748170376,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.1310416671137015,
       "step": 12
     },
     {
-      "completion_length": 444.9375,
-      "epoch": 2.914285714285714,
-      "grad_norm": 6.142345428466797,
-      "kl": 0.0024347002035938203,
+      "completion_length": 476.40625,
+      "epoch": 3.571428571428571,
+      "grad_norm": 6.207679748535156,
+      "kl": 0.0019041266859858297,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.35571448106202297,
-      "reward_std": 0.4904380216903519,
-      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "reward": 0.31079755048267543,
+      "reward_std": 0.6472759782045614,
+      "rewards/concensus_correctness_reward_func": 0.014000000432133675,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1848082299984526,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.245172556838952,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11090624704957008,
+      "rewards/xmlcount_reward_func": -0.010874999687075615,
       "step": 14
     },
     {
-      "completion_length": 379.0,
-      "epoch": 3.2285714285714286,
-      "grad_norm": 27.03514289855957,
-      "kl": 0.002847899341511286,
+      "completion_length": 426.375,
+      "epoch": 4.0,
+      "grad_norm": 13.98342227935791,
+      "kl": 0.0012475498854958762,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0918640744957058,
-      "reward_std": 0.8158379501917146,
-      "rewards/concensus_correctness_reward_func": 0.08072727376764471,
-      "rewards/consensus_reward_func": 0.09090909090909091,
+      "reward": 0.41232059461375076,
+      "reward_std": 0.5356247463884453,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2727272727272727,
-      "rewards/question_recreation_reward_func": 0.46945498477328906,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.21273726566384235,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17804545705968683,
+      "rewards/xmlcount_reward_func": 0.19958333671092987,
       "step": 16
     },
     {
-      "completion_length": 405.8125,
-      "epoch": 3.685714285714286,
-      "grad_norm": 57.54035949707031,
-      "kl": 0.004750865380628966,
+      "completion_length": 402.59375,
+      "epoch": 4.571428571428571,
+      "grad_norm": 115.46556854248047,
+      "kl": 0.0036373182701936457,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.3959474053699523,
-      "reward_std": 0.5196433831006289,
-      "rewards/concensus_correctness_reward_func": 0.10868749767541885,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.3796658612554893,
+      "reward_std": 1.0177593523985706,
+      "rewards/concensus_correctness_reward_func": 0.06731250043958426,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1865098994167056,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.29647835373179987,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10075000347569585,
+      "rewards/xmlcount_reward_func": -0.10912500461563468,
       "step": 18
     },
     {
-      "completion_length": 341.6363636363636,
-      "epoch": 4.0,
-      "grad_norm": 4.428709983825684,
-      "kl": 0.007117496922464025,
+      "completion_length": 380.2083333333333,
+      "epoch": 5.0,
+      "grad_norm": 15.835192680358887,
+      "kl": 0.0029354243500468633,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.07562183148481628,
-      "reward_std": 0.8177409039979632,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.43359189790983993,
+      "reward_std": 0.3545907170822223,
+      "rewards/concensus_correctness_reward_func": 0.019916666050752003,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3233491080728444,
+      "rewards/question_recreation_reward_func": 0.1713419142179191,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.2477272776040164,
+      "rewards/xmlcount_reward_func": 0.15899999563892683,
       "step": 20
     },
     {
-      "epoch": 4.0,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.580423071980476e-06,
-      "train_runtime": 1193.0864,
-      "train_samples_per_second": 0.268,
-      "train_steps_per_second": 0.017
+      "train_loss": 2.0149204829067457e-06,
+      "train_runtime": 695.2878,
+      "train_samples_per_second": 0.46,
+      "train_steps_per_second": 0.029
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 5,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {