End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6771256923675537e-06,
-    "train_runtime": 1570.9687,
-    "train_samples": 45,
-    "train_samples_per_second": 0.204,
-    "train_steps_per_second": 0.013
+    "train_loss": 1.4123068467597477e-05,
+    "train_runtime": 912.59,
+    "train_samples": 28,
+    "train_samples_per_second": 0.351,
+    "train_steps_per_second": 0.022
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a71119f57dc3b9ad51bc9c9da148f9cb8978c78fe61ed1a30b579f78ab011a47
+oid sha256:c52e84b004a3e5c713b11f8968d01577c9c390b7794349e66fda679569e01991
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6771256923675537e-06,
-    "train_runtime": 1570.9687,
-    "train_samples": 45,
-    "train_samples_per_second": 0.204,
-    "train_steps_per_second": 0.013
+    "train_loss": 1.4123068467597477e-05,
+    "train_runtime": 912.59,
+    "train_samples": 28,
+    "train_samples_per_second": 0.351,
+    "train_steps_per_second": 0.022
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.3555555555555556,
+  "epoch": 5.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 399.6875,
-      "epoch": 0.35555555555555557,
-      "grad_norm": 5.867144584655762,
+      "completion_length": 342.75,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 7.878459930419922,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 0.34115806699264795,
-      "reward_std": 0.8479255854908843,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": 0.0,
+      "reward": 0.6154589154757559,
+      "reward_std": 0.6660762839310337,
+      "rewards/concensus_correctness_reward_func": 0.02356250025331974,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3224080764921382,
+      "rewards/question_recreation_reward_func": 0.3011464198352769,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1062499996914994,
+      "rewards/xmlcount_reward_func": 0.10324999876320362,
       "step": 2
     },
     {
-      "completion_length": 411.25,
-      "epoch": 0.7111111111111111,
-      "grad_norm": 11.264700889587402,
-      "kl": 0.002446393948048353,
+      "completion_length": 336.0833333333333,
+      "epoch": 1.0,
+      "grad_norm": 7.0500383377075195,
+      "kl": 0.0016962609418745462,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.9096214124001563,
-      "reward_std": 1.0422866375884041,
-      "rewards/concensus_correctness_reward_func": 0.24900000169873238,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.4091525425513585,
+      "reward_std": 0.4993896021818121,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.24934017968189437,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3088608750452598,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16128124808892608,
+      "rewards/xmlcount_reward_func": 0.1002916693687439,
       "step": 4
     },
     {
-      "completion_length": 350.53846153846155,
-      "epoch": 1.0,
-      "grad_norm": 5.569336414337158,
-      "kl": 0.0010672689330441733,
+      "completion_length": 417.8125,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 6.578821659088135,
+      "kl": 0.001149166975665139,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.848451804369688,
-      "reward_std": 0.7304959391387036,
-      "rewards/concensus_correctness_reward_func": 0.13415384751099807,
-      "rewards/consensus_reward_func": 0.07692307692307693,
+      "reward": 0.25584341608919203,
+      "reward_std": 0.26554050942650065,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2843364061644444,
-      "rewards/soft_format_reward_func": 0.019230769230769232,
+      "rewards/question_recreation_reward_func": 0.23634341242723167,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.33380769336452853,
+      "rewards/xmlcount_reward_func": 0.019499999238178134,
       "step": 6
     },
     {
-      "completion_length": 329.59375,
-      "epoch": 1.3555555555555556,
-      "grad_norm": 8.78986930847168,
-      "kl": 0.0013028316243435256,
+      "completion_length": 275.5,
+      "epoch": 2.0,
+      "grad_norm": 68.87647247314453,
+      "kl": 0.037389953858413115,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8564973357133567,
-      "reward_std": 0.7181617806345457,
-      "rewards/concensus_correctness_reward_func": 0.12018750235438347,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.5189908021129668,
+      "reward_std": 0.4999783500873794,
+      "rewards/concensus_correctness_reward_func": 0.06299999852975209,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.27527859923429787,
+      "rewards/question_recreation_reward_func": 0.24640748760430142,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.33603124506771564,
+      "rewards/xmlcount_reward_func": 0.1262500062584877,
       "step": 8
     },
     {
-      "completion_length": 412.84375,
-      "epoch": 1.7111111111111112,
-      "grad_norm": 8.718709945678711,
-      "kl": 0.0015225716415443458,
+      "completion_length": 281.375,
+      "epoch": 2.571428571428571,
+      "grad_norm": 77.03890228271484,
+      "kl": 0.005192478878598195,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.9363826819462702,
-      "reward_std": 1.2138491035439074,
-      "rewards/concensus_correctness_reward_func": 0.23456250131130219,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.3187829144881107,
+      "reward_std": 0.3999681150307879,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3287576822331175,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.25787666445830837,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.0449375044554472,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.06090624816715717,
       "step": 10
     },
     {
-      "completion_length": 329.03846153846155,
-      "epoch": 2.0,
-      "grad_norm": 5.415107250213623,
-      "kl": 0.0015216450556181371,
+      "completion_length": 370.375,
+      "epoch": 3.0,
+      "grad_norm": 5.019116401672363,
+      "kl": 0.003112593602660733,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9755114456638694,
-      "reward_std": 1.0228999268191938,
-      "rewards/concensus_correctness_reward_func": 0.10446153925015376,
-      "rewards/consensus_reward_func": 0.15384615384615385,
+      "reward": 0.8822482178608576,
+      "reward_std": 0.9128863718360662,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.39270375231997323,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.4516648973027865,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.32449999566261584,
+      "rewards/xmlcount_reward_func": 0.2639166663090388,
       "step": 12
     },
     {
-      "completion_length": 453.9375,
-      "epoch": 2.3555555555555556,
-      "grad_norm": 6.7381486892700195,
-      "kl": 0.0014999880240793573,
+      "completion_length": 390.5,
+      "epoch": 3.571428571428571,
+      "grad_norm": 108.28805541992188,
+      "kl": 0.03872919351852033,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.4135700060869567,
-      "reward_std": 0.9733264712849632,
-      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.3723300788551569,
+      "reward_std": 0.5348989417543635,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.25222625577589497,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2545175752020441,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0862812465056777,
+      "rewards/xmlcount_reward_func": 0.055312504060566425,
       "step": 14
     },
     {
-      "completion_length": 368.4375,
-      "epoch": 2.7111111111111112,
-      "grad_norm": 5.401186466217041,
-      "kl": 0.0026795226440299302,
+      "completion_length": 387.0833333333333,
+      "epoch": 4.0,
+      "grad_norm": 2.8006181716918945,
+      "kl": 0.0044067831089099245,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.840499909594655,
-      "reward_std": 1.103903231152799,
-      "rewards/concensus_correctness_reward_func": 0.17475000023841858,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.3455103983481725,
+      "reward_std": 0.43331493251025677,
+      "rewards/concensus_correctness_reward_func": 0.03141666700442632,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2922186286887154,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.25059374623621505,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2329062558710575,
+      "rewards/xmlcount_reward_func": -0.01983333347986142,
       "step": 16
     },
     {
-      "completion_length": 453.7307692307692,
-      "epoch": 3.0,
-      "grad_norm": 9.587697982788086,
-      "kl": 0.002145178951860334,
+      "completion_length": 363.125,
+      "epoch": 4.571428571428571,
+      "grad_norm": 11.920393943786621,
+      "kl": 0.045275748503627256,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.7904227037842457,
-      "reward_std": 0.9643443526270298,
-      "rewards/concensus_correctness_reward_func": 0.14761538688953108,
-      "rewards/consensus_reward_func": 0.15384615384615385,
+      "reward": 0.4889937532134354,
+      "reward_std": 0.5009566913358867,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3931150149840575,
-      "rewards/soft_format_reward_func": 0.019230769230769232,
-      "rewards/strict_format_reward_func": 0.019230769230769232,
-      "rewards/xmlcount_reward_func": 0.057384609817885436,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.307775016175583,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.06878126203082502,
       "step": 18
     },
     {
-      "completion_length": 275.59375,
-      "epoch": 3.3555555555555556,
-      "grad_norm": 7.655992031097412,
-      "kl": 0.0033149754635815043,
+      "completion_length": 326.625,
+      "epoch": 5.0,
+      "grad_norm": 4.029944896697998,
+      "kl": 0.021408613664486136,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.2541076423949562,
-      "reward_std": 0.9791109445213806,
-      "rewards/concensus_correctness_reward_func": 0.17475000023841858,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.31329474101463956,
+      "reward_std": 0.5688736202816168,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4920764085254632,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.28441971881935996,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4622812569141388,
+      "rewards/xmlcount_reward_func": 0.02887499953309695,
       "step": 20
     },
     {
-      "epoch": 3.3555555555555556,
+      "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.6771256923675537e-06,
-      "train_runtime": 1570.9687,
-      "train_samples_per_second": 0.204,
-      "train_steps_per_second": 0.013
+      "train_loss": 1.4123068467597477e-05,
+      "train_runtime": 912.59,
+      "train_samples_per_second": 0.351,
+      "train_steps_per_second": 0.022
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 7,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {