End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.868829131126404e-05,
-    "train_runtime": 292.9301,
-    "train_samples": 28,
-    "train_samples_per_second": 1.092,
-    "train_steps_per_second": 0.068
+    "train_loss": 3.0076131224632264e-06,
+    "train_runtime": 1232.7412,
+    "train_samples": 31,
+    "train_samples_per_second": 0.26,
+    "train_steps_per_second": 0.016
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0b859de05c5e6b1f81d38ac20fc7adb82668df5c56f3e88cf24c08491b3e5d5
+oid sha256:216db39a492b41e9e91928b6d776a3bd555716e77fb2256c2225b40e54329ac9
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.868829131126404e-05,
-    "train_runtime": 292.9301,
-    "train_samples": 28,
-    "train_samples_per_second": 1.092,
-    "train_steps_per_second": 0.068
+    "train_loss": 3.0076131224632264e-06,
+    "train_runtime": 1232.7412,
+    "train_samples": 31,
+    "train_samples_per_second": 0.26,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 391.0625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 34.6677131652832,
+      "completion_length": 424.75,
+      "epoch": 0.5161290322580645,
+      "grad_norm": 8.77116584777832,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.586152283474803,
-      "reward_std": 0.7006768652936444,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.4542556337546557,
+      "reward_std": 0.6248648471082561,
+      "rewards/concensus_correctness_reward_func": 0.09756249934434891,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.354964779689908,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.1675993852550164,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1686875019222498,
+      "rewards/xmlcount_reward_func": 0.12659374484792352,
       "step": 2
     },
     {
-      "completion_length": 346.2916666666667,
+      "completion_length": 206.83333333333334,
       "epoch": 1.0,
-      "grad_norm": 30.92546272277832,
-      "kl": 0.00109227391173287,
+      "grad_norm": 13.20207405090332,
+      "kl": 0.0012788242932098607,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.3667026895175998,
-      "reward_std": 0.4101825385975341,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.4601796910787622,
+      "reward_std": 0.3873041265954574,
+      "rewards/concensus_correctness_reward_func": 0.09900000095367431,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.27432767119413864,
+      "rewards/final_correctness_reward_func": 0.06666666666666667,
+      "rewards/question_recreation_reward_func": 0.20367967908581097,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09237500295663874,
+      "rewards/xmlcount_reward_func": 0.0908333328862985,
       "step": 4
     },
     {
-      "completion_length": 379.96875,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 13.158846855163574,
-      "kl": 0.003917330877811764,
+      "completion_length": 298.875,
+      "epoch": 1.5161290322580645,
+      "grad_norm": 16.596080780029297,
+      "kl": 0.001379330658892286,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.31149076228030026,
-      "reward_std": 0.45825456036254764,
+      "reward": 0.4755393145605922,
+      "reward_std": 0.55023356952006,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3642720205243677,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.264914311846951,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0527812484651804,
+      "rewards/xmlcount_reward_func": 0.08562499703839421,
       "step": 6
     },
     {
-      "completion_length": 365.0833333333333,
+      "completion_length": 344.1333333333333,
       "epoch": 2.0,
-      "grad_norm": 3.8675999641418457,
-      "kl": 0.0058151181001449,
+      "grad_norm": 16.715166091918945,
+      "kl": 0.0017653932329267264,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.0624708766117692,
-      "reward_std": 0.813691898365505,
+      "reward": 0.4242014608035485,
+      "reward_std": 0.37727057877928016,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.21534589860433093,
+      "rewards/final_correctness_reward_func": 0.06666666666666667,
+      "rewards/question_recreation_reward_func": 0.1570014638826251,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1528750086824099,
+      "rewards/xmlcount_reward_func": 0.20053333044052124,
       "step": 8
     },
     {
-      "completion_length": 302.9375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 126.3558349609375,
-      "kl": 0.023398748788167723,
+      "completion_length": 305.75,
+      "epoch": 2.5161290322580645,
+      "grad_norm": 15.140896797180176,
+      "kl": 0.002574505087977741,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.14244792109820992,
-      "reward_std": 0.5862200064875651,
+      "reward": 0.5482971895253286,
+      "reward_std": 0.4541558200144209,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2068229204742238,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.19789095013402402,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06437499518506229,
+      "rewards/xmlcount_reward_func": 0.1629062481224537,
       "step": 10
     },
     {
-      "completion_length": 415.0833333333333,
+      "completion_length": 405.9,
       "epoch": 3.0,
-      "grad_norm": 2.95449161529541,
-      "kl": 0.0026224162381064766,
+      "grad_norm": 14.037881851196289,
+      "kl": 0.004018065452692099,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.2249434565504392,
-      "reward_std": 0.304003962315619,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.9634811220069727,
+      "reward_std": 0.5718606707329551,
+      "rewards/concensus_correctness_reward_func": 0.1040666659673055,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3470267872326076,
+      "rewards/final_correctness_reward_func": 0.13333333333333333,
+      "rewards/question_recreation_reward_func": 0.3595478010674318,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1220833311478297,
+      "rewards/xmlcount_reward_func": 0.36653333008289335,
       "step": 12
     },
     {
-      "completion_length": 420.625,
-      "epoch": 3.571428571428571,
-      "grad_norm": 6.17542028427124,
-      "kl": 0.01191169011144666,
+      "completion_length": 335.46875,
+      "epoch": 3.5161290322580645,
+      "grad_norm": 23.11191749572754,
+      "kl": 0.0027655707817757502,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.3435917226597667,
-      "reward_std": 0.6618271541155991,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.9505715684499592,
+      "reward_std": 0.5874458824982867,
+      "rewards/concensus_correctness_reward_func": 0.12018750235438347,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.25265423604287207,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3109778331127018,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02843749849125743,
+      "rewards/xmlcount_reward_func": 0.39440625021234155,
       "step": 14
     },
     {
-      "completion_length": 370.5416666666667,
+      "completion_length": 302.3,
       "epoch": 4.0,
-      "grad_norm": 2.7796130180358887,
-      "kl": 0.00765374192269519,
+      "grad_norm": 7.85057258605957,
+      "kl": 0.004924081599650284,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.7821583213905493,
-      "reward_std": 0.5152762839570642,
-      "rewards/concensus_correctness_reward_func": 0.025416667262713116,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 0.43081799112260344,
+      "reward_std": 0.5724140242363016,
+      "rewards/concensus_correctness_reward_func": 0.020666666825612388,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.42182498921950656,
+      "rewards/final_correctness_reward_func": 0.06666666666666667,
+      "rewards/question_recreation_reward_func": 0.22335132583975792,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16824999699989954,
+      "rewards/xmlcount_reward_func": 0.12013333241144816,
       "step": 16
     },
     {
-      "completion_length": 353.8125,
-      "epoch": 4.571428571428571,
-      "grad_norm": 60.58961868286133,
-      "kl": 0.12441448158642743,
+      "completion_length": 281.4375,
+      "epoch": 4.516129032258064,
+      "grad_norm": 17.545211791992188,
+      "kl": 0.003997791551228147,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0001,
-      "reward": 0.4139303692500107,
-      "reward_std": 0.47191220574313775,
+      "loss": 0.0,
+      "reward": 0.5256107244640589,
+      "reward_std": 0.6561538505629869,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3153991232975386,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.19464197289198637,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0985312545672059,
+      "rewards/xmlcount_reward_func": 0.2059687429573387,
       "step": 18
     },
     {
-      "completion_length": 449.9166666666667,
+      "completion_length": 322.43333333333334,
       "epoch": 5.0,
-      "grad_norm": 6.070877552032471,
-      "kl": 0.014101767272222787,
+      "grad_norm": 12.730913162231445,
+      "kl": 0.008367715034789095,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.4589709974825382,
-      "reward_std": 0.4924936294555664,
+      "reward": 0.5111197280387084,
+      "reward_std": 0.6876047144333521,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.06666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.24751267271737257,
+      "rewards/final_correctness_reward_func": 0.13333333333333333,
+      "rewards/question_recreation_reward_func": 0.19185306827227275,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2114583309739828,
+      "rewards/xmlcount_reward_func": 0.11926666597525279,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.868829131126404e-05,
-      "train_runtime": 292.9301,
-      "train_samples_per_second": 1.092,
-      "train_steps_per_second": 0.068
+      "train_loss": 3.0076131224632264e-06,
+      "train_runtime": 1232.7412,
+      "train_samples_per_second": 0.26,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68a334da34334ee9f3ad32fb53be8bc118158f3c7dc8fc3ef460aeb8a0cb803c
+oid sha256:1e43aa6fe32facf9e7aaa21f4a7707ea5472944f1653e3413312342178d2b2bd
 size 5880