End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.9015867059788436e-06,
-    "train_runtime": 305.0961,
+    "train_loss": 5.147978663444519e-06,
+    "train_runtime": 321.291,
     "train_samples": 28,
-    "train_samples_per_second": 1.049,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 0.996,
+    "train_steps_per_second": 0.062
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:304816fb30c57f3ac45d3548d29b983f155d35d9c57a59eab9c3c862d5d64c56
+oid sha256:f90acace1a001e94571bd8a5f19370ebf128457a0a6188be7d96a2d1cca1aa75
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.9015867059788436e-06,
-    "train_runtime": 305.0961,
+    "train_loss": 5.147978663444519e-06,
+    "train_runtime": 321.291,
     "train_samples": 28,
-    "train_samples_per_second": 1.049,
-    "train_steps_per_second": 0.066
+    "train_samples_per_second": 0.996,
+    "train_steps_per_second": 0.062
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 352.65625,
+      "completion_length": 342.96875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 4.6517510414123535,
+      "grad_norm": 5.793670654296875,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 1.9860531073063612,
-      "reward_std": 1.5548657272011042,
-      "rewards/concensus_correctness_reward_func": 0.29881250113248825,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": -0.0,
+      "reward": 2.103108564391732,
+      "reward_std": 1.210430834442377,
+      "rewards/concensus_correctness_reward_func": 0.31837500631809235,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6224281177856028,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.538921034662053,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5648125018924475,
+      "rewards/xmlcount_reward_func": 0.6833125085104257,
       "step": 2
     },
     {
-      "completion_length": 332.875,
+      "completion_length": 395.7083333333333,
       "epoch": 1.0,
-      "grad_norm": 2.9446356296539307,
-      "kl": 0.0012306679078998666,
+      "grad_norm": 3.316585063934326,
+      "kl": 0.001008606399409473,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.2811362047990165,
-      "reward_std": 1.5899524418637156,
-      "rewards/concensus_correctness_reward_func": 0.4385000020265579,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 1.8855041489005089,
+      "reward_std": 1.0255667748861015,
+      "rewards/concensus_correctness_reward_func": 0.22416667143503824,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.5530528823534647,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.6617124912639459,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7062499970197678,
+      "rewards/xmlcount_reward_func": 0.41629168142875034,
       "step": 4
     },
     {
-      "completion_length": 440.5,
+      "completion_length": 314.9375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 6.86065149307251,
-      "kl": 0.0020645546283049043,
+      "grad_norm": 9.464887619018555,
+      "kl": 0.0030845321743981913,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.1326183322817087,
-      "reward_std": 1.1055441808421165,
-      "rewards/concensus_correctness_reward_func": 0.10881249979138374,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 2.451855756342411,
+      "reward_std": 1.917446899227798,
+      "rewards/concensus_correctness_reward_func": 0.5015000030398369,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5236495696008205,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.616605784278363,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.31265624798834324,
+      "rewards/xmlcount_reward_func": 0.6462500058114529,
       "step": 6
     },
     {
-      "completion_length": 393.625,
+      "completion_length": 364.4583333333333,
       "epoch": 2.0,
-      "grad_norm": 2.8339314460754395,
-      "kl": 0.004773770022438839,
+      "grad_norm": 3.3119776248931885,
+      "kl": 0.004234734262960653,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.5451089590787888,
-      "reward_std": 1.4853824774424236,
-      "rewards/concensus_correctness_reward_func": 0.5268333355585734,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 1.529069220026334,
+      "reward_std": 1.0711190002039075,
+      "rewards/concensus_correctness_reward_func": 0.17208333810170492,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5557756368070841,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7534442593653997,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5458333169420561,
+      "rewards/xmlcount_reward_func": 0.35354167968034744,
       "step": 8
     },
     {
-      "completion_length": 363.90625,
+      "completion_length": 305.3125,
       "epoch": 2.571428571428571,
-      "grad_norm": 6.822569847106934,
-      "kl": 0.0058538312441669405,
+      "grad_norm": 8.920665740966797,
+      "kl": 0.006920879794051871,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.7394325397908688,
-      "reward_std": 0.8727849050483201,
-      "rewards/concensus_correctness_reward_func": 0.1463750023394823,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.1907591819763184,
+      "reward_std": 0.7096769977360964,
+      "rewards/concensus_correctness_reward_func": 0.2839374989271164,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6529012635583058,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6164779150858521,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7526562623679638,
+      "rewards/xmlcount_reward_func": 0.7278437400236726,
       "step": 10
     },
     {
-      "completion_length": 447.375,
+      "completion_length": 272.625,
       "epoch": 3.0,
-      "grad_norm": 2.446692943572998,
-      "kl": 0.004516709595918655,
+      "grad_norm": 3.3465256690979004,
+      "kl": 0.0063869420264381915,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.8546138207117715,
-      "reward_std": 1.2566993081321318,
-      "rewards/concensus_correctness_reward_func": 0.4098333418369293,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.122172156969706,
+      "reward_std": 0.5801232105586678,
+      "rewards/concensus_correctness_reward_func": 0.2299166719118754,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.7225304792324702,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.6718804923196634,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8055833478768667,
+      "rewards/xmlcount_reward_func": 0.720374991496404,
       "step": 12
     },
     {
-      "completion_length": 343.59375,
+      "completion_length": 354.84375,
       "epoch": 3.571428571428571,
-      "grad_norm": 4.266261100769043,
-      "kl": 0.006598696825676598,
+      "grad_norm": 5.003468036651611,
+      "kl": 0.016760691243689507,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.571171674877405,
-      "reward_std": 0.9833136084489524,
-      "rewards/concensus_correctness_reward_func": 0.5100625045597553,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 1.8695098906755447,
+      "reward_std": 1.2882013304333668,
+      "rewards/concensus_correctness_reward_func": 0.2758750058710575,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.6207654234021902,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5972911030985415,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5653437469154596,
+      "rewards/xmlcount_reward_func": 0.4963437505066395,
       "step": 14
     },
     {
-      "completion_length": 429.5833333333333,
+      "completion_length": 503.0,
       "epoch": 4.0,
-      "grad_norm": 3.0463383197784424,
-      "kl": 0.0061233706849937635,
+      "grad_norm": 4.424561023712158,
+      "kl": 0.0056549445337926345,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.9036124100287755,
-      "reward_std": 1.4660836717424293,
-      "rewards/concensus_correctness_reward_func": 0.38425000322361785,
+      "reward": 1.4202980945507686,
+      "reward_std": 1.4158651794617374,
+      "rewards/concensus_correctness_reward_func": 0.17708333333333334,
       "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6183623724306623,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.5948813930153847,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4843333313862483,
+      "rewards/xmlcount_reward_func": 0.06499999016523361,
       "step": 16
     },
     {
-      "completion_length": 392.375,
+      "completion_length": 401.0,
       "epoch": 4.571428571428571,
-      "grad_norm": 5.570501804351807,
-      "kl": 0.0060581645084312186,
+      "grad_norm": 5.2076849937438965,
+      "kl": 0.007878506650740746,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.6768330512568355,
-      "reward_std": 1.189737519598566,
-      "rewards/concensus_correctness_reward_func": 0.6989375054836273,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 2.6692092455923557,
+      "reward_std": 0.7308505216351477,
+      "rewards/concensus_correctness_reward_func": 0.48237501084804535,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5462080566212535,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6628029751591384,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.556687492877245,
+      "rewards/xmlcount_reward_func": 0.7740312479436398,
       "step": 18
     },
     {
-      "completion_length": 399.125,
+      "completion_length": 262.9166666666667,
       "epoch": 5.0,
-      "grad_norm": 2.896327018737793,
-      "kl": 0.007528022746555507,
+      "grad_norm": 3.2780354022979736,
+      "kl": 0.008317624412787458,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.009764472643534,
-      "reward_std": 0.8362502666811148,
-      "rewards/concensus_correctness_reward_func": 0.2730833323051532,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 2.023281673590342,
+      "reward_std": 0.8200806056459745,
+      "rewards/concensus_correctness_reward_func": 0.09125000238418579,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.7085560858249664,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6847400286545356,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5281249980131785,
+      "rewards/xmlcount_reward_func": 0.9139583359162012,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.9015867059788436e-06,
-      "train_runtime": 305.0961,
-      "train_samples_per_second": 1.049,
-      "train_steps_per_second": 0.066
+      "train_loss": 5.147978663444519e-06,
+      "train_runtime": 321.291,
+      "train_samples_per_second": 0.996,
+      "train_steps_per_second": 0.062
     }
   ],
   "logging_steps": 2,