End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.13018367738695816,
-    "train_runtime": 156.4524,
+    "train_loss": 4.218931023788173,
+    "train_runtime": 150.0721,
     "train_samples": 28,
-    "train_samples_per_second": 2.045,
-    "train_steps_per_second": 0.128
+    "train_samples_per_second": 2.132,
+    "train_steps_per_second": 0.133
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09902d935bd395c801fb71060b0337f6d4c7c0b18ee92806c4fe88710b1e6539
+oid sha256:6188060d6e2d0b5b56db43601cefcac377d580d4030d8fd26d3aa0f9cc9f0871
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.13018367738695816,
-    "train_runtime": 156.4524,
+    "train_loss": 4.218931023788173,
+    "train_runtime": 150.0721,
     "train_samples": 28,
-    "train_samples_per_second": 2.045,
-    "train_steps_per_second": 0.128
+    "train_samples_per_second": 2.132,
+    "train_steps_per_second": 0.133
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 293.4375,
+      "completion_length": 221.84375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 19.606327056884766,
+      "grad_norm": 34.56943893432617,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 2.809790687635541,
-      "reward_std": 0.7336142733256565,
-      "rewards/concensus_correctness_reward_func": 0.6699999943375587,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 3.9288951698690653,
+      "reward_std": 1.1124803191050887,
+      "rewards/concensus_correctness_reward_func": 1.294749990105629,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7068532090634108,
+      "rewards/question_recreation_reward_func": 0.7205826984718442,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.4641875000670552,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.8354375129565597,
       "step": 2
     },
     {
-      "completion_length": 154.33333333333334,
+      "completion_length": 180.04166666666666,
       "epoch": 1.0,
-      "grad_norm": 31385.81640625,
-      "kl": 988.6893634458538,
+      "grad_norm": 35.82617950439453,
+      "kl": 0.10452407443275054,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.7415,
-      "reward": 5.859939485788345,
-      "reward_std": 0.5496701752999797,
-      "rewards/concensus_correctness_reward_func": 1.8143333196640015,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 0.0001,
+      "reward": 6.0388092796007795,
+      "reward_std": 0.38356546566744026,
+      "rewards/concensus_correctness_reward_func": 1.939833328127861,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8880644589662552,
+      "rewards/question_recreation_reward_func": 0.9063508858283361,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.1158749957879384,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.1509583344062169,
       "step": 4
     },
     {
-      "completion_length": 170.40625,
+      "completion_length": 200.03125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 41.371925354003906,
-      "kl": 439.73223623854574,
+      "grad_norm": 26.39034080505371,
+      "kl": 2.5766645671101287,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.4397,
-      "reward": 5.452930800616741,
-      "reward_std": 0.863416745676659,
-      "rewards/concensus_correctness_reward_func": 1.5522499978542328,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0026,
+      "reward": 5.809181839227676,
+      "reward_std": 0.4371991759398952,
+      "rewards/concensus_correctness_reward_func": 1.7268749997019768,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9189307866618037,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9328693635761738,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.1380000114440918,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.1650625001639128,
       "step": 6
     },
     {
-      "completion_length": 209.33333333333334,
+      "completion_length": 182.83333333333334,
       "epoch": 2.0,
-      "grad_norm": 761.4453125,
-      "kl": 58.6377448773322,
+      "grad_norm": 25.702749252319336,
+      "kl": 54736.78004365539,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.044,
-      "reward": 5.8887307445208235,
-      "reward_std": 0.20041452534496784,
-      "rewards/concensus_correctness_reward_func": 1.608999987443288,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 41.0526,
+      "reward": 5.609723289807637,
+      "reward_std": 0.38764867421317223,
+      "rewards/concensus_correctness_reward_func": 1.5794999798138936,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.9412724177042643,
+      "rewards/question_recreation_reward_func": 0.9849732766548792,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3333333333333333,
-      "rewards/xmlcount_reward_func": 1.171791672706604,
+      "rewards/strict_format_reward_func": 0.2708333333333333,
+      "rewards/xmlcount_reward_func": 1.1910833368698757,
       "step": 8
     },
     {
-      "completion_length": 165.40625,
+      "completion_length": 207.09375,
       "epoch": 2.571428571428571,
-      "grad_norm": 212.7679443359375,
-      "kl": 50.77081588609144,
+      "grad_norm": 45.36337661743164,
+      "kl": 2.440667485818267,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0508,
-      "reward": 5.44428825378418,
-      "reward_std": 0.6909935071162181,
-      "rewards/concensus_correctness_reward_func": 1.563937496393919,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0024,
+      "reward": 5.608146898448467,
+      "reward_std": 0.3853573816741118,
+      "rewards/concensus_correctness_reward_func": 1.5514374934136868,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9163194932043552,
+      "rewards/question_recreation_reward_func": 0.9941156804561615,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1359062530100346,
+      "rewards/xmlcount_reward_func": 1.1719687543809414,
       "step": 10
     },
     {
-      "completion_length": 159.29166666666666,
+      "completion_length": 177.625,
       "epoch": 3.0,
-      "grad_norm": 50.51971435546875,
-      "kl": 2.4051825224111476,
+      "grad_norm": 4181.13427734375,
+      "kl": 214.9502143630137,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0018,
-      "reward": 6.1459734837214155,
-      "reward_std": 0.8073063851334155,
-      "rewards/concensus_correctness_reward_func": 1.7600833177566528,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 0.1612,
+      "reward": 5.501051505406697,
+      "reward_std": 0.4152323539601639,
+      "rewards/concensus_correctness_reward_func": 1.3809999922911327,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.9614734997351965,
+      "rewards/question_recreation_reward_func": 0.9169266273578008,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.195250004529953,
+      "rewards/xmlcount_reward_func": 1.140625,
       "step": 12
     },
     {
-      "completion_length": 205.71875,
+      "completion_length": 186.71875,
       "epoch": 3.571428571428571,
-      "grad_norm": 45.35451889038086,
-      "kl": 0.8525015097111464,
+      "grad_norm": 140.88648986816406,
+      "kl": 308.47393083851784,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0009,
-      "reward": 5.763527229428291,
-      "reward_std": 0.38918617014132906,
-      "rewards/concensus_correctness_reward_func": 1.6226874887943268,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 0.3085,
+      "reward": 6.149121820926666,
+      "reward_std": 1.6182817905792035,
+      "rewards/concensus_correctness_reward_func": 2.0923124998807907,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.961495965719223,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.9713093787431717,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1793437525629997,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.1323750019073486,
       "step": 14
     },
     {
-      "completion_length": 156.08333333333334,
+      "completion_length": 160.58333333333334,
       "epoch": 4.0,
-      "grad_norm": 37.47374725341797,
-      "kl": 12.36140370865663,
+      "grad_norm": 8.718925476074219,
+      "kl": 1.639391768258065,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0093,
-      "reward": 4.952808876832326,
-      "reward_std": 0.9242231525325527,
-      "rewards/concensus_correctness_reward_func": 1.3158333251873653,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0012,
+      "reward": 5.377435078223546,
+      "reward_std": 0.8237529636050264,
+      "rewards/concensus_correctness_reward_func": 1.4001666555802028,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8504755397637686,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.9555600434541702,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3541666666666667,
-      "rewards/xmlcount_reward_func": 1.1823333352804184,
+      "rewards/strict_format_reward_func": 0.3333333333333333,
+      "rewards/xmlcount_reward_func": 1.1883749961853027,
       "step": 16
     },
     {
-      "completion_length": 187.3125,
+      "completion_length": 189.34375,
       "epoch": 4.571428571428571,
-      "grad_norm": 196.08172607421875,
-      "kl": 13.175090974196792,
+      "grad_norm": 155.6063232421875,
+      "kl": 8.625189328100532,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0132,
-      "reward": 5.641463592648506,
-      "reward_std": 1.753083570394665,
-      "rewards/concensus_correctness_reward_func": 1.4763124845921993,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0086,
+      "reward": 5.694100014865398,
+      "reward_std": 0.40210114730143687,
+      "rewards/concensus_correctness_reward_func": 1.6176249831914902,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9286198541522026,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9089125599712133,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1115312539041042,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0894374996423721,
       "step": 18
     },
     {
-      "completion_length": 162.95833333333334,
+      "completion_length": 172.16666666666666,
       "epoch": 5.0,
-      "grad_norm": 17.197851181030273,
-      "kl": 0.9747528317384422,
+      "grad_norm": 57570.94140625,
+      "kl": 869.4708892249813,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0007,
-      "reward": 4.92093700170517,
-      "reward_std": 0.4740578230509224,
-      "rewards/concensus_correctness_reward_func": 1.1977500021457672,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.6521,
+      "reward": 4.955635358889897,
+      "reward_std": 0.4001488390301044,
+      "rewards/concensus_correctness_reward_func": 1.3370000024636586,
+      "rewards/consensus_reward_func": 1.0833333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9836036910613378,
+      "rewards/question_recreation_reward_func": 0.9936353713274002,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1979166666666667,
+      "rewards/strict_format_reward_func": 0.3541666666666667,
+      "rewards/xmlcount_reward_func": 1.1875,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.13018367738695816,
-      "train_runtime": 156.4524,
-      "train_samples_per_second": 2.045,
-      "train_steps_per_second": 0.128
+      "train_loss": 4.218931023788173,
+      "train_runtime": 150.0721,
+      "train_samples_per_second": 2.132,
+      "train_steps_per_second": 0.133
     }
   ],
   "logging_steps": 2,