End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.0804759515158366e-06,
-    "train_runtime": 995.493,
+    "train_loss": 4.450418418855406e-06,
+    "train_runtime": 738.8047,
     "train_samples": 28,
-    "train_samples_per_second": 0.321,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.433,
+    "train_steps_per_second": 0.027
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0403b82b383c121d10fb846c29821fb18eb4fb5ace741198cd7540cfbf30a0bb
+oid sha256:a71bfd5d7a375609d49a5ba815e7754adfaa9e8208a5701ed4ce6980614bb254
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.0804759515158366e-06,
-    "train_runtime": 995.493,
+    "train_loss": 4.450418418855406e-06,
+    "train_runtime": 738.8047,
     "train_samples": 28,
-    "train_samples_per_second": 0.321,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.433,
+    "train_steps_per_second": 0.027
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 413.875,
+      "completion_length": 301.21875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 4.378029823303223,
+      "grad_norm": 5.458982944488525,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 1.2516802661120892,
-      "reward_std": 0.7530619776807725,
-      "rewards/concensus_correctness_reward_func": 0.024312499910593033,
+      "loss": 0.0,
+      "reward": 0.8577872766181827,
+      "reward_std": 0.46707520788913826,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6293677696958184,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.39259978220798075,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4104999899864197,
+      "rewards/xmlcount_reward_func": 0.3792500018607825,
       "step": 2
     },
     {
-      "completion_length": 291.25,
+      "completion_length": 325.5416666666667,
       "epoch": 1.0,
-      "grad_norm": 3.2720723152160645,
-      "kl": 0.0011097663567246248,
+      "grad_norm": 3.2459182739257812,
+      "kl": 0.0009404258089489304,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.5765880445639293,
-      "reward_std": 1.161915241740644,
-      "rewards/concensus_correctness_reward_func": 0.2580833335717519,
+      "reward": 2.266234209140142,
+      "reward_std": 0.8061389351884524,
+      "rewards/concensus_correctness_reward_func": 0.2762499948342641,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.4780464159945647,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5760675361379981,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5071250026424726,
+      "rewards/xmlcount_reward_func": 0.6639166685442129,
       "step": 4
     },
     {
-      "completion_length": 282.34375,
+      "completion_length": 407.34375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 4.900054454803467,
-      "kl": 0.00138632424568641,
+      "grad_norm": 4.7337965965271,
+      "kl": 0.001618697562662419,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.6931456290185452,
-      "reward_std": 0.8773337011225522,
-      "rewards/concensus_correctness_reward_func": 0.2776250019669533,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.2271797712892294,
+      "reward_std": 0.874489885813091,
+      "rewards/concensus_correctness_reward_func": 0.0599374994635582,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5367393875494599,
+      "rewards/question_recreation_reward_func": 0.5180235160514712,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.566281258885283,
+      "rewards/xmlcount_reward_func": 0.39921875204890966,
       "step": 6
     },
     {
-      "completion_length": 311.1666666666667,
+      "completion_length": 400.25,
       "epoch": 2.0,
-      "grad_norm": 2.8332316875457764,
-      "kl": 0.001555235983687453,
+      "grad_norm": 3.1473844051361084,
+      "kl": 0.0021925697510596365,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.434648111462593,
-      "reward_std": 0.8368850518018007,
-      "rewards/concensus_correctness_reward_func": 0.15258333086967468,
+      "reward": 1.2754728893438976,
+      "reward_std": 1.3214016997565825,
+      "rewards/concensus_correctness_reward_func": 0.1516666660706202,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5519814131160578,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4740562033839524,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.48008333270748454,
+      "rewards/xmlcount_reward_func": 0.1497499942779541,
       "step": 8
     },
     {
-      "completion_length": 332.78125,
+      "completion_length": 281.25,
       "epoch": 2.571428571428571,
-      "grad_norm": 8.892011642456055,
-      "kl": 0.0025279877954744734,
+      "grad_norm": 4.605147838592529,
+      "kl": 0.005273919930914417,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.1487721176818013,
-      "reward_std": 1.5603560641975491,
-      "rewards/concensus_correctness_reward_func": 0.8391250036656857,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.8144206423312426,
+      "reward_std": 0.9190629746881314,
+      "rewards/concensus_correctness_reward_func": 0.1829374972730875,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4981784080155194,
+      "rewards/question_recreation_reward_func": 0.5419206258375198,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.43646875710692257,
+      "rewards/xmlcount_reward_func": 0.6520624924451113,
       "step": 10
     },
     {
-      "completion_length": 433.125,
+      "completion_length": 249.16666666666666,
       "epoch": 3.0,
-      "grad_norm": 2.816394329071045,
-      "kl": 0.0021395664516603574,
+      "grad_norm": 3.6658992767333984,
+      "kl": 0.006474080708964418,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.3233340829610825,
-      "reward_std": 1.4634119508167107,
-      "rewards/concensus_correctness_reward_func": 0.23983333011468252,
+      "reward": 2.2421703040599823,
+      "reward_std": 0.6761051618183652,
+      "rewards/concensus_correctness_reward_func": 0.4072500020265579,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6403757482767105,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6540869710346063,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19312502443790436,
+      "rewards/xmlcount_reward_func": 0.6808333434164524,
       "step": 12
     },
     {
-      "completion_length": 386.6875,
+      "completion_length": 318.09375,
       "epoch": 3.571428571428571,
-      "grad_norm": 4.575733184814453,
-      "kl": 0.003652094746939838,
+      "grad_norm": 3.974397659301758,
+      "kl": 0.005749276504502632,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.2941738106310368,
-      "reward_std": 1.153965707984753,
-      "rewards/concensus_correctness_reward_func": 0.08868750184774399,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.4769462142139673,
+      "reward_std": 1.422497031337116,
+      "rewards/concensus_correctness_reward_func": 0.7006249986588955,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7049238011240959,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5655399393290281,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1880624971818179,
+      "rewards/xmlcount_reward_func": 0.6482812603935599,
       "step": 14
     },
     {
-      "completion_length": 381.0833333333333,
+      "completion_length": 269.2083333333333,
       "epoch": 4.0,
-      "grad_norm": 1.8834401369094849,
-      "kl": 0.0027937811391893774,
+      "grad_norm": 3.439955949783325,
+      "kl": 0.004462281513648729,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.3959101090828578,
-      "reward_std": 0.6646982571110129,
-      "rewards/concensus_correctness_reward_func": 0.0625,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.0723049951096375,
+      "reward_std": 0.4619913004959623,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.6112434626556933,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.4110549579684933,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5555000041301051,
+      "rewards/xmlcount_reward_func": 0.49458332856496173,
       "step": 16
     },
     {
-      "completion_length": 389.96875,
+      "completion_length": 338.09375,
       "epoch": 4.571428571428571,
-      "grad_norm": 4.4097514152526855,
-      "kl": 0.0038497644927701913,
+      "grad_norm": 4.146206855773926,
+      "kl": 0.014700129351695068,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.3296699151396751,
-      "reward_std": 0.6159828560194001,
-      "rewards/concensus_correctness_reward_func": 0.05287500098347664,
+      "reward": 1.4003828410059214,
+      "reward_std": 0.6882670563645661,
+      "rewards/concensus_correctness_reward_func": 0.055937498807907104,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6360449412604794,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5066953104687855,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5782499890774488,
+      "rewards/xmlcount_reward_func": 0.4627500004135072,
       "step": 18
     },
     {
-      "completion_length": 292.125,
+      "completion_length": 311.4166666666667,
       "epoch": 5.0,
-      "grad_norm": 3.710397243499756,
-      "kl": 0.004927143124708285,
+      "grad_norm": 3.3017971515655518,
+      "kl": 0.006459277220225583,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.3796936435004075,
-      "reward_std": 0.7642435291782022,
-      "rewards/concensus_correctness_reward_func": 0.08008333543936412,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 1.3662489453951518,
+      "reward_std": 1.030937065758432,
+      "rewards/concensus_correctness_reward_func": 0.13033332924048105,
+      "rewards/consensus_reward_func": 0.16666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.44686032521228,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5561239557961623,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5194166600704193,
+      "rewards/xmlcount_reward_func": 0.5131250048677126,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.0804759515158366e-06,
-      "train_runtime": 995.493,
-      "train_samples_per_second": 0.321,
-      "train_steps_per_second": 0.02
+      "train_loss": 4.450418418855406e-06,
+      "train_runtime": 738.8047,
+      "train_samples_per_second": 0.433,
+      "train_steps_per_second": 0.027
     }
   ],
   "logging_steps": 2,