End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5ac1c82aaa66c513fd77814dbee1cf88eb092c2bcef2c9de69f3f8af333eb621
+oid sha256:06a74e6e2d014916f90ee690dce3c50c22bdd08d43b69654327d11f78b7067b2
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0266644716903102e-05,
-    "train_runtime": 1124.2903,
-    "train_samples": 14,
-    "train_samples_per_second": 0.285,
-    "train_steps_per_second": 0.018
+    "train_loss": 3.4546391179901545e-05,
+    "train_runtime": 941.4145,
+    "train_samples": 15,
+    "train_samples_per_second": 0.34,
+    "train_steps_per_second": 0.021
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0266644716903102e-05,
-    "train_runtime": 1124.2903,
-    "train_samples": 14,
-    "train_samples_per_second": 0.285,
-    "train_steps_per_second": 0.018
+    "train_loss": 3.4546391179901545e-05,
+    "train_runtime": 941.4145,
+    "train_samples": 15,
+    "train_samples_per_second": 0.34,
+    "train_steps_per_second": 0.021
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 229.0,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 1.1000111103057861,
-      "kl": 0.030057710246182978,
+      "completion_length": 198.40625,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 8.360007286071777,
+      "kl": 0.03296396473888308,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 5.323144108057022,
-      "reward_std": 2.5642264559865,
-      "rewards/concensus_correctness_reward_func": 1.693624995648861,
-      "rewards/consensus_reward_func": 1.125,
+      "reward": 6.321916818618774,
+      "reward_std": 1.7188230901956558,
+      "rewards/concensus_correctness_reward_func": 2.3461874574422836,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.49539405573159456,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.7122499942779541,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.48694803565740585,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.9887812659144402,
       "step": 2
     },
     {
-      "completion_length": 220.125,
+      "completion_length": 228.32142857142858,
       "epoch": 1.0,
-      "grad_norm": 0.7904133796691895,
-      "kl": 0.032162196934223175,
+      "grad_norm": 0.6417061686515808,
+      "kl": 0.029710494380976473,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 5.786092758178711,
-      "reward_std": 2.145986179510752,
-      "rewards/concensus_correctness_reward_func": 1.8214166661103566,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "reward": 5.377460071018764,
+      "reward_std": 1.0338971018791199,
+      "rewards/concensus_correctness_reward_func": 1.8204285672732763,
+      "rewards/consensus_reward_func": 1.3571428571428572,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.5735511605938276,
+      "rewards/final_correctness_reward_func": 0.7857142857142857,
+      "rewards/question_recreation_reward_func": 0.5691385694912502,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.8494583268960317,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.8093214460781643,
       "step": 4
     },
     {
-      "completion_length": 222.65625,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 1.0802266597747803,
-      "kl": 0.03376023273449391,
+      "completion_length": 184.96875,
+      "epoch": 1.5333333333333332,
+      "grad_norm": 1.40841543674469,
+      "kl": 0.04421051638200879,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 4.889157712459564,
-      "reward_std": 2.493479371070862,
-      "rewards/concensus_correctness_reward_func": 1.5853749886155128,
-      "rewards/consensus_reward_func": 1.125,
+      "reward": 5.1417646408081055,
+      "reward_std": 2.2220220267772675,
+      "rewards/concensus_correctness_reward_func": 1.5205624923110008,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.41631394904106855,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.4983896389603615,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.746843758970499,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 1.060312531888485,
       "step": 6
     },
     {
-      "completion_length": 214.08333333333334,
+      "completion_length": 243.64285714285714,
       "epoch": 2.0,
-      "grad_norm": 0.48023688793182373,
-      "kl": 0.05418917319426934,
+      "grad_norm": 0.5562111735343933,
+      "kl": 0.02644094345825059,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 5.540527900060018,
-      "reward_std": 2.529421851038933,
-      "rewards/concensus_correctness_reward_func": 1.573499987522761,
-      "rewards/consensus_reward_func": 0.9166666666666666,
+      "reward": 5.550810728754316,
+      "reward_std": 1.8111653668539864,
+      "rewards/concensus_correctness_reward_func": 1.8958571127482824,
+      "rewards/consensus_reward_func": 1.2142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.75,
-      "rewards/question_recreation_reward_func": 0.455111101269722,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.8244166771570841,
+      "rewards/final_correctness_reward_func": 1.0714285714285714,
+      "rewards/question_recreation_reward_func": 0.5573822898524148,
+      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.7404285839625767,
       "step": 8
     },
     {
-      "completion_length": 232.0625,
-      "epoch": 2.571428571428571,
-      "grad_norm": 1.2091493606567383,
-      "kl": 0.03521083341911435,
+      "completion_length": 186.96875,
+      "epoch": 2.533333333333333,
+      "grad_norm": 0.8356194496154785,
+      "kl": 0.03581268759444356,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 6.261362463235855,
-      "reward_std": 1.929440539330244,
-      "rewards/concensus_correctness_reward_func": 2.2173124700784683,
-      "rewards/consensus_reward_func": 1.375,
+      "reward": 4.26269268989563,
+      "reward_std": 1.9406834915280342,
+      "rewards/concensus_correctness_reward_func": 1.3394375033676624,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5,
-      "rewards/question_recreation_reward_func": 0.45192492194473743,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7015000097453594,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5125052509829402,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.8638750165700912,
       "step": 10
     },
     {
-      "completion_length": 213.33333333333334,
+      "completion_length": 207.5,
       "epoch": 3.0,
-      "grad_norm": 1.8901488780975342,
-      "kl": 0.028847349652399618,
+      "grad_norm": 0.9257772564888,
+      "kl": 0.038460450513022285,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 5.242655277252197,
-      "reward_std": 2.291464234391848,
-      "rewards/concensus_correctness_reward_func": 1.4964999854564667,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "reward": 6.022549152374268,
+      "reward_std": 2.5043100884982517,
+      "rewards/concensus_correctness_reward_func": 2.333285689353943,
+      "rewards/consensus_reward_func": 1.2857142857142858,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.0,
-      "rewards/question_recreation_reward_func": 0.54886361459891,
+      "rewards/final_correctness_reward_func": 0.7857142857142857,
+      "rewards/question_recreation_reward_func": 0.4640133615050997,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.9681249956289927,
+      "rewards/strict_format_reward_func": 0.14285714285714285,
+      "rewards/xmlcount_reward_func": 1.0109642914363317,
       "step": 12
     },
     {
-      "completion_length": 205.09375,
-      "epoch": 3.571428571428571,
-      "grad_norm": 1.1510249376296997,
-      "kl": 0.03378852782770991,
+      "completion_length": 193.4375,
+      "epoch": 3.533333333333333,
+      "grad_norm": 0.8213157057762146,
+      "kl": 0.046339198597706854,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 6.820627868175507,
-      "reward_std": 2.49598478525877,
-      "rewards/concensus_correctness_reward_func": 2.3831249997019768,
-      "rewards/consensus_reward_func": 1.4375,
+      "reward": 4.843554437160492,
+      "reward_std": 1.867899501696229,
+      "rewards/concensus_correctness_reward_func": 1.601437471807003,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.5625,
-      "rewards/question_recreation_reward_func": 0.4795652935281396,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.4974917620420456,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.9266875386238098,
+      "rewards/strict_format_reward_func": 0.09375,
+      "rewards/xmlcount_reward_func": 0.9008749946951866,
       "step": 14
     },
     {
-      "completion_length": 222.29166666666666,
+      "completion_length": 214.35714285714286,
       "epoch": 4.0,
-      "grad_norm": 0.42290571331977844,
-      "kl": 0.032701820290337004,
+      "grad_norm": 0.7088996171951294,
+      "kl": 0.04368409647473267,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.584882616996765,
-      "reward_std": 1.665024643143018,
-      "rewards/concensus_correctness_reward_func": 1.4733333190282185,
-      "rewards/consensus_reward_func": 1.0833333333333333,
+      "reward": 5.192878893443516,
+      "reward_std": 2.4513211250305176,
+      "rewards/concensus_correctness_reward_func": 1.527428584439414,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8333333333333334,
-      "rewards/question_recreation_reward_func": 0.5142994125684103,
+      "rewards/final_correctness_reward_func": 1.1428571428571428,
+      "rewards/question_recreation_reward_func": 0.4679503919822829,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.041666666666666664,
-      "rewards/xmlcount_reward_func": 0.6389166712760925,
+      "rewards/strict_format_reward_func": 0.10714285714285714,
+      "rewards/xmlcount_reward_func": 0.9475000245230538,
       "step": 16
     },
     {
-      "completion_length": 253.375,
-      "epoch": 4.571428571428571,
-      "grad_norm": 1.0985242128372192,
-      "kl": 0.03926029335707426,
+      "completion_length": 200.65625,
+      "epoch": 4.533333333333333,
+      "grad_norm": 0.7495886087417603,
+      "kl": 0.027507278718985617,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.272556304931641,
-      "reward_std": 2.2469240203499794,
-      "rewards/concensus_correctness_reward_func": 2.0731874853372574,
-      "rewards/consensus_reward_func": 1.3125,
+      "reward": 6.123600482940674,
+      "reward_std": 1.2783842608332634,
+      "rewards/concensus_correctness_reward_func": 2.145937453955412,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.4375,
-      "rewards/question_recreation_reward_func": 0.5434937998652458,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.5176317282021046,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.905875027179718,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 1.0225312784314156,
       "step": 18
     },
     {
-      "completion_length": 257.7916666666667,
+      "completion_length": 201.32142857142858,
       "epoch": 5.0,
-      "grad_norm": 0.572248637676239,
-      "kl": 0.0269426885060966,
+      "grad_norm": 0.6576955914497375,
+      "kl": 0.04337019233831337,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 6.045199592908223,
-      "reward_std": 1.806124746799469,
-      "rewards/concensus_correctness_reward_func": 2.25041663646698,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "reward": 6.447563511984689,
+      "reward_std": 1.7632163337298803,
+      "rewards/concensus_correctness_reward_func": 2.3713571344103133,
+      "rewards/consensus_reward_func": 1.7142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.490699624021848,
+      "rewards/final_correctness_reward_func": 1.0,
+      "rewards/question_recreation_reward_func": 0.4367780275642872,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.616583322485288,
+      "rewards/strict_format_reward_func": 0.08928571428571429,
+      "rewards/xmlcount_reward_func": 0.8358571614537921,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.0266644716903102e-05,
-      "train_runtime": 1124.2903,
-      "train_samples_per_second": 0.285,
-      "train_steps_per_second": 0.018
+      "train_loss": 3.4546391179901545e-05,
+      "train_runtime": 941.4145,
+      "train_samples_per_second": 0.34,
+      "train_steps_per_second": 0.021
     }
   ],
   "logging_steps": 2,