End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:689db96dd937d9a9be7b853a60f125f06315cd2e043483523dd59590deab5b3d
+oid sha256:3c0d7928e032561f5258fa745870b894d509e0a595c1acc3e131da4c8a9bf631
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.728446294346213e-07,
-    "train_runtime": 730.6569,
+    "train_loss": 6.986480116211169e-07,
+    "train_runtime": 683.7363,
     "train_samples": 11,
-    "train_samples_per_second": 0.438,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.468,
+    "train_steps_per_second": 0.029
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.728446294346213e-07,
-    "train_runtime": 730.6569,
+    "train_loss": 6.986480116211169e-07,
+    "train_runtime": 683.7363,
     "train_samples": 11,
-    "train_samples_per_second": 0.438,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.468,
+    "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 538.03125,
+      "completion_length": 549.3125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.4673408269882202,
-      "kl": 0.0006751879664079752,
+      "grad_norm": 0.3689783811569214,
+      "kl": 0.0007112065031833481,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.6300931870937347,
-      "reward_std": 0.7030879985541105,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.9230690384283662,
+      "reward_std": 0.788214260712266,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4146869406104088,
+      "rewards/question_recreation_reward_func": 0.5151627659797668,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15290625393390656,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.13446874730288982,
       "step": 2
     },
     {
-      "completion_length": 711.4642857142857,
+      "completion_length": 487.2857142857143,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.3041817545890808,
-      "kl": 0.0005705223551818303,
+      "grad_norm": 0.4033486545085907,
+      "kl": 0.000800232303195766,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.960112737757819,
-      "reward_std": 1.290339435849871,
-      "rewards/concensus_correctness_reward_func": 0.0447142847946712,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.7979355369295392,
+      "reward_std": 1.1481456458568573,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3419698966400964,
+      "rewards/question_recreation_reward_func": 0.23982839499201095,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21628572046756744,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.30810716109616415,
       "step": 4
     },
     {
-      "completion_length": 592.1428571428571,
+      "completion_length": 508.85714285714283,
       "epoch": 2.0,
-      "grad_norm": 0.321912944316864,
-      "kl": 0.0006380494700611702,
+      "grad_norm": 0.39565178751945496,
+      "kl": 0.0007155536773747631,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.8947670417172569,
-      "reward_std": 1.1553706526756287,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.3527063301631383,
+      "reward_std": 0.7333320592130933,
+      "rewards/concensus_correctness_reward_func": 0.032285713723727634,
       "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42183847086770193,
+      "rewards/question_recreation_reward_func": 0.4924206478255136,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.09792858362197876,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.43514285555907656,
       "step": 6
     },
     {
-      "completion_length": 634.625,
+      "completion_length": 580.4375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.33803796768188477,
-      "kl": 0.0006307054318313021,
+      "grad_norm": 0.33275121450424194,
+      "kl": 0.000691800465574488,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.5797345377504826,
-      "reward_std": 1.2402313686907291,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.9022217746824026,
+      "reward_std": 1.1502521634101868,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42010954208672047,
+      "rewards/question_recreation_reward_func": 0.411784278228879,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.04350000247359276,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.1857499945908785,
       "step": 8
     },
     {
-      "completion_length": 491.17857142857144,
+      "completion_length": 479.7142857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.42815327644348145,
-      "kl": 0.0007511325987122421,
+      "grad_norm": 0.3662431240081787,
+      "kl": 0.0007571371804390635,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.4952862433024815,
-      "reward_std": 0.8029870901788984,
-      "rewards/concensus_correctness_reward_func": -0.02121428506714957,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6258149264114243,
+      "reward_std": 0.8473422399588993,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3294648144926344,
+      "rewards/question_recreation_reward_func": 0.38060061846460613,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18703571813447134,
+      "rewards/xmlcount_reward_func": 0.17378571948834828,
       "step": 10
     },
     {
-      "completion_length": 463.64285714285717,
+      "completion_length": 482.92857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.4289427101612091,
-      "kl": 0.0007512595122014838,
+      "grad_norm": 0.37978729605674744,
+      "kl": 0.0007649853567792368,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8499365193503243,
-      "reward_std": 1.04093365584101,
-      "rewards/concensus_correctness_reward_func": 0.026857141937528337,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.1056848870856422,
+      "reward_std": 1.221989563533238,
+      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.3951150540794645,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.38929200704608646,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14225000249488012,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.18960715191704886,
       "step": 12
     },
     {
-      "completion_length": 461.9375,
+      "completion_length": 493.59375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 1.1750767230987549,
-      "kl": 0.0008141517610056326,
+      "grad_norm": 0.450574666261673,
+      "kl": 0.0006990073961787857,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.48488674685359,
-      "reward_std": 0.6717841476202011,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.0993756093084812,
+      "reward_std": 1.1431811600923538,
+      "rewards/concensus_correctness_reward_func": 0.02824999950826168,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.35335549898445606,
+      "rewards/question_recreation_reward_func": 0.47850061021745205,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0690312460064888,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.20200001541525126,
       "step": 14
     },
     {
-      "completion_length": 501.0357142857143,
+      "completion_length": 512.7142857142857,
       "epoch": 5.363636363636363,
-      "grad_norm": 3.0457513332366943,
-      "kl": 0.0008896194464926209,
+      "grad_norm": 0.4654828906059265,
+      "kl": 0.000716834728206907,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.0617176720074244,
-      "reward_std": 1.299524188041687,
+      "reward": 1.275504863687924,
+      "reward_std": 1.3435967394283839,
       "rewards/concensus_correctness_reward_func": 0.017857142857142856,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43861049839428495,
+      "rewards/question_recreation_reward_func": 0.44961203208991457,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17667856173855917,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.27232142431395395,
       "step": 16
     },
     {
-      "completion_length": 508.67857142857144,
+      "completion_length": 389.9642857142857,
       "epoch": 6.0,
-      "grad_norm": 0.4187767803668976,
-      "kl": 0.0007303775720564383,
+      "grad_norm": 0.3991488218307495,
+      "kl": 0.0007890741939523391,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.8850597121885845,
-      "reward_std": 0.9508567367281232,
-      "rewards/concensus_correctness_reward_func": 0.017857142857142856,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 0.8548716825566122,
+      "reward_std": 0.8182058547224317,
+      "rewards/concensus_correctness_reward_func": 0.026785714285714284,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.35448830681187765,
+      "rewards/question_recreation_reward_func": 0.4289074093103409,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.2091428658791951,
+      "rewards/xmlcount_reward_func": 0.309892869421414,
       "step": 18
     },
     {
-      "completion_length": 513.375,
+      "completion_length": 476.0,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.36682870984077454,
-      "kl": 0.0008761969875195064,
+      "grad_norm": 0.29800254106521606,
+      "kl": 0.0008042592162382789,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.7504257028922439,
-      "reward_std": 0.8404918480664492,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.7705026343464851,
+      "reward_std": 1.122593019157648,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42698819376528263,
+      "rewards/question_recreation_reward_func": 0.3620963804423809,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01093749888241291,
+      "rewards/xmlcount_reward_func": 0.1505937424954027,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.728446294346213e-07,
-      "train_runtime": 730.6569,
-      "train_samples_per_second": 0.438,
-      "train_steps_per_second": 0.027
+      "train_loss": 6.986480116211169e-07,
+      "train_runtime": 683.7363,
+      "train_samples_per_second": 0.468,
+      "train_steps_per_second": 0.029
     }
   ],
   "logging_steps": 2,