End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70e54e2b20eb3104143fb865bd7a8983dc631252410e7fab2594eb7b6745c545
+oid sha256:30a843b15014a48a5ba1d16e2e6946c6e6554bcd3ca5d9c32bfc2eeb72ff5776
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.857676602412539e-06,
-    "train_runtime": 2337.2993,
+    "train_loss": 1.8495638073545707e-06,
+    "train_runtime": 2244.6519,
     "train_samples": 14,
-    "train_samples_per_second": 0.137,
+    "train_samples_per_second": 0.143,
     "train_steps_per_second": 0.009
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.857676602412539e-06,
-    "train_runtime": 2337.2993,
+    "train_loss": 1.8495638073545707e-06,
+    "train_runtime": 2244.6519,
     "train_samples": 14,
-    "train_samples_per_second": 0.137,
+    "train_samples_per_second": 0.143,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 289.78125,
+      "completion_length": 300.40625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.6841569542884827,
-      "kl": 0.002072006231173873,
+      "grad_norm": 1.0302059650421143,
+      "kl": 0.0021278798958519474,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 4.195545099675655,
-      "reward_std": 4.08403642475605,
-      "rewards/concensus_correctness_reward_func": 2.697062471881509,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.249599501490593,
+      "reward_std": 2.7760701924562454,
+      "rewards/concensus_correctness_reward_func": 1.5001874696463346,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.44960761722177267,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5388495153747499,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3457499984651804,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.27306250343099236,
       "step": 2
     },
     {
-      "completion_length": 264.375,
+      "completion_length": 266.1666666666667,
       "epoch": 1.0,
-      "grad_norm": 0.37077268958091736,
-      "kl": 0.001950297368845592,
+      "grad_norm": 0.6066269874572754,
+      "kl": 0.0020866892494571707,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 5.263659656047821,
-      "reward_std": 2.924636865655581,
-      "rewards/concensus_correctness_reward_func": 2.8666666646798453,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 6.608254214127858,
+      "reward_std": 6.732338075836499,
+      "rewards/concensus_correctness_reward_func": 4.280166693031788,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.1666666666666667,
-      "rewards/question_recreation_reward_func": 0.5234094858169556,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.4993377576271693,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3735833478470643,
+      "rewards/xmlcount_reward_func": 0.41208334267139435,
       "step": 4
     },
     {
-      "completion_length": 302.875,
+      "completion_length": 297.875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.4881802499294281,
-      "kl": 0.0022304924204945564,
+      "grad_norm": 0.6188510060310364,
+      "kl": 0.002236725325929001,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.062274783849716,
-      "reward_std": 3.534151140600443,
-      "rewards/concensus_correctness_reward_func": 1.586687445640564,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.6490293107926846,
+      "reward_std": 3.6529240906238556,
+      "rewards/concensus_correctness_reward_func": 2.129249967634678,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4068685621023178,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.35549790132790804,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3187187500298023,
+      "rewards/xmlcount_reward_func": 0.2267812555655837,
       "step": 6
     },
     {
-      "completion_length": 264.0833333333333,
+      "completion_length": 283.375,
       "epoch": 2.0,
-      "grad_norm": 0.399483859539032,
-      "kl": 0.0018781623512040824,
+      "grad_norm": 0.48119544982910156,
+      "kl": 0.0021176343240464726,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.2692328045765557,
-      "reward_std": 2.7361116347213588,
-      "rewards/concensus_correctness_reward_func": 0.8456666668256124,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 4.623672604560852,
+      "reward_std": 4.630963092048963,
+      "rewards/concensus_correctness_reward_func": 2.537750000754992,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 1.25,
-      "rewards/question_recreation_reward_func": 0.4897326702872912,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5768810758988062,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.433833343287309,
+      "rewards/xmlcount_reward_func": 0.5923750052849451,
       "step": 8
     },
     {
-      "completion_length": 310.625,
+      "completion_length": 307.71875,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.6462966203689575,
-      "kl": 0.001986428673262708,
+      "grad_norm": 0.520293116569519,
+      "kl": 0.0019132717279717326,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 4.808056481182575,
-      "reward_std": 4.114130575209856,
-      "rewards/concensus_correctness_reward_func": 2.7208124957978725,
+      "reward": 5.4057668671011925,
+      "reward_std": 6.585089758038521,
+      "rewards/concensus_correctness_reward_func": 3.252749995328486,
       "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.75,
-      "rewards/question_recreation_reward_func": 0.5032440163195133,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.5540796220302582,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2714999979361892,
+      "rewards/xmlcount_reward_func": 0.2239374970085919,
       "step": 10
     },
     {
-      "completion_length": 326.7916666666667,
+      "completion_length": 299.6666666666667,
       "epoch": 3.0,
-      "grad_norm": 0.23907510936260223,
-      "kl": 0.0020917693230633936,
+      "grad_norm": 0.3321484327316284,
+      "kl": 0.002006225608056411,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.3901617924372354,
-      "reward_std": 3.109941323598226,
-      "rewards/concensus_correctness_reward_func": 1.0677499969800313,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 5.581961214542389,
+      "reward_std": 4.6081631779670715,
+      "rewards/concensus_correctness_reward_func": 3.557416655123234,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.3434951454401016,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.55900310476621,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14558333655198416,
+      "rewards/xmlcount_reward_func": 0.4655416688571374,
       "step": 12
     },
     {
-      "completion_length": 309.96875,
+      "completion_length": 264.6875,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.5326588153839111,
-      "kl": 0.001975269304239191,
+      "grad_norm": 0.5792955756187439,
+      "kl": 0.002444218858727254,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 6.68176244199276,
-      "reward_std": 4.770209355279803,
-      "rewards/concensus_correctness_reward_func": 4.679187453817576,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 3.8108829148113728,
+      "reward_std": 3.8173418045043945,
+      "rewards/concensus_correctness_reward_func": 2.0289374962449074,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6875,
-      "rewards/question_recreation_reward_func": 0.5782312154769897,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.48997668735682964,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.29934375686571,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27634374611079693,
       "step": 14
     },
     {
-      "completion_length": 290.4583333333333,
+      "completion_length": 301.2916666666667,
       "epoch": 4.0,
-      "grad_norm": 0.2591925859451294,
-      "kl": 0.002289272650765876,
+      "grad_norm": 0.3442418873310089,
+      "kl": 0.002062379712394128,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.333374003569285,
-      "reward_std": 1.6044593056042988,
-      "rewards/concensus_correctness_reward_func": 0.6932499731580416,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 3.2509370893239975,
+      "reward_std": 4.266276851296425,
+      "rewards/concensus_correctness_reward_func": 1.8028333137432735,
+      "rewards/consensus_reward_func": 0.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4166666666666667,
-      "rewards/question_recreation_reward_func": 0.3929156760374705,
+      "rewards/final_correctness_reward_func": 0.6666666666666666,
+      "rewards/question_recreation_reward_func": 0.36239541073640186,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4138749949634075,
+      "rewards/xmlcount_reward_func": 0.0857083344211181,
       "step": 16
     },
     {
-      "completion_length": 305.46875,
+      "completion_length": 312.4375,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.6285619735717773,
-      "kl": 0.002280414933920838,
+      "grad_norm": 0.6844708323478699,
+      "kl": 0.0017517954256618395,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.684103332459927,
-      "reward_std": 6.790900774300098,
-      "rewards/concensus_correctness_reward_func": 4.525937512516975,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 5.29012943059206,
+      "reward_std": 5.383719008415937,
+      "rewards/concensus_correctness_reward_func": 3.2063124999403954,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.8125,
-      "rewards/question_recreation_reward_func": 0.6095721758902073,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5631606644019485,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17359375022351742,
+      "rewards/xmlcount_reward_func": 0.3956562525127083,
       "step": 18
     },
     {
-      "completion_length": 256.2083333333333,
+      "completion_length": 239.33333333333334,
       "epoch": 5.0,
-      "grad_norm": 0.3895675539970398,
-      "kl": 0.0022517238976433873,
+      "grad_norm": 0.5124043822288513,
+      "kl": 0.0027045891232167683,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 4.024020781119664,
-      "reward_std": 3.8886889715989432,
-      "rewards/concensus_correctness_reward_func": 1.9681666195392609,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 3.541064421335856,
+      "reward_std": 4.355749766031901,
+      "rewards/concensus_correctness_reward_func": 1.7235833319524925,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.3190208002924919,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.37352269639571506,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.5493333339691162,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5272916654745737,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.857676602412539e-06,
-      "train_runtime": 2337.2993,
-      "train_samples_per_second": 0.137,
+      "train_loss": 1.8495638073545707e-06,
+      "train_runtime": 2244.6519,
+      "train_samples_per_second": 0.143,
       "train_steps_per_second": 0.009
     }
   ],