End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:154982cb8d50218f37d448d3574a7043ae1db5e090a8cae76a9ee294490b1ca9
+oid sha256:73e2e613b0dd50861467299d81a62578cdf4f853b19162a1c3318f33d93078fc
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.819631434813345e-07,
-    "train_runtime": 605.571,
+    "train_loss": 7.299307981156744e-07,
+    "train_runtime": 635.3271,
     "train_samples": 11,
-    "train_samples_per_second": 0.528,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.504,
+    "train_steps_per_second": 0.031
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.819631434813345e-07,
-    "train_runtime": 605.571,
+    "train_loss": 7.299307981156744e-07,
+    "train_runtime": 635.3271,
     "train_samples": 11,
-    "train_samples_per_second": 0.528,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.504,
+    "train_steps_per_second": 0.031
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 474.25,
+      "completion_length": 482.3125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.38834014534950256,
-      "kl": 0.0006797206369810738,
+      "grad_norm": 0.28858432173728943,
+      "kl": 0.0006951168943487573,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.1539140474051237,
-      "reward_std": 1.0125724151730537,
-      "rewards/concensus_correctness_reward_func": 0.04931250005029142,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.1565705612301826,
+      "reward_std": 1.1907042190432549,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.607539027929306,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5826955810189247,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3095625005662441,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.23012500535696745,
       "step": 2
     },
     {
-      "completion_length": 461.5,
+      "completion_length": 586.0714285714286,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.3262319564819336,
-      "kl": 0.0006221135055446732,
+      "grad_norm": 0.39020222425460815,
+      "kl": 0.0006209207266303045,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.6816681027412415,
-      "reward_std": 1.026350736618042,
-      "rewards/concensus_correctness_reward_func": 0.10714285714285714,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 0.30154422777039663,
+      "reward_std": 1.03359112569264,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4933823751551764,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.548151355768953,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.06171427667140961,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.3358928454773767,
       "step": 4
     },
     {
-      "completion_length": 481.67857142857144,
+      "completion_length": 429.92857142857144,
       "epoch": 2.0,
-      "grad_norm": 0.36097216606140137,
-      "kl": 0.0006781607891233372,
+      "grad_norm": 0.2881869971752167,
+      "kl": 0.0007869384634042424,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.5162993669509888,
-      "reward_std": 0.7969776306833539,
-      "rewards/concensus_correctness_reward_func": 0.17764286563864776,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.2450389776911055,
+      "reward_std": 1.1033319916043962,
+      "rewards/concensus_correctness_reward_func": 0.038857142840112956,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6561921834945679,
+      "rewards/question_recreation_reward_func": 0.6028603996549334,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4681785660130637,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.13903571665287018,
       "step": 6
     },
     {
-      "completion_length": 495.84375,
+      "completion_length": 575.78125,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.4024622142314911,
-      "kl": 0.0006359297258313745,
+      "grad_norm": 0.2738513946533203,
+      "kl": 0.0007034756636130624,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.2740945629775524,
-      "reward_std": 1.335190288722515,
-      "rewards/concensus_correctness_reward_func": 0.10687500145286322,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.8351963348686695,
+      "reward_std": 1.111624389886856,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6691257953643799,
+      "rewards/question_recreation_reward_func": 0.7195400707423687,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.24809375405311584,
+      "rewards/xmlcount_reward_func": -0.2593437507748604,
       "step": 8
     },
     {
-      "completion_length": 530.7857142857143,
+      "completion_length": 463.2857142857143,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.4588069021701813,
-      "kl": 0.0009082068240136973,
+      "grad_norm": 0.44207867980003357,
+      "kl": 0.0007244269280428332,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.4613203874656133,
-      "reward_std": 1.3176539880888802,
-      "rewards/concensus_correctness_reward_func": 0.18385714292526245,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.7830644420215062,
+      "reward_std": 0.9246002180235726,
+      "rewards/concensus_correctness_reward_func": 0.03257142858845847,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5581417743648801,
+      "rewards/question_recreation_reward_func": 0.6651001444884709,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2193214339869363,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.003892879400934492,
       "step": 10
     },
     {
-      "completion_length": 389.35714285714283,
+      "completion_length": 562.2142857142857,
       "epoch": 4.0,
-      "grad_norm": 0.41611847281455994,
-      "kl": 0.0007420033341207143,
+      "grad_norm": 30.6977596282959,
+      "kl": 0.0016899453891840363,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.6613011796559606,
-      "reward_std": 0.5857992001942226,
+      "reward": 1.1181254131453378,
+      "reward_std": 1.1169071452958244,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5059083210570472,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.11967858459268298,
+      "rewards/question_recreation_reward_func": 0.5588396957942418,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.20214285169328963,
       "step": 12
     },
     {
-      "completion_length": 391.71875,
+      "completion_length": 540.1875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 13.172017097473145,
-      "kl": 0.0006414422678062692,
+      "grad_norm": 0.3711148202419281,
+      "kl": 0.0006524448617710732,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.8641659906134009,
-      "reward_std": 0.6996540110558271,
-      "rewards/concensus_correctness_reward_func": 0.05325000127777457,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.3576028421521187,
+      "reward_std": 1.0308610945940018,
+      "rewards/concensus_correctness_reward_func": 0.02850000001490116,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.48482225043699145,
+      "rewards/question_recreation_reward_func": 0.630852835252881,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.24796875193715096,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4795000050216913,
       "step": 14
     },
     {
-      "completion_length": 378.42857142857144,
+      "completion_length": 475.5357142857143,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.653586208820343,
-      "kl": 0.0006904835297193911,
+      "grad_norm": 0.3065779209136963,
+      "kl": 0.0007360076498506325,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.928639394896371,
-      "reward_std": 1.0318341595785958,
-      "rewards/concensus_correctness_reward_func": 0.061285714485815594,
+      "reward": 0.8269010931253433,
+      "reward_std": 1.0506312272378378,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5085679335253579,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.6615439312798637,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21592856304986136,
+      "rewards/xmlcount_reward_func": -0.04892856627702713,
       "step": 16
     },
     {
-      "completion_length": 542.4285714285714,
+      "completion_length": 561.9642857142857,
       "epoch": 6.0,
-      "grad_norm": 0.28921207785606384,
-      "kl": 0.0007156268360891513,
+      "grad_norm": 0.3327679932117462,
+      "kl": 0.0006825594796932169,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.6005037022488457,
-      "reward_std": 0.7378830909729004,
+      "reward": 1.0864614759172713,
+      "reward_std": 1.129058620759419,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4484680103404181,
+      "rewards/question_recreation_reward_func": 0.4989971901689257,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.009178570338657923,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.0696071567279952,
       "step": 18
     },
     {
-      "completion_length": 456.40625,
+      "completion_length": 508.5,
       "epoch": 6.7272727272727275,
-      "grad_norm": 12.717683792114258,
-      "kl": 0.001965975017810706,
+      "grad_norm": 0.39833685755729675,
+      "kl": 0.0006583448630408384,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.5806790366768837,
-      "reward_std": 1.1188428923487663,
-      "rewards/concensus_correctness_reward_func": 0.21375000290572643,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.0544417798519135,
+      "reward_std": 1.448256067931652,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5630853101611137,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.28821875527501106,
+      "rewards/question_recreation_reward_func": 0.5876918025314808,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.060499997809529305,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.819631434813345e-07,
-      "train_runtime": 605.571,
-      "train_samples_per_second": 0.528,
-      "train_steps_per_second": 0.033
+      "train_loss": 7.299307981156744e-07,
+      "train_runtime": 635.3271,
+      "train_samples_per_second": 0.504,
+      "train_steps_per_second": 0.031
     }
   ],
   "logging_steps": 2,