End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf1ef94fe975a431651b87e81e45f0e715fc661d941d2a72a324b5d007615aa0
+oid sha256:55d5e762d6b16f7e480efdc78309d0f3e0a564665e3a4f3412757c45a0ca9559
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.606403076148126e-07,
-    "train_runtime": 1871.2935,
+    "train_loss": 6.167098831610929e-07,
+    "train_runtime": 1845.5956,
     "train_samples": 11,
-    "train_samples_per_second": 0.171,
+    "train_samples_per_second": 0.173,
     "train_steps_per_second": 0.011
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.606403076148126e-07,
-    "train_runtime": 1871.2935,
+    "train_loss": 6.167098831610929e-07,
+    "train_runtime": 1845.5956,
     "train_samples": 11,
-    "train_samples_per_second": 0.171,
+    "train_samples_per_second": 0.173,
     "train_steps_per_second": 0.011
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 572.3125,
+      "completion_length": 595.53125,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.3157753050327301,
-      "kl": 0.0006395862073986791,
+      "grad_norm": 0.301849365234375,
+      "kl": 0.0005805464315926656,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.2950003445148468,
-      "reward_std": 0.8744391687214375,
-      "rewards/concensus_correctness_reward_func": 0.007249999791383743,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.8321242239326239,
+      "reward_std": 1.2119891792535782,
+      "rewards/concensus_correctness_reward_func": 0.08924999833106995,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5584065895527601,
+      "rewards/question_recreation_reward_func": 0.545967947691679,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4168437453918159,
+      "rewards/xmlcount_reward_func": -0.05309375561773777,
       "step": 2
     },
     {
-      "completion_length": 535.6785714285714,
+      "completion_length": 530.9642857142857,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.3245207369327545,
-      "kl": 0.0006644960667472333,
+      "grad_norm": 0.2668595016002655,
+      "kl": 0.0006280639125699443,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.224908811705453,
-      "reward_std": 1.4749388354165214,
-      "rewards/concensus_correctness_reward_func": 0.06214285748345511,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.0686943531036377,
+      "reward_std": 0.8797949467386518,
+      "rewards/concensus_correctness_reward_func": 0.029428571462631226,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6173730790615082,
+      "rewards/question_recreation_reward_func": 0.6413014701434544,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.009678568691015244,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18367857060262135,
       "step": 4
     },
     {
-      "completion_length": 536.75,
+      "completion_length": 422.0357142857143,
       "epoch": 2.0,
-      "grad_norm": 0.4441211521625519,
-      "kl": 0.0007640795333177916,
+      "grad_norm": 0.4675261974334717,
+      "kl": 0.0008043012016319803,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.0552446757044112,
-      "reward_std": 1.1172970363071986,
-      "rewards/concensus_correctness_reward_func": 0.06628571450710297,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.0894620471767016,
+      "reward_std": 1.0836134127208166,
+      "rewards/concensus_correctness_reward_func": 0.15628572021211898,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4873517858130591,
+      "rewards/question_recreation_reward_func": 0.3489262882087912,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.26946428418159485,
+      "rewards/xmlcount_reward_func": 0.20925000522817885,
       "step": 6
     },
     {
-      "completion_length": 580.40625,
+      "completion_length": 476.125,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.45746123790740967,
-      "kl": 0.0007559946097899228,
+      "grad_norm": 0.44858428835868835,
+      "kl": 0.0006725056446157396,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.9745578002184629,
-      "reward_std": 1.1468579769134521,
-      "rewards/concensus_correctness_reward_func": 0.054375000298023224,
+      "reward": 1.2273968532681465,
+      "reward_std": 1.167898677289486,
+      "rewards/concensus_correctness_reward_func": 0.1367500051856041,
       "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5592140201479197,
+      "rewards/question_recreation_reward_func": 0.4859906043857336,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.017218757420778275,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27653125301003456,
       "step": 8
     },
     {
-      "completion_length": 434.5,
+      "completion_length": 497.67857142857144,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.42132773995399475,
-      "kl": 0.0008870551828294992,
+      "grad_norm": 0.3868691027164459,
+      "kl": 0.0007043707633524068,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.3714402318000793,
-      "reward_std": 1.0166289508342743,
+      "reward": 1.0494755506515503,
+      "reward_std": 0.908211031130382,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4916902482509613,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.5289040803909302,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.6476071391786847,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16342857160738536,
       "step": 10
     },
     {
-      "completion_length": 503.64285714285717,
+      "completion_length": 504.92857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.29574769735336304,
-      "kl": 0.0006107680507869061,
+      "grad_norm": 0.3196893036365509,
+      "kl": 0.0005888212181162089,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8827149101666042,
-      "reward_std": 0.9164597860404423,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.1278794492994035,
+      "reward_std": 0.8067079143864768,
+      "rewards/concensus_correctness_reward_func": 0.03400000078337533,
+      "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5152863285371235,
+      "rewards/question_recreation_reward_func": 0.5212009080818721,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.20671429804393224,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14410714272941863,
       "step": 12
     },
     {
-      "completion_length": 516.8125,
+      "completion_length": 525.25,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.335713654756546,
-      "kl": 0.0007717612461419776,
+      "grad_norm": 5.975808620452881,
+      "kl": 0.0007298545133380685,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.9521977975964546,
-      "reward_std": 1.2133671417832375,
-      "rewards/concensus_correctness_reward_func": 0.058000000193715096,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.9535989612340927,
+      "reward_std": 1.0574432387948036,
+      "rewards/concensus_correctness_reward_func": 0.025750000029802322,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5835415087640285,
+      "rewards/question_recreation_reward_func": 0.4457552069798112,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.0450312620960176,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.04459376260638237,
       "step": 14
     },
     {
-      "completion_length": 488.89285714285717,
+      "completion_length": 430.07142857142856,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.42530447244644165,
-      "kl": 0.0005963708515212472,
+      "grad_norm": 0.5423234105110168,
+      "kl": 0.000709650871743049,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.2204667670386178,
-      "reward_std": 0.8674233385494777,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.340957292488643,
+      "reward_std": 0.9939541092940739,
+      "rewards/concensus_correctness_reward_func": 0.10199999809265137,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5821810045412609,
+      "rewards/question_recreation_reward_func": 0.5202786922454834,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.38828571726168903,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.2008214145898819,
       "step": 16
     },
     {
-      "completion_length": 550.2857142857143,
+      "completion_length": 482.2142857142857,
       "epoch": 6.0,
-      "grad_norm": 0.340285062789917,
-      "kl": 0.0005592115256669265,
+      "grad_norm": 0.28088223934173584,
+      "kl": 0.0006366945246034968,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.8162363597324916,
-      "reward_std": 1.271778894322259,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.6610284873417446,
+      "reward_std": 1.1703646268163408,
+      "rewards/concensus_correctness_reward_func": 0.03400000078337533,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.31716491814170567,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.3840998560190201,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.10621428276811327,
+      "rewards/xmlcount_reward_func": 0.010785728693008423,
       "step": 18
     },
     {
-      "completion_length": 491.9375,
+      "completion_length": 507.46875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.44386664032936096,
-      "kl": 0.0008113829680951312,
+      "grad_norm": 0.3558220863342285,
+      "kl": 0.0006192766377353109,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.1200277395546436,
-      "reward_std": 1.0833697468042374,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 0.9940839540213346,
+      "reward_std": 0.9333450272679329,
+      "rewards/concensus_correctness_reward_func": 0.05550000071525574,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4294339846819639,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4757089614868164,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.15934374555945396,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.07225000206381083,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.606403076148126e-07,
-      "train_runtime": 1871.2935,
-      "train_samples_per_second": 0.171,
+      "train_loss": 6.167098831610929e-07,
+      "train_runtime": 1845.5956,
+      "train_samples_per_second": 0.173,
       "train_steps_per_second": 0.011
     }
   ],