End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b8714686e222ccd8e46eb04555a5e3feb712a3b75fcbabb8ee3c14f4f836d30
+oid sha256:ab0edee0a5288cb668723b99e99141ac342cc3e26c0e84a9f2fd4065f316c356
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.499961950292345e-07,
-    "train_runtime": 650.8904,
+    "train_loss": 9.828601662320579e-07,
+    "train_runtime": 623.2019,
     "train_samples": 11,
-    "train_samples_per_second": 0.492,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 0.513,
+    "train_steps_per_second": 0.032
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.499961950292345e-07,
-    "train_runtime": 650.8904,
+    "train_loss": 9.828601662320579e-07,
+    "train_runtime": 623.2019,
     "train_samples": 11,
-    "train_samples_per_second": 0.492,
-    "train_steps_per_second": 0.031
+    "train_samples_per_second": 0.513,
+    "train_steps_per_second": 0.032
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 520.0625,
+      "completion_length": 489.0625,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.3609532117843628,
-      "kl": 0.0008632853860035539,
+      "grad_norm": 0.3488386273384094,
+      "kl": 0.001196328055812046,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.4968189150094986,
-      "reward_std": 2.4095203951001167,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.9943452849984169,
+      "reward_std": 1.1032958514988422,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6299751847982407,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5704702585935593,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.14878125302493572,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.23637501150369644,
       "step": 2
     },
     {
-      "completion_length": 439.82142857142856,
+      "completion_length": 552.3928571428571,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.2829833924770355,
-      "kl": 0.000931203573210431,
+      "grad_norm": 0.3434063792228699,
+      "kl": 0.0010169482557103038,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.8967825238193784,
-      "reward_std": 0.9078347980976105,
+      "reward": 0.48578563438994543,
+      "reward_std": 1.083632937499455,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.5199253473963056,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5772499144077301,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14471427989857538,
+      "rewards/xmlcount_reward_func": -0.09146428108215332,
       "step": 4
     },
     {
-      "completion_length": 520.9285714285714,
+      "completion_length": 470.7857142857143,
       "epoch": 2.0,
-      "grad_norm": 0.321284681558609,
-      "kl": 0.0009079717149558876,
+      "grad_norm": 0.284168541431427,
+      "kl": 0.0009620040239367102,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.1936021276882716,
-      "reward_std": 1.2356498071125575,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.356112580214228,
+      "reward_std": 0.7955805148397174,
+      "rewards/concensus_correctness_reward_func": 0.06542857204164777,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.5381021499633789,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6544340295451028,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15550000965595245,
+      "rewards/xmlcount_reward_func": 0.06482142635754176,
       "step": 6
     },
     {
-      "completion_length": 447.59375,
+      "completion_length": 512.0,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.500460147857666,
-      "kl": 0.000998846284346655,
+      "grad_norm": 0.4184742867946625,
+      "kl": 0.0010141520469915122,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.8486154144629836,
-      "reward_std": 0.9767724275588989,
-      "rewards/concensus_correctness_reward_func": 0.017249999567866325,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.6775961369276047,
+      "reward_std": 1.1034768782556057,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5988029353320599,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5896273851394653,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.07631250098347664,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": -0.17765624169260263,
       "step": 8
     },
     {
-      "completion_length": 530.1071428571429,
+      "completion_length": 411.35714285714283,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.3490114212036133,
-      "kl": 0.0008048345916904509,
+      "grad_norm": 0.4660803973674774,
+      "kl": 0.001223136271749224,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.0381762768541063,
-      "reward_std": 2.7257935660226003,
-      "rewards/concensus_correctness_reward_func": 1.4285714285714286,
-      "rewards/consensus_reward_func": 0.5714285714285714,
+      "reward": 1.6157191257391657,
+      "reward_std": 1.052688785961696,
+      "rewards/concensus_correctness_reward_func": 0.13085714408329555,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.35714285714285715,
-      "rewards/question_recreation_reward_func": 0.5029976708548409,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6174691191741398,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17803571479661123,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.34953572707516806,
       "step": 10
     },
     {
-      "completion_length": 534.8571428571429,
+      "completion_length": 425.14285714285717,
       "epoch": 4.0,
-      "grad_norm": 0.7429814338684082,
-      "kl": 0.0011950191900333656,
+      "grad_norm": 0.3487090766429901,
+      "kl": 0.001042323378247342,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8569603604929787,
-      "reward_std": 1.3656096926757268,
+      "reward": 0.7835611743586404,
+      "reward_std": 0.9644212062869754,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21428571428571427,
-      "rewards/question_recreation_reward_func": 0.6343889491898673,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.494311198592186,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": -0.2417143029826028,
+      "rewards/xmlcount_reward_func": -0.03217856798853193,
       "step": 12
     },
     {
-      "completion_length": 471.75,
+      "completion_length": 478.125,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.7710726857185364,
-      "kl": 0.0009791331394808367,
+      "grad_norm": 0.542263388633728,
+      "kl": 0.0011764185037463903,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.7225969061255455,
-      "reward_std": 2.2982127144932747,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "reward": 1.0417498797178268,
+      "reward_std": 1.322362545877695,
+      "rewards/concensus_correctness_reward_func": 0.0572500005364418,
       "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5782218500971794,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5876248776912689,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.19125000620260835,
+      "rewards/xmlcount_reward_func": 0.006250003352761269,
       "step": 14
     },
     {
-      "completion_length": 609.3214285714286,
+      "completion_length": 451.60714285714283,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.23411498963832855,
-      "kl": 0.0008231612487829157,
+      "grad_norm": 0.46988293528556824,
+      "kl": 0.0010228758744363273,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.5272872000932693,
-      "reward_std": 1.1376740336418152,
+      "reward": 1.0151320823601313,
+      "reward_std": 0.8673076469983373,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6187157460621425,
+      "rewards/question_recreation_reward_func": 0.5092392350946154,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.3057142900569098,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.2023214330630643,
       "step": 16
     },
     {
-      "completion_length": 575.2857142857143,
+      "completion_length": 508.32142857142856,
       "epoch": 6.0,
-      "grad_norm": 0.28983500599861145,
-      "kl": 0.0007936374001604106,
+      "grad_norm": 0.4228575527667999,
+      "kl": 0.0009882049967667886,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.147021634238107,
-      "reward_std": 2.643599476133074,
-      "rewards/concensus_correctness_reward_func": 0.7142857142857143,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "reward": 0.5577128806284496,
+      "reward_std": 0.9692163808005196,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21428571428571427,
-      "rewards/question_recreation_reward_func": 0.6069859308855874,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.49635572731494904,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25432143466813223,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": -0.1707857187305178,
       "step": 18
     },
     {
-      "completion_length": 555.125,
+      "completion_length": 449.71875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.38677704334259033,
-      "kl": 0.0008186684717657045,
+      "grad_norm": 0.3519269824028015,
+      "kl": 0.0009945403508027084,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8146747946739197,
-      "reward_std": 0.9930036887526512,
+      "reward": 0.9330334067344666,
+      "reward_std": 0.9351694285869598,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.125,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.49986225739121437,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5555646400898695,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.07581249764189124,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.14309374522417784,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.499961950292345e-07,
-      "train_runtime": 650.8904,
-      "train_samples_per_second": 0.492,
-      "train_steps_per_second": 0.031
+      "train_loss": 9.828601662320579e-07,
+      "train_runtime": 623.2019,
+      "train_samples_per_second": 0.513,
+      "train_steps_per_second": 0.032
     }
   ],
   "logging_steps": 2,