End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cdf2af32b25cecb029d4f86a98fb8dde6fb18d2ec588041760e96f5cf08fa6c9
+oid sha256:a7952a32e077d98f48923352fbfbae1ee736db81d6f481124de57400757a2789
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.443744904212508e-06,
-    "train_runtime": 606.4441,
+    "train_loss": 2.217736880538723e-06,
+    "train_runtime": 583.693,
     "train_samples": 11,
-    "train_samples_per_second": 0.528,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.548,
+    "train_steps_per_second": 0.034
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.443744904212508e-06,
-    "train_runtime": 606.4441,
+    "train_loss": 2.217736880538723e-06,
+    "train_runtime": 583.693,
     "train_samples": 11,
-    "train_samples_per_second": 0.528,
-    "train_steps_per_second": 0.033
+    "train_samples_per_second": 0.548,
+    "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 438.125,
+      "completion_length": 431.25,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.5471447706222534,
-      "kl": 0.002215708445874043,
+      "grad_norm": 0.45781487226486206,
+      "kl": 0.0018781532053253613,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.8272043820470572,
-      "reward_std": 1.137837901711464,
-      "rewards/concensus_correctness_reward_func": 0.017625000327825546,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 0.9149962170049548,
+      "reward_std": 1.1113301180303097,
+      "rewards/concensus_correctness_reward_func": -0.06106249615550041,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4313293769955635,
+      "rewards/question_recreation_reward_func": 0.47158994898200035,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.1126250047236681,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.004468740429729223,
       "step": 2
     },
     {
-      "completion_length": 482.60714285714283,
+      "completion_length": 574.0357142857143,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.5870139002799988,
-      "kl": 0.002228286233730614,
+      "grad_norm": 0.6121382713317871,
+      "kl": 0.002408954140264541,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.0098560793059213,
-      "reward_std": 1.040208854845592,
+      "reward": 1.1950411285672868,
+      "reward_std": 1.0991400139672416,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "rewards/consensus_reward_func": 0.6428571428571429,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.535213223525456,
+      "rewards/question_recreation_reward_func": 0.4831125757523945,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.117500011410032,
+      "rewards/xmlcount_reward_func": 0.06907142698764801,
       "step": 4
     },
     {
-      "completion_length": 526.1428571428571,
+      "completion_length": 406.39285714285717,
       "epoch": 2.0,
-      "grad_norm": 0.46006160974502563,
-      "kl": 0.0019179357581638864,
+      "grad_norm": 1.057285189628601,
+      "kl": 0.0024510998628102243,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.043937521321433,
-      "reward_std": 0.9618620617049081,
-      "rewards/concensus_correctness_reward_func": 0.03914285770484379,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.426652044057846,
+      "reward_std": 1.1198874711990356,
+      "rewards/concensus_correctness_reward_func": 0.10521428499902997,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4214732583080019,
+      "rewards/question_recreation_reward_func": 0.4751520710332053,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.35117857477494646,
+      "rewards/xmlcount_reward_func": 0.3284285898719515,
       "step": 6
     },
     {
-      "completion_length": 390.5,
+      "completion_length": 353.59375,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.6136392951011658,
-      "kl": 0.0033047929900931194,
+      "grad_norm": 0.5928940176963806,
+      "kl": 0.0026474942278582603,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.0768073704093695,
-      "reward_std": 1.0190978571772575,
-      "rewards/concensus_correctness_reward_func": 0.016625000163912773,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 0.9612564370036125,
+      "reward_std": 1.1644172109663486,
+      "rewards/concensus_correctness_reward_func": -0.047937504947185516,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.43793237395584583,
+      "rewards/question_recreation_reward_func": 0.4168814420700073,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.13787500839680433,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2173124998807907,
       "step": 8
     },
     {
-      "completion_length": 503.92857142857144,
+      "completion_length": 635.9642857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 15.232477188110352,
-      "kl": 0.0031253419583663344,
+      "grad_norm": 1.094155192375183,
+      "kl": 0.0022622206015512347,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.1312814440046037,
-      "reward_std": 1.0373194898877824,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.027773882661547,
+      "reward_std": 1.159633185182299,
+      "rewards/concensus_correctness_reward_func": -0.060642855507986884,
+      "rewards/consensus_reward_func": 0.6428571428571429,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5115671583584377,
+      "rewards/question_recreation_reward_func": 0.4313809967466763,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.3161428634609495,
+      "rewards/xmlcount_reward_func": -0.003678555999483381,
       "step": 10
     },
     {
-      "completion_length": 504.75,
+      "completion_length": 423.7857142857143,
       "epoch": 4.0,
-      "grad_norm": 0.32220467925071716,
-      "kl": 0.0019690781399341567,
+      "grad_norm": 5.062568187713623,
+      "kl": 0.0027806301285246654,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.8716988393238613,
-      "reward_std": 0.843503794499806,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.3578734993934631,
+      "reward_std": 1.2459349461964198,
+      "rewards/concensus_correctness_reward_func": 0.1030000022479466,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3871631239141737,
+      "rewards/question_recreation_reward_func": 0.5079449521643775,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19882142330918992,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.15764284985406057,
       "step": 12
     },
     {
-      "completion_length": 401.71875,
+      "completion_length": 465.21875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.5461463928222656,
-      "kl": 0.0028550010902108625,
+      "grad_norm": 0.5010895133018494,
+      "kl": 0.0021581597757176496,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.2372233513742685,
-      "reward_std": 1.1147281527519226,
-      "rewards/concensus_correctness_reward_func": 0.017625000327825546,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.0889395736157894,
+      "reward_std": 1.1303429789841175,
+      "rewards/concensus_correctness_reward_func": 0.09012500196695328,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.561910830438137,
+      "rewards/question_recreation_reward_func": 0.49959580041468143,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.22018748987466097,
+      "rewards/xmlcount_reward_func": 0.12421875074505806,
       "step": 14
     },
     {
-      "completion_length": 452.17857142857144,
+      "completion_length": 368.57142857142856,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.6385740637779236,
-      "kl": 0.002464203496596643,
+      "grad_norm": 0.6077002882957458,
+      "kl": 0.0028997911805553095,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.9881289431027004,
-      "reward_std": 1.0410894038421767,
-      "rewards/concensus_correctness_reward_func": 0.019000000187328885,
-      "rewards/consensus_reward_func": 0.2857142857142857,
+      "reward": 1.1805721010480608,
+      "reward_std": 1.1145481254373277,
+      "rewards/concensus_correctness_reward_func": 0.017714287553514754,
+      "rewards/consensus_reward_func": 0.8571428571428571,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.47170030006340574,
+      "rewards/question_recreation_reward_func": 0.3811078156743731,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2117142943399293,
+      "rewards/xmlcount_reward_func": -0.07539285932268415,
       "step": 16
     },
     {
-      "completion_length": 375.35714285714283,
+      "completion_length": 578.6071428571429,
       "epoch": 6.0,
-      "grad_norm": 0.4907841980457306,
-      "kl": 0.0025090476597792338,
+      "grad_norm": 0.4037664234638214,
+      "kl": 0.0015938405579488193,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.026438516165529,
-      "reward_std": 0.9799486909593854,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 0.9193015290158135,
+      "reward_std": 1.164186647960118,
+      "rewards/concensus_correctness_reward_func": 0.053714283875056675,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4783671290746757,
+      "rewards/question_recreation_reward_func": 0.5321587026119232,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.07142857142857142,
-      "rewards/xmlcount_reward_func": 0.333785708461489,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.023714278425489153,
       "step": 18
     },
     {
-      "completion_length": 499.28125,
+      "completion_length": 455.1875,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.4972382187843323,
-      "kl": 0.0034904882340924814,
+      "grad_norm": 0.43111908435821533,
+      "kl": 0.0029546206642407924,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.2245423384010792,
-      "reward_std": 1.2330018803477287,
-      "rewards/concensus_correctness_reward_func": 0.017625000327825546,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.222639849409461,
+      "reward_std": 0.8690865337848663,
+      "rewards/concensus_correctness_reward_func": -0.053062498569488525,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6299485601484776,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/question_recreation_reward_func": 0.5259210728108883,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.21759376488626003,
+      "rewards/xmlcount_reward_func": 0.34353126026690006,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.443744904212508e-06,
-      "train_runtime": 606.4441,
-      "train_samples_per_second": 0.528,
-      "train_steps_per_second": 0.033
+      "train_loss": 2.217736880538723e-06,
+      "train_runtime": 583.693,
+      "train_samples_per_second": 0.548,
+      "train_steps_per_second": 0.034
     }
   ],
   "logging_steps": 2,