End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:919239020f1ef272d80ae74de18a43341f6f04e511ce9773c56cf7de32f868bb
+oid sha256:fa7590fd90bfde3b1335b7433d356b270696d6765a8d4aaee2338a4785cfad02
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.367493478544929e-07,
-    "train_runtime": 1195.3968,
+    "train_loss": 7.007378655998764e-07,
+    "train_runtime": 1568.7671,
     "train_samples": 11,
-    "train_samples_per_second": 0.268,
-    "train_steps_per_second": 0.017
+    "train_samples_per_second": 0.204,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.367493478544929e-07,
-    "train_runtime": 1195.3968,
+    "train_loss": 7.007378655998764e-07,
+    "train_runtime": 1568.7671,
     "train_samples": 11,
-    "train_samples_per_second": 0.268,
-    "train_steps_per_second": 0.017
+    "train_samples_per_second": 0.204,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 503.5625,
+      "completion_length": 479.21875,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.49610114097595215,
-      "kl": 0.001124829868786037,
+      "grad_norm": 0.497830331325531,
+      "kl": 0.0006894778707646765,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.7473476231098175,
-      "reward_std": 2.6333122961223125,
-      "rewards/concensus_correctness_reward_func": 0.6484999991953373,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.8361779022961855,
+      "reward_std": 0.9613135643303394,
+      "rewards/concensus_correctness_reward_func": 0.0010000000474974513,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5031913556158543,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5334903616458178,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.06440625432878733,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.05168749764561653,
       "step": 2
     },
     {
-      "completion_length": 383.9642857142857,
+      "completion_length": 331.42857142857144,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.6514167189598083,
-      "kl": 0.001142039312981069,
+      "grad_norm": 0.5410258173942566,
+      "kl": 0.0008775076047251267,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.6812755337783268,
-      "reward_std": 1.0149141805512565,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "reward": 1.1655291829790388,
+      "reward_std": 1.250451820237296,
+      "rewards/concensus_correctness_reward_func": 0.03278571353959186,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4729898337806974,
+      "rewards/question_recreation_reward_func": 0.5209220349788666,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0654285613979612,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.2725357147199767,
       "step": 4
     },
     {
-      "completion_length": 421.07142857142856,
+      "completion_length": 436.0,
       "epoch": 2.0,
-      "grad_norm": 12.148874282836914,
-      "kl": 0.001286752222638045,
+      "grad_norm": 0.41036298871040344,
+      "kl": 0.0007396088829929275,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.2420674732753207,
-      "reward_std": 2.832149956907545,
-      "rewards/concensus_correctness_reward_func": 0.7679999981607709,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.134751796722412,
+      "reward_std": 1.0689653626510076,
+      "rewards/concensus_correctness_reward_func": 0.06400000197546822,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14285714285714285,
-      "rewards/question_recreation_reward_func": 0.4981746668262141,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4305018394121102,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.38660714136702673,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.3723928523915155,
       "step": 6
     },
     {
-      "completion_length": 489.6875,
+      "completion_length": 388.8125,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.37291935086250305,
-      "kl": 0.0006464178877649829,
+      "grad_norm": 0.613455593585968,
+      "kl": 0.00076409201574279,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.0829366333782673,
-      "reward_std": 0.9148533008992672,
-      "rewards/concensus_correctness_reward_func": 0.003000000026077032,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.0549859162420034,
+      "reward_std": 0.9957783780992031,
+      "rewards/concensus_correctness_reward_func": 0.0020000000949949026,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4917178675532341,
+      "rewards/question_recreation_reward_func": 0.5592359164729714,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08821876253932714,
+      "rewards/xmlcount_reward_func": 0.24375000409781933,
       "step": 8
     },
     {
-      "completion_length": 370.92857142857144,
+      "completion_length": 389.2857142857143,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.5400116443634033,
-      "kl": 0.0009217662819927293,
+      "grad_norm": 0.4000310003757477,
+      "kl": 0.0007634159077757172,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.7083773016929626,
-      "reward_std": 1.2100985901696342,
-      "rewards/concensus_correctness_reward_func": 0.08057142581258502,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 1.1786940097808838,
+      "reward_std": 0.9912613758019039,
+      "rewards/concensus_correctness_reward_func": 0.045428569827760966,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.33619875247989384,
-      "rewards/soft_format_reward_func": 0.03571428571428571,
-      "rewards/strict_format_reward_func": 0.03571428571428571,
-      "rewards/xmlcount_reward_func": 0.4344642875449998,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.41876543100391117,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.3037857264280319,
       "step": 10
     },
     {
-      "completion_length": 463.35714285714283,
+      "completion_length": 454.42857142857144,
       "epoch": 4.0,
-      "grad_norm": 0.41703319549560547,
-      "kl": 0.0007593121678967561,
+      "grad_norm": 0.46324434876441956,
+      "kl": 0.0006662574347241648,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.9353206838880266,
-      "reward_std": 2.6510137830461775,
-      "rewards/concensus_correctness_reward_func": 2.1451428572514226,
-      "rewards/consensus_reward_func": 0.6428571428571429,
+      "reward": 1.3409278094768524,
+      "reward_std": 1.3151888506753104,
+      "rewards/concensus_correctness_reward_func": 0.08785714408648866,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.6259276547602245,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5007849793348994,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.21782144478389195,
+      "rewards/strict_format_reward_func": 0.03571428571428571,
+      "rewards/xmlcount_reward_func": 0.35942857393196653,
       "step": 12
     },
     {
-      "completion_length": 418.59375,
+      "completion_length": 427.84375,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.47336331009864807,
-      "kl": 0.0006999512879701797,
+      "grad_norm": 0.36418354511260986,
+      "kl": 0.0008765329112065956,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.120914263650775,
-      "reward_std": 1.043250784277916,
-      "rewards/concensus_correctness_reward_func": 0.11731250013690442,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.168391801416874,
+      "reward_std": 1.1325892731547356,
+      "rewards/concensus_correctness_reward_func": 0.051187500124797225,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5624454971402884,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5072980597615242,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1286562653258443,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.20365623105317354,
       "step": 14
     },
     {
-      "completion_length": 458.0357142857143,
+      "completion_length": 434.5,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.4212858974933624,
-      "kl": 0.0008645593355010663,
+      "grad_norm": 0.6217504143714905,
+      "kl": 0.0008525813214613923,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.513169629233224,
-      "reward_std": 2.3110418575150624,
-      "rewards/concensus_correctness_reward_func": 0.7142857142857143,
+      "reward": 0.936353611094611,
+      "reward_std": 0.8693441791193826,
+      "rewards/concensus_correctness_reward_func": 0.022714284913880483,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14285714285714285,
-      "rewards/question_recreation_reward_func": 0.516562534230096,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5036750175058842,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": -0.021250005013176372,
+      "rewards/strict_format_reward_func": 0.07142857142857142,
+      "rewards/xmlcount_reward_func": 0.1956785704408373,
       "step": 16
     },
     {
-      "completion_length": 421.17857142857144,
+      "completion_length": 435.32142857142856,
       "epoch": 6.0,
-      "grad_norm": 0.6799253225326538,
-      "kl": 0.0007702400159489896,
+      "grad_norm": 0.32569819688796997,
+      "kl": 0.000702967617793807,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.195361384323665,
-      "reward_std": 0.9645780750683376,
-      "rewards/concensus_correctness_reward_func": 0.026857141937528337,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 1.2448477915355138,
+      "reward_std": 1.0681161028998238,
+      "rewards/concensus_correctness_reward_func": 0.011214285822851318,
+      "rewards/consensus_reward_func": 0.35714285714285715,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44028993589537485,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
-      "rewards/strict_format_reward_func": 0.05357142857142857,
-      "rewards/xmlcount_reward_func": 0.44250001439026426,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.4135977657777922,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.3736071437597275,
       "step": 18
     },
     {
-      "completion_length": 419.375,
+      "completion_length": 396.90625,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.831315279006958,
-      "kl": 0.0008369779570784885,
+      "grad_norm": 0.40409594774246216,
+      "kl": 0.0007001970298006199,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 3.3066557571291924,
-      "reward_std": 2.3782889246940613,
-      "rewards/concensus_correctness_reward_func": 1.9004999992903322,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.109384085983038,
+      "reward_std": 0.938243392854929,
+      "rewards/concensus_correctness_reward_func": 0.009812500094994903,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.49509322270751,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5014465823769569,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2860625027678907,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27000000420957804,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.367493478544929e-07,
-      "train_runtime": 1195.3968,
-      "train_samples_per_second": 0.268,
-      "train_steps_per_second": 0.017
+      "train_loss": 7.007378655998764e-07,
+      "train_runtime": 1568.7671,
+      "train_samples_per_second": 0.204,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,