End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd28cc28fecdb49bad70398ac84a1466b2c6d1172983630c06940f2a74f141d7
+oid sha256:342bf129597fc53cd394b64a7ddd7e395b4338b4c9243225fe2edd2bd323b1b6
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.651437158529006e-07,
-    "train_runtime": 204.8374,
+    "train_loss": 8.326966678851022e-07,
+    "train_runtime": 209.7465,
     "train_samples": 14,
-    "train_samples_per_second": 1.562,
-    "train_steps_per_second": 0.098
+    "train_samples_per_second": 1.526,
+    "train_steps_per_second": 0.095
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 7.651437158529006e-07,
-    "train_runtime": 204.8374,
+    "train_loss": 8.326966678851022e-07,
+    "train_runtime": 209.7465,
     "train_samples": 14,
-    "train_samples_per_second": 1.562,
-    "train_steps_per_second": 0.098
+    "train_samples_per_second": 1.526,
+    "train_steps_per_second": 0.095
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 281.84375,
+      "completion_length": 290.3125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.5356741547584534,
-      "kl": 0.0008984249216155149,
+      "grad_norm": 0.710826575756073,
+      "kl": 0.0007815487479092553,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.624540038406849,
-      "reward_std": 0.9768942464143038,
-      "rewards/concensus_correctness_reward_func": 0.18112499453127384,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.2464086189866066,
+      "reward_std": 1.0344440340995789,
+      "rewards/concensus_correctness_reward_func": 0.1236874982714653,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.39663382805883884,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.44237736240029335,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5467812605202198,
+      "rewards/xmlcount_reward_func": 0.3053437464404851,
       "step": 2
     },
     {
-      "completion_length": 273.2083333333333,
+      "completion_length": 287.4583333333333,
       "epoch": 1.0,
-      "grad_norm": 0.4353300929069519,
-      "kl": 0.0009453818202018738,
+      "grad_norm": 0.7425183653831482,
+      "kl": 0.0009009348868858069,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.93053005139033,
-      "reward_std": 1.1497112214565277,
-      "rewards/concensus_correctness_reward_func": 0.10149999459584554,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.7921818892161052,
+      "reward_std": 0.7107179313898087,
+      "rewards/concensus_correctness_reward_func": -0.003416667381922404,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.5022799869378408,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.6228068619966507,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.410083328684171,
+      "rewards/xmlcount_reward_func": 0.506125013033549,
       "step": 4
     },
     {
-      "completion_length": 300.0625,
+      "completion_length": 331.6875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.5228692889213562,
-      "kl": 0.0008096267629298382,
+      "grad_norm": 0.4491569697856903,
+      "kl": 0.0009744477138156071,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.3409306183457375,
-      "reward_std": 2.157092723995447,
-      "rewards/concensus_correctness_reward_func": 0.7595624979585409,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.0220835860818624,
+      "reward_std": 0.8010594584047794,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.5241494122892618,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.44264610670506954,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4322187453508377,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3138125156983733,
       "step": 6
     },
     {
-      "completion_length": 281.375,
+      "completion_length": 253.70833333333334,
       "epoch": 2.0,
-      "grad_norm": 0.34011420607566833,
-      "kl": 0.0008021329316155364,
+      "grad_norm": 0.38829076290130615,
+      "kl": 0.0009692620466618488,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.9659400979677835,
-      "reward_std": 3.513501524925232,
-      "rewards/concensus_correctness_reward_func": 2.0618333319822946,
+      "reward": 1.4584412674109142,
+      "reward_std": 1.0012017289797466,
+      "rewards/concensus_correctness_reward_func": 0.04099999740719795,
       "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5879400198658308,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.45281629512707394,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.39950000246365863,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.4437916725873947,
       "step": 8
     },
     {
-      "completion_length": 313.4375,
+      "completion_length": 308.9375,
       "epoch": 2.571428571428571,
-      "grad_norm": 0.5303735733032227,
-      "kl": 0.0008135192110785283,
+      "grad_norm": 0.49039143323898315,
+      "kl": 0.0008223845798056573,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.412702314555645,
-      "reward_std": 2.936381496489048,
-      "rewards/concensus_correctness_reward_func": 0.8495624996721745,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.2423480860888958,
+      "reward_std": 0.783745139837265,
+      "rewards/concensus_correctness_reward_func": 0.047875000862404704,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.552046075463295,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5324730686843395,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.44859375432133675,
+      "rewards/xmlcount_reward_func": 0.34949999302625656,
       "step": 10
     },
     {
-      "completion_length": 278.25,
+      "completion_length": 309.2083333333333,
       "epoch": 3.0,
-      "grad_norm": 0.6402037739753723,
-      "kl": 0.0009136535906388114,
+      "grad_norm": 0.3918669521808624,
+      "kl": 0.001134051368959869,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.114620971182982,
-      "reward_std": 4.722743009527524,
-      "rewards/concensus_correctness_reward_func": 1.7004999990264575,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 1.570088232556979,
+      "reward_std": 1.027662754058838,
+      "rewards/concensus_correctness_reward_func": 0.014833331108093262,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.6666666666666666,
-      "rewards/question_recreation_reward_func": 0.3201208959023158,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.46129654844601947,
+      "rewards/soft_format_reward_func": 0.020833333333333332,
       "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.15650000174840292,
+      "rewards/xmlcount_reward_func": 0.3022916528085868,
       "step": 12
     },
     {
-      "completion_length": 314.34375,
+      "completion_length": 305.65625,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.7036963105201721,
-      "kl": 0.0009236924997821916,
+      "grad_norm": 0.5455784201622009,
+      "kl": 0.000909568763745483,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.5727680698037148,
-      "reward_std": 2.339424160309136,
-      "rewards/concensus_correctness_reward_func": 0.6898125000298023,
+      "reward": 1.0659725815057755,
+      "reward_std": 0.7342658238485456,
+      "rewards/concensus_correctness_reward_func": 0.033874999498948455,
       "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.5561118274927139,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5496913269162178,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.5455937571823597,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.27928124740719795,
       "step": 14
     },
     {
-      "completion_length": 306.2916666666667,
+      "completion_length": 298.0416666666667,
       "epoch": 4.0,
-      "grad_norm": 0.3896789252758026,
-      "kl": 0.0008742265151037524,
+      "grad_norm": 0.39558833837509155,
+      "kl": 0.0010590991975429158,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.8965884745121,
-      "reward_std": 2.90642241636912,
-      "rewards/concensus_correctness_reward_func": 1.7618333299954732,
-      "rewards/consensus_reward_func": 0.3333333333333333,
+      "reward": 2.231153041124344,
+      "reward_std": 2.5654935811956725,
+      "rewards/concensus_correctness_reward_func": 0.8084166646003723,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.2658801128466924,
+      "rewards/question_recreation_reward_func": 0.3254447455207507,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.2647083435828487,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3472916707396507,
       "step": 16
     },
     {
-      "completion_length": 281.15625,
+      "completion_length": 294.40625,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.5173717737197876,
-      "kl": 0.0008403044703300111,
+      "grad_norm": 0.6365107297897339,
+      "kl": 0.0011505173606565222,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 6.907710582017899,
-      "reward_std": 5.539019919931889,
-      "rewards/concensus_correctness_reward_func": 4.515749996528029,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 1.119580751284957,
+      "reward_std": 1.0246936902403831,
+      "rewards/concensus_correctness_reward_func": 0.05025000008754432,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.5219604447484016,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.46676828525960445,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.49500001315027475,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3838124983012676,
       "step": 18
     },
     {
-      "completion_length": 273.4583333333333,
+      "completion_length": 311.0833333333333,
       "epoch": 5.0,
-      "grad_norm": 0.3363378047943115,
-      "kl": 0.0009138845295334855,
+      "grad_norm": 0.448214054107666,
+      "kl": 0.0008349110188040262,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.9740705688794453,
-      "reward_std": 1.3407134215037029,
-      "rewards/concensus_correctness_reward_func": 0.13033332924048105,
-      "rewards/consensus_reward_func": 0.4166666666666667,
+      "reward": 2.4208129569888115,
+      "reward_std": 2.596543009082476,
+      "rewards/concensus_correctness_reward_func": 0.820583333571752,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4434455931186676,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.49702121565739316,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.7961250245571136,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.18654166534543037,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 7.651437158529006e-07,
-      "train_runtime": 204.8374,
-      "train_samples_per_second": 1.562,
-      "train_steps_per_second": 0.098
+      "train_loss": 8.326966678851022e-07,
+      "train_runtime": 209.7465,
+      "train_samples_per_second": 1.526,
+      "train_steps_per_second": 0.095
     }
   ],
   "logging_steps": 2,