End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d25800d7bc59d92c4b2d8e2b6aba42b8be54cf98d510f821895243a50fd8c0a
+oid sha256:ce7d0b3a8ae3c96a75068a0fc3551c78ac1c789c1c12d3486e93a3ae28368467
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.66336280926771e-06,
-    "train_runtime": 594.3308,
+    "train_loss": 1.9629241478469338e-06,
+    "train_runtime": 602.2907,
     "train_samples": 11,
-    "train_samples_per_second": 0.538,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 0.531,
+    "train_steps_per_second": 0.033
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.66336280926771e-06,
-    "train_runtime": 594.3308,
+    "train_loss": 1.9629241478469338e-06,
+    "train_runtime": 602.2907,
     "train_samples": 11,
-    "train_samples_per_second": 0.538,
-    "train_steps_per_second": 0.034
+    "train_samples_per_second": 0.531,
+    "train_steps_per_second": 0.033
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 436.34375,
+      "completion_length": 438.6875,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.36141011118888855,
-      "kl": 0.0018547234940342605,
+      "grad_norm": 0.4479336142539978,
+      "kl": 0.00244811199081596,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.9109464138746262,
-      "reward_std": 2.558285780251026,
-      "rewards/concensus_correctness_reward_func": 0.6572499983012676,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.5811708644032478,
+      "reward_std": 1.1648125760257244,
+      "rewards/concensus_correctness_reward_func": 0.03687500022351742,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6333214156329632,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.22975000832229853,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.5910771004855633,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.2188437543809414,
       "step": 2
     },
     {
-      "completion_length": 396.67857142857144,
+      "completion_length": 450.17857142857144,
       "epoch": 1.3636363636363638,
-      "grad_norm": 0.4285108149051666,
-      "kl": 0.0018387458320440991,
+      "grad_norm": 0.3227308690547943,
+      "kl": 0.0019973710628359447,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.9300821934427534,
-      "reward_std": 2.889446326664516,
-      "rewards/concensus_correctness_reward_func": 1.4821428571428572,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.6385741915021624,
+      "reward_std": 0.5418220609426498,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.5762964018753597,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.5448599372591291,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.05357142857142857,
-      "rewards/xmlcount_reward_func": 0.3180714353386845,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.049142872648579736,
       "step": 4
     },
     {
-      "completion_length": 497.82142857142856,
+      "completion_length": 410.39285714285717,
       "epoch": 2.0,
-      "grad_norm": 0.4547726511955261,
-      "kl": 0.001723200240771153,
+      "grad_norm": 0.42811599373817444,
+      "kl": 0.0021256493138415472,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.4983462222984858,
-      "reward_std": 1.1972560052360808,
-      "rewards/concensus_correctness_reward_func": 0.180857138974326,
+      "reward": 1.1125848761626653,
+      "reward_std": 0.9187261845384326,
+      "rewards/concensus_correctness_reward_func": 0.012857143368039812,
       "rewards/consensus_reward_func": 0.42857142857142855,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6040247817124639,
+      "rewards/question_recreation_reward_func": 0.556370598929269,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.28489285707473755,
+      "rewards/xmlcount_reward_func": 0.11478571753416743,
       "step": 6
     },
     {
-      "completion_length": 431.5625,
+      "completion_length": 436.15625,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.4424420893192291,
-      "kl": 0.001982253073947504,
+      "grad_norm": 0.4794754981994629,
+      "kl": 0.0022159419168019667,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.8508164584636688,
-      "reward_std": 1.9796565789729357,
-      "rewards/concensus_correctness_reward_func": 0.6894999966025352,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.165166961029172,
+      "reward_std": 0.9042700827121735,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5295664072036743,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6742919702082872,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.24112499319016933,
+      "rewards/xmlcount_reward_func": 0.1627499908208847,
       "step": 8
     },
     {
-      "completion_length": 424.2142857142857,
+      "completion_length": 502.0,
       "epoch": 3.3636363636363638,
-      "grad_norm": 0.49577295780181885,
-      "kl": 0.0018019660076658642,
+      "grad_norm": 0.4946661591529846,
+      "kl": 0.002002335253304669,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.2227770473275865,
-      "reward_std": 1.1585649294512612,
-      "rewards/concensus_correctness_reward_func": 0.05357142857142857,
+      "reward": 1.0503257640770503,
+      "reward_std": 0.6738485034023013,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.14285714285714285,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5725984615939004,
+      "rewards/question_recreation_reward_func": 0.5972543060779572,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.4358928608042853,
+      "rewards/xmlcount_reward_func": 0.2923571446112224,
       "step": 10
     },
     {
-      "completion_length": 447.7142857142857,
+      "completion_length": 481.0357142857143,
       "epoch": 4.0,
-      "grad_norm": 0.3329751491546631,
-      "kl": 0.0018632897574986731,
+      "grad_norm": 0.3883371949195862,
+      "kl": 0.0022788067414824453,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.8000368965523583,
-      "reward_std": 2.461202246802194,
-      "rewards/concensus_correctness_reward_func": 0.7511428552014487,
+      "reward": 0.9479880375521523,
+      "reward_std": 1.2018876842090063,
+      "rewards/concensus_correctness_reward_func": 0.006428571684019906,
       "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.5562511852809361,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.6420594262225288,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20692857248442514,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": -0.11121428225721632,
       "step": 12
     },
     {
-      "completion_length": 419.84375,
+      "completion_length": 463.3125,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.4762893319129944,
-      "kl": 0.002035706871538423,
+      "grad_norm": 0.5984495878219604,
+      "kl": 0.001994655540329404,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.4218195006251335,
-      "reward_std": 1.2253588140010834,
-      "rewards/concensus_correctness_reward_func": 0.09375,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.2208586782217026,
+      "reward_std": 0.8369820192456245,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6008819788694382,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.653108686208725,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3990625096485019,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.2240000143647194,
       "step": 14
     },
     {
-      "completion_length": 449.82142857142856,
+      "completion_length": 432.10714285714283,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.4585653841495514,
-      "kl": 0.0018280741226460253,
+      "grad_norm": 0.4461248219013214,
+      "kl": 0.0022216635323794825,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.1226869395801,
-      "reward_std": 2.7912509611674716,
-      "rewards/concensus_correctness_reward_func": 0.8290000004427773,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 1.0449265113898687,
+      "reward_std": 0.8119848519563675,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.07142857142857142,
-      "rewards/question_recreation_reward_func": 0.6304370207445962,
+      "rewards/question_recreation_reward_func": 0.4264265330774443,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16325000673532486,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.31492857209273745,
       "step": 16
     },
     {
-      "completion_length": 530.6785714285714,
+      "completion_length": 446.5357142857143,
       "epoch": 6.0,
-      "grad_norm": 0.3060424327850342,
-      "kl": 0.0014743279898539186,
+      "grad_norm": 0.35580751299858093,
+      "kl": 0.0020483574563903467,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.46565814529146465,
-      "reward_std": 1.046275360243661,
+      "reward": 1.2445995807647705,
+      "reward_std": 1.3314554265567236,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.5714285714285714,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.505515279514449,
+      "rewards/final_correctness_reward_func": 0.07142857142857142,
+      "rewards/question_recreation_reward_func": 0.482528167111533,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.039857140609196255,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.10135714603321892,
       "step": 18
     },
     {
-      "completion_length": 491.15625,
+      "completion_length": 503.75,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.37568986415863037,
-      "kl": 0.0014474634008365683,
+      "grad_norm": 0.4132925570011139,
+      "kl": 0.0020758258033310995,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.8635792769491673,
-      "reward_std": 0.9761131927371025,
-      "rewards/concensus_correctness_reward_func": 0.02225000038743019,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.7664957279339433,
+      "reward_std": 1.0467706695199013,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4623605003580451,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.572433227673173,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3008437482640147,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.24343749973922968,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.66336280926771e-06,
-      "train_runtime": 594.3308,
-      "train_samples_per_second": 0.538,
-      "train_steps_per_second": 0.034
+      "train_loss": 1.9629241478469338e-06,
+      "train_runtime": 602.2907,
+      "train_samples_per_second": 0.531,
+      "train_steps_per_second": 0.033
     }
   ],
   "logging_steps": 2,