End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f2b825de5c4d57d40a7bc1380030819a4905b6a286892a59f092b09182683ea
+oid sha256:787c1b6870872275de71786492e664a8a01ff80e6d13dde1255058a3e01a776f
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0758711596281501e-06,
-    "train_runtime": 1457.5295,
+    "train_loss": 7.601985771543696e-07,
+    "train_runtime": 1270.4185,
     "train_samples": 11,
-    "train_samples_per_second": 0.22,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.252,
+    "train_steps_per_second": 0.016
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.0758711596281501e-06,
-    "train_runtime": 1457.5295,
+    "train_loss": 7.601985771543696e-07,
+    "train_runtime": 1270.4185,
     "train_samples": 11,
-    "train_samples_per_second": 0.22,
-    "train_steps_per_second": 0.014
+    "train_samples_per_second": 0.252,
+    "train_steps_per_second": 0.016
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 446.9375,
+      "completion_length": 511.71875,
       "epoch": 0.7272727272727273,
-      "grad_norm": 0.4233318567276001,
-      "kl": 0.002272412384627387,
+      "grad_norm": 0.39067885279655457,
+      "kl": 0.0008286302181659266,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.0939345434308052,
-      "reward_std": 0.9673900343477726,
-      "rewards/concensus_correctness_reward_func": 0.028937500435858965,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 0.5964680747129023,
+      "reward_std": 0.9820734895765781,
+      "rewards/concensus_correctness_reward_func": 0.04281250014901161,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4613720290362835,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4579368159174919,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16612500604242086,
+      "rewards/xmlcount_reward_func": -0.02928124601021409,
       "step": 2
     },
     {
-      "completion_length": 463.2857142857143,
+      "completion_length": 471.07142857142856,
       "epoch": 1.3636363636363638,
-      "grad_norm": 5.972531795501709,
-      "kl": 0.0009301399861994598,
+      "grad_norm": 0.3098437190055847,
+      "kl": 0.0006938177726364561,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.0194146164825983,
-      "reward_std": 0.8519903314965112,
+      "reward": 1.147034900529044,
+      "reward_std": 0.6715688492570605,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.14285714285714285,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6006288954189846,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.6506777490888324,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2759285888501576,
+      "rewards/xmlcount_reward_func": 0.13921428897551127,
       "step": 4
     },
     {
-      "completion_length": 463.67857142857144,
+      "completion_length": 421.07142857142856,
       "epoch": 2.0,
-      "grad_norm": 0.36018437147140503,
-      "kl": 0.0008266212368783142,
+      "grad_norm": 0.455415278673172,
+      "kl": 0.0008461131448192256,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.669443880873067,
-      "reward_std": 0.7500257960387638,
-      "rewards/concensus_correctness_reward_func": 0.005642857402563095,
-      "rewards/consensus_reward_func": 0.21428571428571427,
+      "reward": 0.5566586937223162,
+      "reward_std": 1.0844121660505022,
+      "rewards/concensus_correctness_reward_func": 0.027214286582810537,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.44790816200631006,
+      "rewards/question_recreation_reward_func": 0.3870158685105188,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.001607143453189305,
+      "rewards/xmlcount_reward_func": -0.14328572366918838,
       "step": 6
     },
     {
-      "completion_length": 482.625,
+      "completion_length": 458.1875,
       "epoch": 2.7272727272727275,
-      "grad_norm": 0.4127800762653351,
-      "kl": 0.0008769522100919858,
+      "grad_norm": 0.6082576513290405,
+      "kl": 0.0008534363078069873,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.102904200553894,
-      "reward_std": 1.1630526334047318,
+      "reward": 0.6228686403483152,
+      "reward_std": 0.912500187754631,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.3125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5856854561716318,
+      "rewards/question_recreation_reward_func": 0.5011811307631433,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.17346875835210085,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.0435625072568655,
       "step": 8
     },
     {
-      "completion_length": 477.60714285714283,
+      "completion_length": 496.9642857142857,
       "epoch": 3.3636363636363638,
-      "grad_norm": 4.251033306121826,
-      "kl": 0.001116385533740478,
+      "grad_norm": 0.948233962059021,
+      "kl": 0.0009808058052190713,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.9688816113131387,
-      "reward_std": 1.0132532545498438,
-      "rewards/concensus_correctness_reward_func": 0.027428571666990007,
-      "rewards/consensus_reward_func": 0.42857142857142855,
+      "reward": 0.9556317691292081,
+      "reward_std": 0.8779624657971519,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4807387867144176,
+      "rewards/question_recreation_reward_func": 0.53256037405559,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03214285948446819,
+      "rewards/strict_format_reward_func": 0.05357142857142857,
+      "rewards/xmlcount_reward_func": 0.15521428414753505,
       "step": 10
     },
     {
-      "completion_length": 433.75,
+      "completion_length": 502.07142857142856,
       "epoch": 4.0,
-      "grad_norm": 0.4217379093170166,
-      "kl": 0.001201488310471177,
+      "grad_norm": 0.4401586949825287,
+      "kl": 0.0007997392427309283,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.2074503217424666,
-      "reward_std": 1.034340739250183,
+      "reward": 0.8197848323200431,
+      "reward_std": 0.7523607734058585,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.35714285714285715,
+      "rewards/consensus_reward_func": 0.2857142857142857,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.40602174614156994,
+      "rewards/question_recreation_reward_func": 0.3942848172570978,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.42642856602157864,
+      "rewards/xmlcount_reward_func": 0.12192857159035546,
       "step": 12
     },
     {
-      "completion_length": 492.0,
+      "completion_length": 416.46875,
       "epoch": 4.7272727272727275,
-      "grad_norm": 0.751480758190155,
-      "kl": 0.001145575792179443,
+      "grad_norm": 0.7907354831695557,
+      "kl": 0.0008191062152036466,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.3288952857255936,
-      "reward_std": 0.8053327365778387,
+      "reward": 1.2800959646701813,
+      "reward_std": 0.8047939604148269,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.5,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5272390320897102,
+      "rewards/question_recreation_reward_func": 0.495877243578434,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30165624246001244,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.20609374903142452,
       "step": 14
     },
     {
-      "completion_length": 502.10714285714283,
+      "completion_length": 508.32142857142856,
       "epoch": 5.363636363636363,
-      "grad_norm": 0.45700931549072266,
-      "kl": 0.0007722628402656742,
+      "grad_norm": 0.4276641607284546,
+      "kl": 0.0006800852133892477,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.8481947886092323,
-      "reward_std": 0.8680226675101689,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.8740463554859161,
+      "reward_std": 0.9160092515604836,
+      "rewards/concensus_correctness_reward_func": 0.027214286582810537,
       "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5282662468297141,
+      "rewards/question_recreation_reward_func": 0.37847493695361273,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10564285516738892,
+      "rewards/strict_format_reward_func": 0.017857142857142856,
+      "rewards/xmlcount_reward_func": 0.23621429290090287,
       "step": 16
     },
     {
-      "completion_length": 408.0357142857143,
+      "completion_length": 475.75,
       "epoch": 6.0,
-      "grad_norm": 0.5316712260246277,
-      "kl": 0.0015365564114680247,
+      "grad_norm": 2.526139736175537,
+      "kl": 0.0008833917007515473,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.0890584758349828,
-      "reward_std": 0.9097863137722015,
-      "rewards/concensus_correctness_reward_func": 0.005642857402563095,
+      "reward": 0.6509425001485007,
+      "reward_std": 1.037146440574101,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.21428571428571427,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5976299004895347,
-      "rewards/soft_format_reward_func": 0.017857142857142856,
+      "rewards/question_recreation_reward_func": 0.5270139213119235,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.017857142857142856,
-      "rewards/xmlcount_reward_func": 0.23578572060380662,
+      "rewards/xmlcount_reward_func": -0.10821428469249181,
       "step": 18
     },
     {
-      "completion_length": 512.625,
+      "completion_length": 509.5,
       "epoch": 6.7272727272727275,
-      "grad_norm": 0.35905343294143677,
-      "kl": 0.0009294488772866316,
+      "grad_norm": 0.34697097539901733,
+      "kl": 0.0007815364369889721,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.6413433700799942,
-      "reward_std": 0.8869485668838024,
+      "reward": 0.924950797110796,
+      "reward_std": 1.0309381671249866,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.46356214582920074,
+      "rewards/question_recreation_reward_func": 0.5376695655286312,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": -0.025343740358948708,
+      "rewards/xmlcount_reward_func": 0.05915623530745506,
       "step": 20
     },
     {
       "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.0758711596281501e-06,
-      "train_runtime": 1457.5295,
-      "train_samples_per_second": 0.22,
-      "train_steps_per_second": 0.014
+      "train_loss": 7.601985771543696e-07,
+      "train_runtime": 1270.4185,
+      "train_samples_per_second": 0.252,
+      "train_steps_per_second": 0.016
     }
   ],
   "logging_steps": 2,