End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7530f191c149f7b0d5292a235d969ce2ad02467df8506e8ed61fea7fca99ae6b
+oid sha256:2c961420335e1c924b9cf652210b9e740ef814366585a51c5f199e55c91f3870
 size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.039206534249387e-07,
-    "train_runtime": 1696.5238,
+    "train_loss": 7.587115760543383e-07,
+    "train_runtime": 1705.0529,
     "train_samples": 14,
-    "train_samples_per_second": 0.189,
+    "train_samples_per_second": 0.188,
     "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.039206534249387e-07,
-    "train_runtime": 1696.5238,
+    "train_loss": 7.587115760543383e-07,
+    "train_runtime": 1705.0529,
     "train_samples": 14,
-    "train_samples_per_second": 0.189,
+    "train_samples_per_second": 0.188,
     "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 355.34375,
+      "completion_length": 347.9375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 0.4525608420372009,
-      "kl": 0.0007631201224285178,
+      "grad_norm": 0.5356497168540955,
+      "kl": 0.000872969270858448,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 1.244677685201168,
-      "reward_std": 1.107690967619419,
-      "rewards/concensus_correctness_reward_func": 0.023250000202097,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 1.167648907750845,
+      "reward_std": 1.1919554769992828,
+      "rewards/concensus_correctness_reward_func": 0.12287499941885471,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.47730267606675625,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5234926212579012,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2441249992698431,
+      "rewards/xmlcount_reward_func": 0.2087812628597021,
       "step": 2
     },
     {
-      "completion_length": 246.16666666666666,
+      "completion_length": 235.58333333333334,
       "epoch": 1.0,
-      "grad_norm": 0.693422794342041,
-      "kl": 0.001025922130793333,
+      "grad_norm": 0.37739747762680054,
+      "kl": 0.0007977873416772733,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 4.679427087306976,
-      "reward_std": 4.66946305334568,
-      "rewards/concensus_correctness_reward_func": 2.991499947384,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 11.468140810728073,
+      "reward_std": 7.96479082852602,
+      "rewards/concensus_correctness_reward_func": 8.333333333333334,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.48263533289233845,
+      "rewards/final_correctness_reward_func": 1.25,
+      "rewards/question_recreation_reward_func": 0.566015842060248,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3719583402077357,
+      "rewards/xmlcount_reward_func": 0.4854583491881688,
       "step": 4
     },
     {
-      "completion_length": 356.53125,
+      "completion_length": 303.59375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 0.6480845212936401,
-      "kl": 0.0009030676665133797,
+      "grad_norm": 0.6155076026916504,
+      "kl": 0.0009255852637579665,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.5064457170665264,
-      "reward_std": 0.656183360144496,
-      "rewards/concensus_correctness_reward_func": 2.525937500875443,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 3.4901900067925453,
+      "reward_std": 2.660517193377018,
+      "rewards/concensus_correctness_reward_func": 1.9703124985098839,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4468831270933151,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.43297142535448074,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2211250076070428,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3837812524288893,
       "step": 6
     },
     {
-      "completion_length": 231.91666666666666,
+      "completion_length": 284.3333333333333,
       "epoch": 2.0,
-      "grad_norm": 0.46470630168914795,
-      "kl": 0.0009404434337435911,
+      "grad_norm": 0.33494114875793457,
+      "kl": 0.0008494727759777257,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 4.598389546076457,
-      "reward_std": 2.8161835273106894,
-      "rewards/concensus_correctness_reward_func": 2.592499998087684,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 6.63089241584142,
+      "reward_std": 6.649104615052541,
+      "rewards/concensus_correctness_reward_func": 4.177083333333333,
+      "rewards/consensus_reward_func": 0.6666666666666666,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5833333333333334,
-      "rewards/question_recreation_reward_func": 0.4207228670517604,
+      "rewards/final_correctness_reward_func": 0.8333333333333334,
+      "rewards/question_recreation_reward_func": 0.47814221680164337,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5018333295981089,
+      "rewards/xmlcount_reward_func": 0.4756666806836923,
       "step": 8
     },
     {
-      "completion_length": 307.21875,
+      "completion_length": 296.71875,
       "epoch": 2.571428571428571,
-      "grad_norm": 7.176942348480225,
-      "kl": 0.000870315772772301,
+      "grad_norm": 0.6769649982452393,
+      "kl": 0.00095233663887484,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.048548925668001,
-      "reward_std": 2.1782662458717823,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 3.9268490821123123,
+      "reward_std": 3.860744908452034,
+      "rewards/concensus_correctness_reward_func": 1.9869375247508287,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.4979239506646991,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5795054230839014,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3474999964237213,
+      "rewards/xmlcount_reward_func": 0.3447812532540411,
       "step": 10
     },
     {
-      "completion_length": 270.125,
+      "completion_length": 272.3333333333333,
       "epoch": 3.0,
-      "grad_norm": 0.5190132260322571,
-      "kl": 0.000891704325719426,
+      "grad_norm": 0.36784347891807556,
+      "kl": 0.0010108925537982334,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.5302210549513497,
-      "reward_std": 3.810038983821869,
-      "rewards/concensus_correctness_reward_func": 1.8356666633238394,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 4.578291356563568,
+      "reward_std": 3.1783340821663537,
+      "rewards/concensus_correctness_reward_func": 2.742249995470047,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.4175960123538971,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.39724966883659363,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.27695833146572113,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.5846250007549921,
       "step": 12
     },
     {
-      "completion_length": 319.84375,
+      "completion_length": 335.53125,
       "epoch": 3.571428571428571,
-      "grad_norm": 0.4634810984134674,
-      "kl": 0.000814613900729455,
+      "grad_norm": 0.46538424491882324,
+      "kl": 0.0007929047133075073,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.5906156599521637,
-      "reward_std": 2.001572407782078,
-      "rewards/concensus_correctness_reward_func": 0.8713124996284023,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 7.037881502881646,
+      "reward_std": 6.554215002804995,
+      "rewards/concensus_correctness_reward_func": 5.057874999940395,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.625,
-      "rewards/question_recreation_reward_func": 0.5372093431651592,
+      "rewards/question_recreation_reward_func": 0.5687252599745989,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.338343751616776,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16128124482929707,
       "step": 14
     },
     {
-      "completion_length": 344.0,
+      "completion_length": 267.2916666666667,
       "epoch": 4.0,
-      "grad_norm": 0.2937107980251312,
-      "kl": 0.0008087421010714024,
+      "grad_norm": 0.366767555475235,
+      "kl": 0.0009911478749321152,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 4.022416894634564,
-      "reward_std": 2.649322579304377,
-      "rewards/concensus_correctness_reward_func": 2.5197499996672073,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.2440696557362876,
+      "reward_std": 3.304353336493174,
+      "rewards/concensus_correctness_reward_func": 1.7609166651964188,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.45241697629292804,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.3033614456653595,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.2794166716436545,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5131250148018202,
       "step": 16
     },
     {
-      "completion_length": 277.5625,
+      "completion_length": 321.5,
       "epoch": 4.571428571428571,
-      "grad_norm": 0.5213144421577454,
-      "kl": 0.0009557601952110417,
+      "grad_norm": 0.6150040626525879,
+      "kl": 0.000730342733731959,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 4.3840112537145615,
-      "reward_std": 3.753405824303627,
-      "rewards/concensus_correctness_reward_func": 2.700999985449016,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 4.889002770185471,
+      "reward_std": 3.9090433791279793,
+      "rewards/concensus_correctness_reward_func": 2.761312510818243,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5050424672663212,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.5080652236938477,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17796874605119228,
+      "rewards/xmlcount_reward_func": 0.24462500400841236,
       "step": 18
     },
     {
-      "completion_length": 295.25,
+      "completion_length": 279.125,
       "epoch": 5.0,
-      "grad_norm": 0.4308670461177826,
-      "kl": 0.000916544619637231,
+      "grad_norm": 0.4653594195842743,
+      "kl": 0.0008820409517890463,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.2748198676854372,
-      "reward_std": 0.6756910532712936,
-      "rewards/concensus_correctness_reward_func": -0.009500000004967054,
-      "rewards/consensus_reward_func": 0.08333333333333333,
+      "reward": 5.272506097952525,
+      "reward_std": 5.008866022030513,
+      "rewards/concensus_correctness_reward_func": 3.3964999988675117,
+      "rewards/consensus_reward_func": 0.5833333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.31840314840277034,
+      "rewards/final_correctness_reward_func": 0.5833333333333334,
+      "rewards/question_recreation_reward_func": 0.35333968202273053,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.549249991774559,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.3351666610687971,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.039206534249387e-07,
-      "train_runtime": 1696.5238,
-      "train_samples_per_second": 0.189,
+      "train_loss": 7.587115760543383e-07,
+      "train_runtime": 1705.0529,
+      "train_samples_per_second": 0.188,
       "train_steps_per_second": 0.012
     }
   ],