End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.795698285102844e-06,
-    "train_runtime": 888.0539,
+    "train_loss": 2.3273200440598886e-06,
+    "train_runtime": 1372.7943,
     "train_samples": 28,
-    "train_samples_per_second": 0.36,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 0.233,
+    "train_steps_per_second": 0.015
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:716c51f6b908026e2fb48353189918f547eed630b088a9f652264811eedb01b2
+oid sha256:d71038f168b8aa200ae7cb43ccaa712c5886315f8cc7af1b64c5f29f5e5daa00
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.795698285102844e-06,
-    "train_runtime": 888.0539,
+    "train_loss": 2.3273200440598886e-06,
+    "train_runtime": 1372.7943,
     "train_samples": 28,
-    "train_samples_per_second": 0.36,
-    "train_steps_per_second": 0.023
+    "train_samples_per_second": 0.233,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 375.6875,
+      "completion_length": 289.28125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 8.718708992004395,
+      "grad_norm": 6.842979431152344,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.0768319834023714,
-      "reward_std": 1.5650975175230997,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.7364051938639022,
+      "reward_std": 0.5221564862295054,
+      "rewards/concensus_correctness_reward_func": 0.03368750028312206,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.26595700718462467,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.26012394070858136,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0016250004991889,
+      "rewards/xmlcount_reward_func": 0.25509375100955367,
       "step": 2
     },
     {
-      "completion_length": 329.25,
+      "completion_length": 318.9583333333333,
       "epoch": 1.0,
-      "grad_norm": 5.104424476623535,
-      "kl": 0.0013852004000606637,
+      "grad_norm": 4.884227752685547,
+      "kl": 0.0009200210721852878,
       "learning_rate": 4.864543104251586e-07,
-      "loss": -0.0,
-      "reward": 0.6561937442359825,
-      "reward_std": 0.41659088157697016,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 0.5973201192294558,
+      "reward_std": 0.4795330392080359,
+      "rewards/concensus_correctness_reward_func": 0.03033333271741867,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.36636040002728504,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2964034474765261,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03983332713445028,
+      "rewards/xmlcount_reward_func": 0.27058333344757557,
       "step": 4
     },
     {
-      "completion_length": 463.3125,
+      "completion_length": 357.0,
       "epoch": 1.5714285714285714,
-      "grad_norm": 5.5438008308410645,
-      "kl": 0.0020392315836943453,
+      "grad_norm": 27.056697845458984,
+      "kl": 0.001567345678267884,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.9918126103002578,
-      "reward_std": 1.2127672019269085,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.397412788355723,
+      "reward_std": 1.0327481220738264,
+      "rewards/concensus_correctness_reward_func": 0.05624999850988388,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.2850313719827682,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.30856906017288566,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.10571874887682498,
+      "rewards/xmlcount_reward_func": -0.029906259791459888,
       "step": 6
     },
     {
-      "completion_length": 450.0,
+      "completion_length": 345.2083333333333,
       "epoch": 2.0,
-      "grad_norm": 4.290306568145752,
-      "kl": 0.0052049736113986,
+      "grad_norm": 2.828078031539917,
+      "kl": 0.0016559154222098489,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.7020065200825533,
-      "reward_std": 0.9027306487162908,
-      "rewards/concensus_correctness_reward_func": 0.1305833359559377,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9617004801208774,
+      "reward_std": 0.5262383962981403,
+      "rewards/concensus_correctness_reward_func": 0.022583333154519398,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.33833985899885494,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.3627421515993774,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14975000793735185,
+      "rewards/xmlcount_reward_func": 0.3263749995579322,
       "step": 8
     },
     {
-      "completion_length": 390.21875,
+      "completion_length": 325.71875,
       "epoch": 2.571428571428571,
-      "grad_norm": 10.614115715026855,
-      "kl": 0.002438371942844242,
+      "grad_norm": 14.418530464172363,
+      "kl": 0.003461128544586245,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.8925170208094642,
-      "reward_std": 0.6647714290884323,
-      "rewards/concensus_correctness_reward_func": 0.09793750196695328,
+      "reward": 0.3613777115242556,
+      "reward_std": 0.4356158603914082,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.2163607826223597,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.32094020998920314,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3282187497243285,
+      "rewards/xmlcount_reward_func": 0.04043750301934779,
       "step": 10
     },
     {
-      "completion_length": 337.5416666666667,
+      "completion_length": 381.4166666666667,
       "epoch": 3.0,
-      "grad_norm": 5.469453811645508,
-      "kl": 0.011523220338858664,
+      "grad_norm": 8.072898864746094,
+      "kl": 0.0024351389050328485,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.9385191053152084,
-      "reward_std": 1.1355751731122534,
-      "rewards/concensus_correctness_reward_func": 0.03566666692495346,
+      "reward": 0.9499049770335356,
+      "reward_std": 1.0589883048087358,
+      "rewards/concensus_correctness_reward_func": 0.10408333192269008,
       "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.42685242121418315,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.4167799516580999,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.22600000848372778,
+      "rewards/xmlcount_reward_func": 0.26237500086426735,
       "step": 12
     },
     {
-      "completion_length": 373.09375,
+      "completion_length": 374.65625,
       "epoch": 3.571428571428571,
-      "grad_norm": 8.957090377807617,
-      "kl": 0.004819704467081465,
+      "grad_norm": 5.15366792678833,
+      "kl": 0.004279123459127732,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 1.161799500521738,
-      "reward_std": 1.5693250097101554,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.7568356324918568,
+      "reward_std": 0.8572800485417247,
+      "rewards/concensus_correctness_reward_func": 0.0175624992698431,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.300299494585488,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4311481327749789,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04899999685585499,
+      "rewards/xmlcount_reward_func": 0.24562499951571226,
       "step": 14
     },
     {
-      "completion_length": 302.4166666666667,
+      "completion_length": 361.4583333333333,
       "epoch": 4.0,
-      "grad_norm": 4.58036994934082,
-      "kl": 0.01090930521604605,
+      "grad_norm": 3.1721510887145996,
+      "kl": 0.006174589856527746,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.801399719590942,
-      "reward_std": 0.8878546565926323,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.31145076857258874,
+      "reward_std": 0.5888905140260855,
+      "rewards/concensus_correctness_reward_func": 0.04975000023841858,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.35802473189930123,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.31557577041288215,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1933749926586946,
+      "rewards/xmlcount_reward_func": -0.1372083347911636,
       "step": 16
     },
     {
-      "completion_length": 341.0,
+      "completion_length": 345.4375,
       "epoch": 4.571428571428571,
-      "grad_norm": 6.268459796905518,
-      "kl": 0.006092046292906161,
+      "grad_norm": 10.782058715820312,
+      "kl": 0.003293925830803346,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.0851099799619988,
-      "reward_std": 1.165190162428189,
-      "rewards/concensus_correctness_reward_func": 0.7229375019669533,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9296363415196538,
+      "reward_std": 0.794563317351276,
+      "rewards/concensus_correctness_reward_func": 0.13431249745190144,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.27095376315992326,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.34785505570471287,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03378124814480543,
+      "rewards/xmlcount_reward_func": 0.2599687548354268,
       "step": 18
     },
     {
-      "completion_length": 336.75,
+      "completion_length": 335.3333333333333,
       "epoch": 5.0,
-      "grad_norm": 5.2020440101623535,
-      "kl": 0.0050251311719572795,
+      "grad_norm": 11.1777982711792,
+      "kl": 0.003027383572771214,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.7938545711028079,
-      "reward_std": 0.8512527600105386,
-      "rewards/concensus_correctness_reward_func": 0.1572500024922192,
-      "rewards/consensus_reward_func": 0.16666666666666666,
+      "reward": 1.3776022079400718,
+      "reward_std": 1.4989564453426283,
+      "rewards/concensus_correctness_reward_func": 0.8520000005761782,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.22452121305589876,
+      "rewards/question_recreation_reward_func": 0.34335220186039805,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16208333366860947,
+      "rewards/xmlcount_reward_func": 0.0989166609942913,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.795698285102844e-06,
-      "train_runtime": 888.0539,
-      "train_samples_per_second": 0.36,
-      "train_steps_per_second": 0.023
+      "train_loss": 2.3273200440598886e-06,
+      "train_runtime": 1372.7943,
+      "train_samples_per_second": 0.233,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 2,