End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 1040.7425,
-    "train_samples": 2,
-    "train_samples_per_second": 0.038,
-    "train_steps_per_second": 0.01
+    "train_loss": -0.01780639439821243,
+    "train_runtime": 1859.9628,
+    "train_samples": 41,
+    "train_samples_per_second": 0.022,
+    "train_steps_per_second": 0.005
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50ff9b6b649894c959c15a8db61291438695eb00e2bce04995391923a10370f5
+oid sha256:286fdb3e49aa32388e88e4345400cdc49ec2dfb94e3383b066d15d273eb6e95d
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0,
-    "train_runtime": 1040.7425,
-    "train_samples": 2,
-    "train_samples_per_second": 0.038,
-    "train_steps_per_second": 0.01
+    "train_loss": -0.01780639439821243,
+    "train_runtime": 1859.9628,
+    "train_samples": 41,
+    "train_samples_per_second": 0.022,
+    "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 10.0,
+  "epoch": 0.4878048780487805,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,37 +16,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 14.0,
-      "completions/max_terminated_length": 14.0,
-      "completions/mean_length": 14.0,
-      "completions/mean_terminated_length": 14.0,
-      "completions/min_length": 14.0,
-      "completions/min_terminated_length": 14.0,
-      "epoch": 2.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
+      "completions/max_length": 127.0,
+      "completions/max_terminated_length": 127.0,
+      "completions/mean_length": 59.25,
+      "completions/mean_terminated_length": 59.25,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
+      "epoch": 0.0975609756097561,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 25.766353607177734,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "num_tokens": 1136.0,
-      "reward": 1.2776422500610352,
-      "reward_std": 0.0,
+      "loss": 0.0186,
+      "num_tokens": 1498.0,
+      "reward": 0.049393012188374996,
+      "reward_std": 0.048807840794324875,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.027642276138067245,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.02114301174879074,
+      "rewards/question_recreation_reward_func/std": 0.0122159318998456,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.02824999950826168,
+      "rewards/xmlcount_reward_func/std": 0.05649999901652336,
       "step": 2
     },
     {
@@ -56,37 +56,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 14.0,
-      "completions/max_terminated_length": 14.0,
-      "completions/mean_length": 14.0,
-      "completions/mean_terminated_length": 14.0,
-      "completions/min_length": 14.0,
-      "completions/min_terminated_length": 14.0,
-      "epoch": 4.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 132.0,
+      "completions/max_terminated_length": 132.0,
+      "completions/mean_length": 53.75,
+      "completions/mean_terminated_length": 53.75,
+      "completions/min_length": 24.5,
+      "completions/min_terminated_length": 24.5,
+      "epoch": 0.1951219512195122,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 19.915145874023438,
+      "kl": 0.0023184213787317276,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.0,
-      "num_tokens": 2272.0,
-      "reward": 1.2890243530273438,
-      "reward_std": 0.0,
+      "loss": 0.0505,
+      "num_tokens": 2952.0,
+      "reward": 0.04935022257268429,
+      "reward_std": 0.05023687332868576,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.039024390280246735,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.018100222572684288,
+      "rewards/question_recreation_reward_func/std": 0.012556762900203466,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.03125,
+      "rewards/xmlcount_reward_func/std": 0.0625,
       "step": 4
     },
     {
@@ -96,37 +96,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 14.0,
-      "completions/max_terminated_length": 14.0,
-      "completions/mean_length": 14.0,
-      "completions/mean_terminated_length": 14.0,
-      "completions/min_length": 14.0,
-      "completions/min_terminated_length": 14.0,
-      "epoch": 6.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 67.5,
+      "completions/max_terminated_length": 67.5,
+      "completions/mean_length": 28.25,
+      "completions/mean_terminated_length": 28.25,
+      "completions/min_length": 5.5,
+      "completions/min_terminated_length": 5.5,
+      "epoch": 0.2926829268292683,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 97.34249877929688,
+      "kl": 0.01801438198890537,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0,
-      "num_tokens": 3408.0,
-      "reward": 1.2776422500610352,
-      "reward_std": 0.0,
+      "loss": -0.16,
+      "num_tokens": 4202.0,
+      "reward": 0.114079300314188,
+      "reward_std": 0.08150303550064564,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.027642276138067245,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.08282929984852672,
+      "rewards/question_recreation_reward_func/std": 0.03306010598316789,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.03125,
+      "rewards/xmlcount_reward_func/std": 0.0625,
       "step": 6
     },
     {
@@ -136,37 +136,37 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 14.0,
-      "completions/max_terminated_length": 14.0,
-      "completions/mean_length": 14.0,
-      "completions/mean_terminated_length": 14.0,
-      "completions/min_length": 14.0,
-      "completions/min_terminated_length": 14.0,
-      "epoch": 8.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 75.5,
+      "completions/max_terminated_length": 75.5,
+      "completions/mean_length": 36.875,
+      "completions/mean_terminated_length": 36.875,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
+      "epoch": 0.3902439024390244,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 50.915489196777344,
+      "kl": 0.019150954321958125,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0,
-      "num_tokens": 4544.0,
-      "reward": 1.2890243530273438,
-      "reward_std": 0.0,
+      "loss": -0.0363,
+      "num_tokens": 5521.0,
+      "reward": 0.13781297951936722,
+      "reward_std": 0.03136043483391404,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.039024390280246735,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.028437979985028505,
+      "rewards/question_recreation_reward_func/std": 0.010751228081062436,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.109375,
+      "rewards/xmlcount_reward_func/std": 0.13200797885656357,
       "step": 8
     },
     {
@@ -176,53 +176,53 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 14.0,
-      "completions/max_terminated_length": 14.0,
-      "completions/mean_length": 14.0,
-      "completions/mean_terminated_length": 14.0,
-      "completions/min_length": 14.0,
-      "completions/min_terminated_length": 14.0,
-      "epoch": 10.0,
-      "frac_reward_zero_std": 1.0,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 120.5,
+      "completions/max_terminated_length": 120.5,
+      "completions/mean_length": 44.25,
+      "completions/mean_terminated_length": 44.25,
+      "completions/min_length": 12.5,
+      "completions/min_terminated_length": 12.5,
+      "epoch": 0.4878048780487805,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 31.54133415222168,
+      "kl": 0.02116560866124928,
       "learning_rate": 1.507684480352292e-08,
-      "loss": 0.0,
-      "num_tokens": 5680.0,
-      "reward": 1.2776422500610352,
-      "reward_std": 0.0,
+      "loss": 0.0382,
+      "num_tokens": 6899.0,
+      "reward": 0.08553153276443481,
+      "reward_std": 0.019916290184482932,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 1.0,
-      "rewards/final_correctness_reward_func/std": 1.154700517654419,
-      "rewards/question_recreation_reward_func/mean": 0.027642276138067245,
-      "rewards/question_recreation_reward_func/std": 0.0,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.013406533282250166,
+      "rewards/question_recreation_reward_func/std": 0.013068773550912738,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.07212499901652336,
+      "rewards/xmlcount_reward_func/std": 0.09649057686328888,
       "step": 10
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.4878048780487805,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.0,
-      "train_runtime": 1040.7425,
-      "train_samples_per_second": 0.038,
-      "train_steps_per_second": 0.01
+      "train_loss": -0.01780639439821243,
+      "train_runtime": 1859.9628,
+      "train_samples_per_second": 0.022,
+      "train_steps_per_second": 0.005
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 5680,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 6899,
+  "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {
     "TrainerControl": {