End of training

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,72 @@
+---
+base_model: Gensyn/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Instruct-Gensyn-Swarm-invisible_domestic_impala
+tags:
+- generated_from_trainer
+- rl-swarm
+- grpo
+- gensyn
+- I am invisible domestic impala
+- unsloth
+- trl
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Instruct-Gensyn-Swarm-invisible_domestic_impala
+
+This model is a fine-tuned version of [Gensyn/Qwen2.5-1.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="itufilum/Qwen2.5-1.5B-Instruct-Gensyn-Swarm-invisible_domestic_impala", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.2
+- Transformers: 4.51.3
+- Pytorch: 2.5.1
+- Datasets: 3.6.0
+- Tokenizers: 0.21.1
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Gensyn/Qwen2.5-1.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj",
+    "gate_proj",
+    "k_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f5da1c07c97beeb9b16cf1ca9d1e7213b896f2c4eaf53f529ace2d1e4e58fc0
+size 73911112

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 1.1221799382710174e-06,
+    "train_runtime": 1585.3495,
+    "train_samples": 29,
+    "train_samples_per_second": 1.009,
+    "train_steps_per_second": 0.063
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 1.1221799382710174e-06,
+    "train_runtime": 1585.3495,
+    "train_samples": 29,
+    "train_samples_per_second": 1.009,
+    "train_steps_per_second": 0.063
+}

trainer_state.json

@@ -0,0 +1,993 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 12.551724137931034,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 261.9375,
+      "epoch": 0.27586206896551724,
+      "grad_norm": 0.5876455307006836,
+      "kl": 0.0,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0,
+      "reward": 1.4498194679617882,
+      "reward_std": 1.1243495009839535,
+      "rewards/concensus_correctness_reward_func": 0.11999999731779099,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.37772573810070753,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.592718742787838,
+      "step": 2
+    },
+    {
+      "completion_length": 279.15625,
+      "epoch": 0.5517241379310345,
+      "grad_norm": 0.7141730785369873,
+      "kl": 0.0020650627120630816,
+      "learning_rate": 5e-06,
+      "loss": 0.0,
+      "reward": 1.2884040176868439,
+      "reward_std": 1.1638742461800575,
+      "rewards/concensus_correctness_reward_func": 0.12012499943375587,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5039352774620056,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3830937538295984,
+      "step": 4
+    },
+    {
+      "completion_length": 325.8125,
+      "epoch": 0.8275862068965517,
+      "grad_norm": 0.5803059935569763,
+      "kl": 0.000953761518758256,
+      "learning_rate": 4.99475706559428e-06,
+      "loss": 0.0,
+      "reward": 1.294271882623434,
+      "reward_std": 0.8707563504576683,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.5154593642801046,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.4194375015795231,
+      "step": 6
+    },
+    {
+      "completion_length": 361.75,
+      "epoch": 1.0,
+      "grad_norm": 0.20475846529006958,
+      "kl": 0.0010331896017305553,
+      "learning_rate": 4.979050253066064e-06,
+      "loss": 0.0,
+      "reward": 1.6736069202423096,
+      "reward_std": 1.0094650149345399,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.552256902679801,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.05,
+      "rewards/xmlcount_reward_func": 0.5713500022888184,
+      "step": 8
+    },
+    {
+      "completion_length": 384.46875,
+      "epoch": 1.2758620689655173,
+      "grad_norm": 0.43686386942863464,
+      "kl": 0.0012712998868664727,
+      "learning_rate": 4.952945442245598e-06,
+      "loss": 0.0,
+      "reward": 1.1086738537997007,
+      "reward_std": 0.8807330783456564,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4103300729766488,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4483437407761812,
+      "step": 10
+    },
+    {
+      "completion_length": 347.28125,
+      "epoch": 1.5517241379310345,
+      "grad_norm": 0.6117482781410217,
+      "kl": 0.0009661500589572825,
+      "learning_rate": 4.916552125781529e-06,
+      "loss": 0.0,
+      "reward": 1.2644198425114155,
+      "reward_std": 0.891217265278101,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5375448148697615,
+      "rewards/soft_format_reward_func": 0.03125,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.3987499997019768,
+      "step": 12
+    },
+    {
+      "completion_length": 249.9375,
+      "epoch": 1.8275862068965516,
+      "grad_norm": 0.6757048964500427,
+      "kl": 0.0011229460214963183,
+      "learning_rate": 4.870022949890676e-06,
+      "loss": 0.0,
+      "reward": 1.8286586850881577,
+      "reward_std": 1.1331563144922256,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5068149194121361,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.5562187545001507,
+      "step": 14
+    },
+    {
+      "completion_length": 275.35,
+      "epoch": 2.0,
+      "grad_norm": 0.31559568643569946,
+      "kl": 0.00098627534462139,
+      "learning_rate": 4.813553074106761e-06,
+      "loss": 0.0,
+      "reward": 2.0888511657714846,
+      "reward_std": 1.4333394169807434,
+      "rewards/concensus_correctness_reward_func": 0.13020000457763672,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.6,
+      "rewards/question_recreation_reward_func": 0.7344011664390564,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.05,
+      "rewards/xmlcount_reward_func": 0.5742499828338623,
+      "step": 16
+    },
+    {
+      "completion_length": 302.5,
+      "epoch": 2.2758620689655173,
+      "grad_norm": 0.6794589161872864,
+      "kl": 0.0009313603804912418,
+      "learning_rate": 4.747379352713489e-06,
+      "loss": 0.0,
+      "reward": 2.012173980474472,
+      "reward_std": 1.3338787257671356,
+      "rewards/concensus_correctness_reward_func": 0.10331249982118607,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.5153615120798349,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4559999890625477,
+      "step": 18
+    },
+    {
+      "completion_length": 259.78125,
+      "epoch": 2.5517241379310347,
+      "grad_norm": 0.7855917811393738,
+      "kl": 0.0010634321151883341,
+      "learning_rate": 4.671779341295378e-06,
+      "loss": 0.0,
+      "reward": 1.7313391268253326,
+      "reward_std": 1.3928996622562408,
+      "rewards/concensus_correctness_reward_func": 0.26087500154972076,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5740891546010971,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.45887498930096626,
+      "step": 20
+    },
+    {
+      "completion_length": 266.65625,
+      "epoch": 2.8275862068965516,
+      "grad_norm": 0.746308445930481,
+      "kl": 0.0011307917884550989,
+      "learning_rate": 4.587070132573178e-06,
+      "loss": 0.0,
+      "reward": 1.402635745704174,
+      "reward_std": 0.9156965278089046,
+      "rewards/concensus_correctness_reward_func": 0.015625,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4964794850675389,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5467812465503812,
+      "step": 22
+    },
+    {
+      "completion_length": 327.05,
+      "epoch": 3.0,
+      "grad_norm": 0.23480382561683655,
+      "kl": 0.00111085653770715,
+      "learning_rate": 4.493607026406802e-06,
+      "loss": 0.0,
+      "reward": 1.1529444456100464,
+      "reward_std": 0.523932871222496,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5830944389104843,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.34484999179840087,
+      "step": 24
+    },
+    {
+      "completion_length": 336.8125,
+      "epoch": 3.2758620689655173,
+      "grad_norm": 0.8294694423675537,
+      "kl": 0.0010076407925225794,
+      "learning_rate": 4.391782039544239e-06,
+      "loss": 0.0,
+      "reward": 1.3900708109140396,
+      "reward_std": 1.3332865573465824,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.1875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.46316459868103266,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4761562645435333,
+      "step": 26
+    },
+    {
+      "completion_length": 353.28125,
+      "epoch": 3.5517241379310347,
+      "grad_norm": 0.6492119431495667,
+      "kl": 0.0011136386019643396,
+      "learning_rate": 4.282022261367074e-06,
+      "loss": 0.0,
+      "reward": 1.7649768702685833,
+      "reward_std": 1.058425359427929,
+      "rewards/concensus_correctness_reward_func": 0.015625,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6061330996453762,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5025937429745682,
+      "step": 28
+    },
+    {
+      "completion_length": 281.6875,
+      "epoch": 3.8275862068965516,
+      "grad_norm": 0.8953530192375183,
+      "kl": 0.001063827752659563,
+      "learning_rate": 4.164788062529203e-06,
+      "loss": 0.0,
+      "reward": 2.2271154075860977,
+      "reward_std": 1.9515271224081516,
+      "rewards/concensus_correctness_reward_func": 0.640625,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.57742790132761,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6184374950826168,
+      "step": 30
+    },
+    {
+      "completion_length": 268.9,
+      "epoch": 4.0,
+      "grad_norm": 0.5744255781173706,
+      "kl": 0.001324763905722648,
+      "learning_rate": 4.040571164002319e-06,
+      "loss": 0.0,
+      "reward": 1.4024577617645264,
+      "reward_std": 1.0153881907463074,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3,
+      "rewards/question_recreation_reward_func": 0.6460577547550201,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.05,
+      "rewards/xmlcount_reward_func": 0.4063999891281128,
+      "step": 32
+    },
+    {
+      "completion_length": 315.125,
+      "epoch": 4.275862068965517,
+      "grad_norm": 1.165102243423462,
+      "kl": 0.000977253686869517,
+      "learning_rate": 3.909892574627267e-06,
+      "loss": 0.0,
+      "reward": 1.1481780782341957,
+      "reward_std": 1.0175574347376823,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.3283343203365803,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4448437672108412,
+      "step": 34
+    },
+    {
+      "completion_length": 285.3125,
+      "epoch": 4.551724137931035,
+      "grad_norm": 0.6390895843505859,
+      "kl": 0.0010886134841712192,
+      "learning_rate": 3.773300405821908e-06,
+      "loss": 0.0,
+      "reward": 1.2504443675279617,
+      "reward_std": 1.2421578019857407,
+      "rewards/concensus_correctness_reward_func": 0.07575000077486038,
+      "rewards/consensus_reward_func": 0.1875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4656319017522037,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3340625036507845,
+      "step": 36
+    },
+    {
+      "completion_length": 261.6875,
+      "epoch": 4.827586206896552,
+      "grad_norm": 0.6622844934463501,
+      "kl": 0.0011051035835407674,
+      "learning_rate": 3.631367572611348e-06,
+      "loss": 0.0,
+      "reward": 1.0794794410467148,
+      "reward_std": 0.8599886745214462,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5531356520950794,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.24509375356137753,
+      "step": 38
+    },
+    {
+      "completion_length": 317.2,
+      "epoch": 5.0,
+      "grad_norm": 0.6508445143699646,
+      "kl": 0.0012014877400361,
+      "learning_rate": 3.484689390623218e-06,
+      "loss": 0.0,
+      "reward": 1.2601001858711243,
+      "reward_std": 0.9332681000232697,
+      "rewards/concensus_correctness_reward_func": 0.09599999785423279,
+      "rewards/consensus_reward_func": 0.1,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5755501747131347,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.46354998350143434,
+      "step": 40
+    },
+    {
+      "completion_length": 348.0625,
+      "epoch": 5.275862068965517,
+      "grad_norm": 0.4629547595977783,
+      "kl": 0.0010141393868252635,
+      "learning_rate": 3.333881079127052e-06,
+      "loss": 0.0,
+      "reward": 1.3980103582143784,
+      "reward_std": 1.1040666736662388,
+      "rewards/concensus_correctness_reward_func": 0.13574999943375587,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.6644478440284729,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3790625077672303,
+      "step": 42
+    },
+    {
+      "completion_length": 273.65625,
+      "epoch": 5.551724137931035,
+      "grad_norm": 0.7342892289161682,
+      "kl": 0.0011725125659722835,
+      "learning_rate": 3.1795751805908578e-06,
+      "loss": 0.0,
+      "reward": 2.422099143266678,
+      "reward_std": 2.033245772123337,
+      "rewards/concensus_correctness_reward_func": 0.7265625,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5944116413593292,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3980000060983002,
+      "step": 44
+    },
+    {
+      "completion_length": 379.8125,
+      "epoch": 5.827586206896552,
+      "grad_norm": 0.652703046798706,
+      "kl": 0.0010911910067079589,
+      "learning_rate": 3.0224189075781886e-06,
+      "loss": 0.0,
+      "reward": 1.2743461802601814,
+      "reward_std": 1.3854865245521069,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4171899161301553,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.42203125543892384,
+      "step": 46
+    },
+    {
+      "completion_length": 318.95,
+      "epoch": 6.0,
+      "grad_norm": 0.22338756918907166,
+      "kl": 0.001001153700053692,
+      "learning_rate": 2.8630714281137263e-06,
+      "loss": 0.0,
+      "reward": 1.5304819822311402,
+      "reward_std": 1.059793508052826,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.38448197543621065,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.5209999918937683,
+      "step": 48
+    },
+    {
+      "completion_length": 370.46875,
+      "epoch": 6.275862068965517,
+      "grad_norm": 0.6062008738517761,
+      "kl": 0.0010415500364615582,
+      "learning_rate": 2.702201100903511e-06,
+      "loss": 0.0,
+      "reward": 1.4714525565505028,
+      "reward_std": 1.0109544545412064,
+      "rewards/concensus_correctness_reward_func": 0.0755000002682209,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4757650615647435,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4826874975115061,
+      "step": 50
+    },
+    {
+      "completion_length": 339.28125,
+      "epoch": 6.551724137931035,
+      "grad_norm": 0.6081772446632385,
+      "kl": 0.000928263645619154,
+      "learning_rate": 2.5404826720062544e-06,
+      "loss": 0.0,
+      "reward": 1.6288059502840042,
+      "reward_std": 1.1489972844719887,
+      "rewards/concensus_correctness_reward_func": 0.03125,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5305872187018394,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3794687408953905,
+      "step": 52
+    },
+    {
+      "completion_length": 294.8125,
+      "epoch": 6.827586206896552,
+      "grad_norm": 0.7885801196098328,
+      "kl": 0.0011745219817385077,
+      "learning_rate": 2.3785944447138804e-06,
+      "loss": 0.0,
+      "reward": 1.3874307684600353,
+      "reward_std": 1.1232963781803846,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5388057269155979,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.42912499886006117,
+      "step": 54
+    },
+    {
+      "completion_length": 335.2,
+      "epoch": 7.0,
+      "grad_norm": 0.35713574290275574,
+      "kl": 0.0010486021172255277,
+      "learning_rate": 2.2172154345117896e-06,
+      "loss": 0.0,
+      "reward": 0.9908310234546661,
+      "reward_std": 0.8158440917730332,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.2,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.2,
+      "rewards/question_recreation_reward_func": 0.24703103601932525,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.31880000084638593,
+      "step": 56
+    },
+    {
+      "completion_length": 269.875,
+      "epoch": 7.275862068965517,
+      "grad_norm": 0.7617030143737793,
+      "kl": 0.0012455169853637926,
+      "learning_rate": 2.0570225210519433e-06,
+      "loss": 0.0,
+      "reward": 1.2766155302524567,
+      "reward_std": 0.9013489130884409,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.498896773904562,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.046875,
+      "rewards/xmlcount_reward_func": 0.5433437526226044,
+      "step": 58
+    },
+    {
+      "completion_length": 359.75,
+      "epoch": 7.551724137931035,
+      "grad_norm": 0.4766957759857178,
+      "kl": 0.0010567399804131128,
+      "learning_rate": 1.8986876090843668e-06,
+      "loss": 0.0,
+      "reward": 1.335220292210579,
+      "reward_std": 1.233029380440712,
+      "rewards/concensus_correctness_reward_func": 0.03125,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4641265389509499,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4023437686264515,
+      "step": 60
+    },
+    {
+      "completion_length": 317.21875,
+      "epoch": 7.827586206896552,
+      "grad_norm": 0.8781487941741943,
+      "kl": 0.0012130630420870148,
+      "learning_rate": 1.7428748102551237e-06,
+      "loss": 0.0,
+      "reward": 1.2447976544499397,
+      "reward_std": 0.9147735517472029,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5317039042711258,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4318437557667494,
+      "step": 62
+    },
+    {
+      "completion_length": 291.95,
+      "epoch": 8.0,
+      "grad_norm": 0.44187262654304504,
+      "kl": 0.0014347493182867766,
+      "learning_rate": 1.5902376575912815e-06,
+      "loss": 0.0,
+      "reward": 3.6168224096298216,
+      "reward_std": 2.845296561717987,
+      "rewards/concensus_correctness_reward_func": 2.0,
+      "rewards/consensus_reward_func": 0.3,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3,
+      "rewards/question_recreation_reward_func": 0.3940222471952438,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.05,
+      "rewards/xmlcount_reward_func": 0.5728000104427338,
+      "step": 64
+    },
+    {
+      "completion_length": 306.9375,
+      "epoch": 8.275862068965518,
+      "grad_norm": 1.304425835609436,
+      "kl": 0.0012883242161478847,
+      "learning_rate": 1.4414163643562755e-06,
+      "loss": 0.0,
+      "reward": 1.414997085928917,
+      "reward_std": 0.9384825639426708,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.1875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5457158237695694,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4630312602967024,
+      "step": 66
+    },
+    {
+      "completion_length": 275.09375,
+      "epoch": 8.551724137931034,
+      "grad_norm": 0.7772114872932434,
+      "kl": 0.0014374244710779749,
+      "learning_rate": 1.2970351387729875e-06,
+      "loss": 0.0,
+      "reward": 1.7658802941441536,
+      "reward_std": 1.1716703101992607,
+      "rewards/concensus_correctness_reward_func": 0.11599999666213989,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5250677652657032,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.49981250055134296,
+      "step": 68
+    },
+    {
+      "completion_length": 330.53125,
+      "epoch": 8.827586206896552,
+      "grad_norm": 0.8543239235877991,
+      "kl": 0.0013485400995705277,
+      "learning_rate": 1.1576995658775405e-06,
+      "loss": 0.0,
+      "reward": 1.2814906537532806,
+      "reward_std": 1.0425735749304295,
+      "rewards/concensus_correctness_reward_func": 0.09137500077486038,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.4742719018831849,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.30959376133978367,
+      "step": 70
+    },
+    {
+      "completion_length": 403.5,
+      "epoch": 9.0,
+      "grad_norm": 0.21790927648544312,
+      "kl": 0.001029879879206419,
+      "learning_rate": 1.0239940674851943e-06,
+      "loss": 0.0,
+      "reward": 1.454481029510498,
+      "reward_std": 0.7960291028022766,
+      "rewards/concensus_correctness_reward_func": 0.05,
+      "rewards/consensus_reward_func": 0.1,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.2,
+      "rewards/question_recreation_reward_func": 0.46478102207183836,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.6146999955177307,
+      "step": 72
+    },
+    {
+      "completion_length": 258.78125,
+      "epoch": 9.275862068965518,
+      "grad_norm": 0.9624819755554199,
+      "kl": 0.0016905390948522836,
+      "learning_rate": 8.964794509221508e-07,
+      "loss": 0.0,
+      "reward": 1.8803276717662811,
+      "reward_std": 1.6410220339894295,
+      "rewards/concensus_correctness_reward_func": 0.18275000154972076,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5239214114844799,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.720531240105629,
+      "step": 74
+    },
+    {
+      "completion_length": 320.03125,
+      "epoch": 9.551724137931034,
+      "grad_norm": 0.7233691811561584,
+      "kl": 0.0010870415571844205,
+      "learning_rate": 7.756905568047393e-07,
+      "loss": 0.0,
+      "reward": 1.6069397777318954,
+      "reward_std": 1.0762585289776325,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5746584795415401,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.641656244173646,
+      "step": 76
+    },
+    {
+      "completion_length": 288.40625,
+      "epoch": 9.827586206896552,
+      "grad_norm": 0.6633221507072449,
+      "kl": 0.0014429407165152952,
+      "learning_rate": 6.621340157319998e-07,
+      "loss": 0.0,
+      "reward": 1.9194584637880325,
+      "reward_std": 1.9881719145923853,
+      "rewards/concensus_correctness_reward_func": 0.640625,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.46661476604640484,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5465937443077564,
+      "step": 78
+    },
+    {
+      "completion_length": 348.0,
+      "epoch": 10.0,
+      "grad_norm": 0.4076433479785919,
+      "kl": 0.001379342400468886,
+      "learning_rate": 5.562861233008774e-07,
+      "loss": 0.0,
+      "reward": 0.835002475976944,
+      "reward_std": 0.8312704622745514,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3,
+      "rewards/question_recreation_reward_func": 0.41810246780514715,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11689999997615814,
+      "step": 80
+    },
+    {
+      "completion_length": 284.5625,
+      "epoch": 10.275862068965518,
+      "grad_norm": 13.410774230957031,
+      "kl": 0.002484311757143587,
+      "learning_rate": 4.5859084235697236e-07,
+      "loss": 0.0,
+      "reward": 1.334033541381359,
+      "reward_std": 1.2119258306920528,
+      "rewards/concensus_correctness_reward_func": 0.12262500077486038,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.591752303764224,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4009062615223229,
+      "step": 82
+    },
+    {
+      "completion_length": 283.3125,
+      "epoch": 10.551724137931034,
+      "grad_norm": 0.5552597641944885,
+      "kl": 0.0015608746980433352,
+      "learning_rate": 3.6945794086007706e-07,
+      "loss": 0.0,
+      "reward": 1.3054824657738209,
+      "reward_std": 0.9629417397081852,
+      "rewards/concensus_correctness_reward_func": 0.08137500286102295,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.5141699481755495,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4443124867975712,
+      "step": 84
+    },
+    {
+      "completion_length": 349.125,
+      "epoch": 10.827586206896552,
+      "grad_norm": 0.5144564509391785,
+      "kl": 0.0014357663603732362,
+      "learning_rate": 2.892612731749414e-07,
+      "loss": 0.0,
+      "reward": 2.4467398822307587,
+      "reward_std": 2.3171303123235703,
+      "rewards/concensus_correctness_reward_func": 0.7409999966621399,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6876149065792561,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.43999999947845936,
+      "step": 86
+    },
+    {
+      "completion_length": 266.25,
+      "epoch": 11.0,
+      "grad_norm": 0.537667453289032,
+      "kl": 0.0013617533957585692,
+      "learning_rate": 2.1833721199614992e-07,
+      "loss": 0.0,
+      "reward": 1.3557676315307616,
+      "reward_std": 1.1246003568172456,
+      "rewards/concensus_correctness_reward_func": 0.09620000123977661,
+      "rewards/consensus_reward_func": 0.1,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1,
+      "rewards/question_recreation_reward_func": 0.37711760550737383,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.6574500024318695,
+      "step": 88
+    },
+    {
+      "completion_length": 298.59375,
+      "epoch": 11.275862068965518,
+      "grad_norm": 0.7764319181442261,
+      "kl": 0.0013516054386855103,
+      "learning_rate": 1.5698323748414123e-07,
+      "loss": 0.0,
+      "reward": 1.9731452241539955,
+      "reward_std": 2.4217655174434185,
+      "rewards/concensus_correctness_reward_func": 0.8366875015199184,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5108639299869537,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.29746875166893005,
+      "step": 90
+    },
+    {
+      "completion_length": 265.59375,
+      "epoch": 11.551724137931034,
+      "grad_norm": 0.7998268008232117,
+      "kl": 0.0019906353700207546,
+      "learning_rate": 1.054566895300324e-07,
+      "loss": 0.0,
+      "reward": 1.2235557958483696,
+      "reward_std": 0.7784941829741001,
+      "rewards/concensus_correctness_reward_func": 0.04337500035762787,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5579620627686381,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4659687466919422,
+      "step": 92
+    },
+    {
+      "completion_length": 302.46875,
+      "epoch": 11.827586206896552,
+      "grad_norm": 0.5803750157356262,
+      "kl": 0.0012688033384620212,
+      "learning_rate": 6.397368838268497e-08,
+      "loss": 0.0,
+      "reward": 1.6255936734378338,
+      "reward_std": 1.3891006745398045,
+      "rewards/concensus_correctness_reward_func": 0.1693749986588955,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5416874596849084,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3207812551409006,
+      "step": 94
+    },
+    {
+      "completion_length": 357.85,
+      "epoch": 12.0,
+      "grad_norm": 0.24747301638126373,
+      "kl": 0.0011559076607227325,
+      "learning_rate": 3.270822816527325e-08,
+      "loss": 0.0,
+      "reward": 1.146351419389248,
+      "reward_std": 1.193832767009735,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4,
+      "rewards/question_recreation_reward_func": 0.5318014442920684,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.025,
+      "rewards/xmlcount_reward_func": 0.18954999446868898,
+      "step": 96
+    },
+    {
+      "completion_length": 286.65625,
+      "epoch": 12.275862068965518,
+      "grad_norm": 0.8936555981636047,
+      "kl": 0.001531564790639095,
+      "learning_rate": 1.1791447083465136e-08,
+      "loss": 0.0,
+      "reward": 2.939312696456909,
+      "reward_std": 2.551104363054037,
+      "rewards/concensus_correctness_reward_func": 1.25,
+      "rewards/consensus_reward_func": 0.25,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.48412513616494834,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4551875004544854,
+      "step": 98
+    },
+    {
+      "completion_length": 366.78125,
+      "epoch": 12.551724137931034,
+      "grad_norm": 0.5418797135353088,
+      "kl": 0.001285279548028484,
+      "learning_rate": 1.3110773862126669e-09,
+      "loss": 0.0,
+      "reward": 1.3908307328820229,
+      "reward_std": 1.0579438470304012,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6382682472467422,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4244375005364418,
+      "step": 100
+    },
+    {
+      "epoch": 12.551724137931034,
+      "step": 100,
+      "total_flos": 0.0,
+      "train_loss": 1.1221799382710174e-06,
+      "train_runtime": 1585.3495,
+      "train_samples_per_second": 1.009,
+      "train_steps_per_second": 0.063
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 15,
+  "save_steps": 25,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f307b3433a765f2d18c7e7a0be84b4fc6c9d21a9f869b9f12d65d0f88b52dfe
+size 6008