End of training

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,71 @@
+---
+base_model: Gensyn/Qwen2.5-1.5B-Instruct
+library_name: transformers
+model_name: Qwen2.5-1.5B-Instruct-Gensyn-Swarm-slithering_sneaky_chinchilla
+tags:
+- generated_from_trainer
+- rl-swarm
+- grpo
+- gensyn
+- I am slithering sneaky chinchilla
+- trl
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Instruct-Gensyn-Swarm-slithering_sneaky_chinchilla
+
+This model is a fine-tuned version of [Gensyn/Qwen2.5-1.5B-Instruct](https://huggingface.co/Gensyn/Qwen2.5-1.5B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="wyceee/Qwen2.5-1.5B-Instruct-Gensyn-Swarm-slithering_sneaky_chinchilla", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.2
+- Transformers: 4.51.3
+- Pytorch: 2.5.1
+- Datasets: 3.5.0
+- Tokenizers: 0.21.1
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 5.14280412971857e-06,
+    "train_runtime": 2155.4049,
+    "train_samples": 140,
+    "train_samples_per_second": 0.742,
+    "train_steps_per_second": 0.046
+}

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.51.3"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaf4fcdefb7f362fdb7f02a78b98eb604363f6c7c50d3aeb3a683bdc0f851555
+size 4996670464

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7d10e11ce64502f5270013311d5eb4c6a7e0de8af92fc6e0961226fae7a98a4
+size 1178224960

model.safetensors.index.json

@@ -0,0 +1,345 @@
+{
+  "metadata": {
+    "total_size": 6174857216
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 5.14280412971857e-06,
+    "train_runtime": 2155.4049,
+    "train_samples": 140,
+    "train_samples_per_second": 0.742,
+    "train_steps_per_second": 0.046
+}

trainer_state.json

@@ -0,0 +1,993 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.571428571428571,
+  "eval_steps": 500,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 313.65625,
+      "epoch": 0.11428571428571428,
+      "grad_norm": 1.9183201789855957,
+      "kl": 0.0,
+      "learning_rate": 1.6666666666666665e-07,
+      "loss": 0.0,
+      "reward": 5.733217548578978,
+      "reward_std": 3.8726130831055343,
+      "rewards/concensus_correctness_reward_func": 3.3072499986737967,
+      "rewards/consensus_reward_func": 0.6875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.5423112083226442,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3211562526412308,
+      "step": 2
+    },
+    {
+      "completion_length": 302.6875,
+      "epoch": 0.22857142857142856,
+      "grad_norm": 2.035726308822632,
+      "kl": 0.0008348686824319884,
+      "learning_rate": 5e-07,
+      "loss": 0.0,
+      "reward": 3.6854598224163055,
+      "reward_std": 1.4634849466383457,
+      "rewards/concensus_correctness_reward_func": 1.4465625032316893,
+      "rewards/consensus_reward_func": 0.625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6805536039173603,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4802187574096024,
+      "step": 4
+    },
+    {
+      "completion_length": 278.5,
+      "epoch": 0.34285714285714286,
+      "grad_norm": 2.9881222248077393,
+      "kl": 0.0006843123064754764,
+      "learning_rate": 4.994757065594279e-07,
+      "loss": 0.0,
+      "reward": 1.408921517431736,
+      "reward_std": 0.8394484423333779,
+      "rewards/concensus_correctness_reward_func": 0.07199999992735684,
+      "rewards/consensus_reward_func": 0.25,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5614215070381761,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3848750018514693,
+      "step": 6
+    },
+    {
+      "completion_length": 308.09375,
+      "epoch": 0.45714285714285713,
+      "grad_norm": 2.3815231323242188,
+      "kl": 0.0008725040206627455,
+      "learning_rate": 4.979050253066063e-07,
+      "loss": 0.0,
+      "reward": 5.5596274845302105,
+      "reward_std": 2.3076624351087958,
+      "rewards/concensus_correctness_reward_func": 3.843499973183498,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5385024221614003,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3026250071125105,
+      "step": 8
+    },
+    {
+      "completion_length": 287.6875,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 2.3809289932250977,
+      "kl": 0.0008873967562976759,
+      "learning_rate": 4.952945442245597e-07,
+      "loss": 0.0,
+      "reward": 4.663692280650139,
+      "reward_std": 3.257206997834146,
+      "rewards/concensus_correctness_reward_func": 2.7571874954737723,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5873797507956624,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3816250069066882,
+      "step": 10
+    },
+    {
+      "completion_length": 267.8125,
+      "epoch": 0.6857142857142857,
+      "grad_norm": 2.1084818840026855,
+      "kl": 0.001183122909424128,
+      "learning_rate": 4.916552125781528e-07,
+      "loss": 0.0,
+      "reward": 5.68206575140357,
+      "reward_std": 5.3108197445981205,
+      "rewards/concensus_correctness_reward_func": 3.338500021956861,
+      "rewards/consensus_reward_func": 0.5625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5143157998099923,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.6105000090319663,
+      "step": 12
+    },
+    {
+      "completion_length": 306.0625,
+      "epoch": 0.8,
+      "grad_norm": 2.397752523422241,
+      "kl": 0.0014343660077429377,
+      "learning_rate": 4.870022949890676e-07,
+      "loss": 0.0,
+      "reward": 4.247058918699622,
+      "reward_std": 1.4322406734863762,
+      "rewards/concensus_correctness_reward_func": 2.0690625309944153,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.4255589717067778,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5649375049397349,
+      "step": 14
+    },
+    {
+      "completion_length": 331.75,
+      "epoch": 0.9142857142857143,
+      "grad_norm": 2.2839765548706055,
+      "kl": 0.001408537311363034,
+      "learning_rate": 4.81355307410676e-07,
+      "loss": 0.0,
+      "reward": 2.6294993720948696,
+      "reward_std": 2.1304319854825735,
+      "rewards/concensus_correctness_reward_func": 0.8625625013373792,
+      "rewards/consensus_reward_func": 0.5,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.652124403975904,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3023125068284571,
+      "step": 16
+    },
+    {
+      "completion_length": 254.75,
+      "epoch": 1.0,
+      "grad_norm": 1.293320894241333,
+      "kl": 0.001520198837776358,
+      "learning_rate": 4.747379352713488e-07,
+      "loss": 0.0,
+      "reward": 4.4054756959279375,
+      "reward_std": 0.8638230375945568,
+      "rewards/concensus_correctness_reward_func": 2.0449999949584403,
+      "rewards/consensus_reward_func": 0.75,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5937257781624794,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5167499954501787,
+      "step": 18
+    },
+    {
+      "completion_length": 325.9375,
+      "epoch": 1.1142857142857143,
+      "grad_norm": 1.9340304136276245,
+      "kl": 0.0019464795368548948,
+      "learning_rate": 4.6717793412953776e-07,
+      "loss": 0.0,
+      "reward": 5.210880044847727,
+      "reward_std": 1.6739974903757684,
+      "rewards/concensus_correctness_reward_func": 3.3118124761313084,
+      "rewards/consensus_reward_func": 0.5625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5269739665091038,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.29396875062957406,
+      "step": 20
+    },
+    {
+      "completion_length": 272.71875,
+      "epoch": 1.2285714285714286,
+      "grad_norm": 2.2866125106811523,
+      "kl": 0.002068110006803181,
+      "learning_rate": 4.5870701325731773e-07,
+      "loss": 0.0,
+      "reward": 3.6153114959597588,
+      "reward_std": 2.5713650833349675,
+      "rewards/concensus_correctness_reward_func": 1.5542500228621066,
+      "rewards/consensus_reward_func": 0.4375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5716865402646363,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.48937500920146704,
+      "step": 22
+    },
+    {
+      "completion_length": 302.375,
+      "epoch": 1.342857142857143,
+      "grad_norm": 6.6668572425842285,
+      "kl": 0.0021221214192337357,
+      "learning_rate": 4.4936070264068016e-07,
+      "loss": 0.0,
+      "reward": 1.9632033314555883,
+      "reward_std": 0.8504583928734064,
+      "rewards/concensus_correctness_reward_func": 0.20187499769963324,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.45967210712842643,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.4579062513075769,
+      "step": 24
+    },
+    {
+      "completion_length": 284.8125,
+      "epoch": 1.457142857142857,
+      "grad_norm": 2.070162534713745,
+      "kl": 0.0021677847471437417,
+      "learning_rate": 4.391782039544238e-07,
+      "loss": 0.0,
+      "reward": 1.9739556834101677,
+      "reward_std": 0.8432966666496213,
+      "rewards/concensus_correctness_reward_func": 0.17362499982118607,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5859243981540203,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5894062593579292,
+      "step": 26
+    },
+    {
+      "completion_length": 314.625,
+      "epoch": 1.5714285714285714,
+      "grad_norm": 2.243846893310547,
+      "kl": 0.002329640177777037,
+      "learning_rate": 4.282022261367073e-07,
+      "loss": 0.0,
+      "reward": 2.3326709028333426,
+      "reward_std": 1.4158438248559833,
+      "rewards/concensus_correctness_reward_func": 0.09337499784305692,
+      "rewards/consensus_reward_func": 0.6875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6531084361486137,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.44556250888854265,
+      "step": 28
+    },
+    {
+      "completion_length": 304.75,
+      "epoch": 1.6857142857142857,
+      "grad_norm": 2.563140869140625,
+      "kl": 0.002116368035785854,
+      "learning_rate": 4.1647880625292027e-07,
+      "loss": 0.0,
+      "reward": 3.5053839487954974,
+      "reward_std": 2.4146184872370213,
+      "rewards/concensus_correctness_reward_func": 1.5751250311732292,
+      "rewards/consensus_reward_func": 0.125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.4747902047820389,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.517968756146729,
+      "step": 30
+    },
+    {
+      "completion_length": 281.3125,
+      "epoch": 1.8,
+      "grad_norm": 2.0132243633270264,
+      "kl": 0.002754501736490056,
+      "learning_rate": 4.040571164002318e-07,
+      "loss": 0.0,
+      "reward": 4.006643671542406,
+      "reward_std": 1.8082885849289596,
+      "rewards/concensus_correctness_reward_func": 2.0200000014156103,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5669561615213752,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.46656251489184797,
+      "step": 32
+    },
+    {
+      "completion_length": 320.8125,
+      "epoch": 1.9142857142857141,
+      "grad_norm": 2.2226650714874268,
+      "kl": 0.0029091214746586047,
+      "learning_rate": 3.909892574627266e-07,
+      "loss": 0.0,
+      "reward": 4.280492004007101,
+      "reward_std": 2.970120156183839,
+      "rewards/concensus_correctness_reward_func": 1.7401250004768372,
+      "rewards/consensus_reward_func": 0.625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.601523166289553,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5013437522575259,
+      "step": 34
+    },
+    {
+      "completion_length": 316.5,
+      "epoch": 2.0,
+      "grad_norm": 1.4613192081451416,
+      "kl": 0.002774594468064606,
+      "learning_rate": 3.773300405821908e-07,
+      "loss": 0.0,
+      "reward": 3.184830774863561,
+      "reward_std": 2.355151594034396,
+      "rewards/concensus_correctness_reward_func": 1.0445833352083962,
+      "rewards/consensus_reward_func": 0.5833333333333334,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.6418723997970422,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.5608749911189079,
+      "step": 36
+    },
+    {
+      "completion_length": 316.625,
+      "epoch": 2.1142857142857143,
+      "grad_norm": 1.8819042444229126,
+      "kl": 0.002866531016479712,
+      "learning_rate": 3.6313675726113475e-07,
+      "loss": 0.0,
+      "reward": 3.52888186275959,
+      "reward_std": 0.9872541772201657,
+      "rewards/concensus_correctness_reward_func": 1.5981874950230122,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.6021005599759519,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2660937544424087,
+      "step": 38
+    },
+    {
+      "completion_length": 283.75,
+      "epoch": 2.2285714285714286,
+      "grad_norm": 2.4403116703033447,
+      "kl": 0.003229207592085004,
+      "learning_rate": 3.484689390623218e-07,
+      "loss": 0.0,
+      "reward": 4.119999956339598,
+      "reward_std": 1.6003942100796849,
+      "rewards/concensus_correctness_reward_func": 2.0809374977834523,
+      "rewards/consensus_reward_func": 0.5,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.556437520775944,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5763750001788139,
+      "step": 40
+    },
+    {
+      "completion_length": 283.5625,
+      "epoch": 2.342857142857143,
+      "grad_norm": 2.141371250152588,
+      "kl": 0.003269614593591541,
+      "learning_rate": 3.3338810791270517e-07,
+      "loss": 0.0,
+      "reward": 2.2994888741523027,
+      "reward_std": 1.130831709713675,
+      "rewards/concensus_correctness_reward_func": 0.3513124962337315,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.47858266485854983,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.46959375590085983,
+      "step": 42
+    },
+    {
+      "completion_length": 258.0625,
+      "epoch": 2.4571428571428573,
+      "grad_norm": 3.21244215965271,
+      "kl": 0.005789920185634401,
+      "learning_rate": 3.179575180590857e-07,
+      "loss": 0.0,
+      "reward": 4.591513024177402,
+      "reward_std": 1.265730170533061,
+      "rewards/concensus_correctness_reward_func": 1.9743749988265336,
+      "rewards/consensus_reward_func": 0.75,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.5825754599645734,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.47206250205636024,
+      "step": 44
+    },
+    {
+      "completion_length": 249.09375,
+      "epoch": 2.571428571428571,
+      "grad_norm": 2.6296839714050293,
+      "kl": 0.005435560931800865,
+      "learning_rate": 3.022418907578188e-07,
+      "loss": 0.0,
+      "reward": 4.408128134906292,
+      "reward_std": 2.0285469442605972,
+      "rewards/concensus_correctness_reward_func": 2.1132499971427023,
+      "rewards/consensus_reward_func": 0.6875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5583155920030549,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5334375011734664,
+      "step": 46
+    },
+    {
+      "completion_length": 308.3125,
+      "epoch": 2.685714285714286,
+      "grad_norm": 2.3870928287506104,
+      "kl": 0.003486459288978949,
+      "learning_rate": 2.863071428113726e-07,
+      "loss": 0.0,
+      "reward": 2.0714636370539665,
+      "reward_std": 1.128730148426257,
+      "rewards/concensus_correctness_reward_func": 0.17793750471173553,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5705886241048574,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.44793750811368227,
+      "step": 48
+    },
+    {
+      "completion_length": 350.34375,
+      "epoch": 2.8,
+      "grad_norm": 2.112053632736206,
+      "kl": 0.0031924354407237843,
+      "learning_rate": 2.7022011009035107e-07,
+      "loss": 0.0,
+      "reward": 2.2104606479406357,
+      "reward_std": 1.1965517563512549,
+      "rewards/concensus_correctness_reward_func": 0.26393750053830445,
+      "rewards/consensus_reward_func": 0.5625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5878356443718076,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.23368750466033816,
+      "step": 50
+    },
+    {
+      "completion_length": 282.90625,
+      "epoch": 2.914285714285714,
+      "grad_norm": 168.68026733398438,
+      "kl": 0.059496873698662966,
+      "learning_rate": 2.540482672006254e-07,
+      "loss": 0.0001,
+      "reward": 2.359485674649477,
+      "reward_std": 0.6959111683536321,
+      "rewards/concensus_correctness_reward_func": 0.4997500032186508,
+      "rewards/consensus_reward_func": 0.25,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.5438919421285391,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.5345937591046095,
+      "step": 52
+    },
+    {
+      "completion_length": 354.4583333333333,
+      "epoch": 3.0,
+      "grad_norm": 2.093106985092163,
+      "kl": 0.00446942588314414,
+      "learning_rate": 2.37859444471388e-07,
+      "loss": 0.0,
+      "reward": 3.0545214464267096,
+      "reward_std": 2.0393191116551557,
+      "rewards/concensus_correctness_reward_func": 1.0114166662096977,
+      "rewards/consensus_reward_func": 0.16666666666666666,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.7018964091936747,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.6745416695872942,
+      "step": 54
+    },
+    {
+      "completion_length": 308.96875,
+      "epoch": 3.1142857142857143,
+      "grad_norm": 1.8336759805679321,
+      "kl": 0.0041244168824050575,
+      "learning_rate": 2.2172154345117894e-07,
+      "loss": 0.0,
+      "reward": 2.694729525479488,
+      "reward_std": 2.6853361323010176,
+      "rewards/concensus_correctness_reward_func": 1.0250000013038516,
+      "rewards/consensus_reward_func": 0.5,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5355733069591224,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.19665626890491694,
+      "step": 56
+    },
+    {
+      "completion_length": 301.71875,
+      "epoch": 3.2285714285714286,
+      "grad_norm": 2.281761884689331,
+      "kl": 0.004181647425866686,
+      "learning_rate": 2.0570225210519433e-07,
+      "loss": 0.0,
+      "reward": 2.7115835566073656,
+      "reward_std": 2.3708576498320326,
+      "rewards/concensus_correctness_reward_func": 0.7873749984428287,
+      "rewards/consensus_reward_func": 0.5,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.4799272818490863,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.38178124325349927,
+      "step": 58
+    },
+    {
+      "completion_length": 295.90625,
+      "epoch": 3.342857142857143,
+      "grad_norm": 2.244230031967163,
+      "kl": 0.0050736206758301705,
+      "learning_rate": 1.8986876090843664e-07,
+      "loss": 0.0,
+      "reward": 4.905442409217358,
+      "reward_std": 1.7945623963605613,
+      "rewards/concensus_correctness_reward_func": 2.4236875250935555,
+      "rewards/consensus_reward_func": 0.6875,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.5710362014360726,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6450937511399388,
+      "step": 60
+    },
+    {
+      "completion_length": 318.59375,
+      "epoch": 3.4571428571428573,
+      "grad_norm": 2.056881904602051,
+      "kl": 0.004618284721800592,
+      "learning_rate": 1.7428748102551234e-07,
+      "loss": 0.0,
+      "reward": 4.192068429663777,
+      "reward_std": 1.9262854177504778,
+      "rewards/concensus_correctness_reward_func": 2.088000003132038,
+      "rewards/consensus_reward_func": 0.4375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.641599677503109,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.39996875007636845,
+      "step": 62
+    },
+    {
+      "completion_length": 286.96875,
+      "epoch": 3.571428571428571,
+      "grad_norm": 2.1847808361053467,
+      "kl": 0.005289344000630081,
+      "learning_rate": 1.5902376575912814e-07,
+      "loss": 0.0,
+      "reward": 2.9830123744904995,
+      "reward_std": 0.6361244827858172,
+      "rewards/concensus_correctness_reward_func": 1.307187500409782,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.47551230591489,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5128125082701445,
+      "step": 64
+    },
+    {
+      "completion_length": 308.125,
+      "epoch": 3.685714285714286,
+      "grad_norm": 1.9466278553009033,
+      "kl": 0.00550089653552277,
+      "learning_rate": 1.4414163643562753e-07,
+      "loss": 0.0,
+      "reward": 4.133690036833286,
+      "reward_std": 0.9184011179022491,
+      "rewards/concensus_correctness_reward_func": 1.6501249980647117,
+      "rewards/consensus_reward_func": 0.75,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5630338042974472,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5299062561243773,
+      "step": 66
+    },
+    {
+      "completion_length": 291.9375,
+      "epoch": 3.8,
+      "grad_norm": 3.8379456996917725,
+      "kl": 0.005396832886617631,
+      "learning_rate": 1.2970351387729872e-07,
+      "loss": 0.0,
+      "reward": 2.430345553904772,
+      "reward_std": 1.710412791930139,
+      "rewards/concensus_correctness_reward_func": 0.707187500782311,
+      "rewards/consensus_reward_func": 0.25,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6847830386832356,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5227500032633543,
+      "step": 68
+    },
+    {
+      "completion_length": 265.625,
+      "epoch": 3.914285714285714,
+      "grad_norm": 2.23298978805542,
+      "kl": 0.005603167533990927,
+      "learning_rate": 1.1576995658775404e-07,
+      "loss": 0.0,
+      "reward": 4.257293211296201,
+      "reward_std": 3.9666671017184854,
+      "rewards/concensus_correctness_reward_func": 1.9901874985080212,
+      "rewards/consensus_reward_func": 0.625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5431681228801608,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4583124993368983,
+      "step": 70
+    },
+    {
+      "completion_length": 291.2083333333333,
+      "epoch": 4.0,
+      "grad_norm": 1.4707306623458862,
+      "kl": 0.006299581126465152,
+      "learning_rate": 1.0239940674851941e-07,
+      "loss": 0.0,
+      "reward": 4.036697139342626,
+      "reward_std": 3.6838483214378357,
+      "rewards/concensus_correctness_reward_func": 1.85033332912523,
+      "rewards/consensus_reward_func": 0.3333333333333333,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.9166666666666666,
+      "rewards/question_recreation_reward_func": 0.5903221443295479,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.3252083510160446,
+      "step": 72
+    },
+    {
+      "completion_length": 355.53125,
+      "epoch": 4.114285714285714,
+      "grad_norm": 2.0232465267181396,
+      "kl": 0.005168267816770822,
+      "learning_rate": 8.964794509221507e-08,
+      "loss": 0.0,
+      "reward": 2.0521673914045095,
+      "reward_std": 1.2210392798297107,
+      "rewards/concensus_correctness_reward_func": 0.3000000002793968,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5546986176632345,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.44746875669807196,
+      "step": 74
+    },
+    {
+      "completion_length": 306.625,
+      "epoch": 4.228571428571429,
+      "grad_norm": 2.0220844745635986,
+      "kl": 0.005869668988452759,
+      "learning_rate": 7.756905568047392e-08,
+      "loss": 0.0,
+      "reward": 3.464091993868351,
+      "reward_std": 3.065386278554797,
+      "rewards/concensus_correctness_reward_func": 1.4443750018253922,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.6136857415549457,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4685312566580251,
+      "step": 76
+    },
+    {
+      "completion_length": 274.53125,
+      "epoch": 4.3428571428571425,
+      "grad_norm": 2.320235013961792,
+      "kl": 0.00582803861470893,
+      "learning_rate": 6.621340157319996e-08,
+      "loss": 0.0,
+      "reward": 4.306629652157426,
+      "reward_std": 3.718445436330512,
+      "rewards/concensus_correctness_reward_func": 2.290749993175268,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.625,
+      "rewards/question_recreation_reward_func": 0.5755982827395201,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.4246562549378723,
+      "step": 78
+    },
+    {
+      "completion_length": 280.65625,
+      "epoch": 4.457142857142857,
+      "grad_norm": 2.4888482093811035,
+      "kl": 0.007190108473878354,
+      "learning_rate": 5.5628612330087724e-08,
+      "loss": 0.0,
+      "reward": 4.881872668862343,
+      "reward_std": 3.823639538139105,
+      "rewards/concensus_correctness_reward_func": 2.405187502503395,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.875,
+      "rewards/question_recreation_reward_func": 0.6339664794504642,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5927187576889992,
+      "step": 80
+    },
+    {
+      "completion_length": 303.5,
+      "epoch": 4.571428571428571,
+      "grad_norm": 4.6361470222473145,
+      "kl": 0.005384259682614356,
+      "learning_rate": 4.5859084235697235e-08,
+      "loss": 0.0,
+      "reward": 3.0980553831905127,
+      "reward_std": 2.1309230010956526,
+      "rewards/concensus_correctness_reward_func": 1.0608749956518295,
+      "rewards/consensus_reward_func": 0.625,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5475553153082728,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.5990000087767839,
+      "step": 82
+    },
+    {
+      "completion_length": 266.28125,
+      "epoch": 4.685714285714286,
+      "grad_norm": 1.8511204719543457,
+      "kl": 0.006429586021113209,
+      "learning_rate": 3.6945794086007705e-08,
+      "loss": 0.0,
+      "reward": 4.905601989477873,
+      "reward_std": 1.8997747544199228,
+      "rewards/concensus_correctness_reward_func": 2.3535624709911644,
+      "rewards/consensus_reward_func": 0.75,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.75,
+      "rewards/question_recreation_reward_func": 0.603758230805397,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.44828125601634383,
+      "step": 84
+    },
+    {
+      "completion_length": 278.03125,
+      "epoch": 4.8,
+      "grad_norm": 1.860162377357483,
+      "kl": 0.00791130104335025,
+      "learning_rate": 2.892612731749414e-08,
+      "loss": 0.0,
+      "reward": 4.628453429788351,
+      "reward_std": 1.4046993185766041,
+      "rewards/concensus_correctness_reward_func": 2.1729374984279275,
+      "rewards/consensus_reward_func": 0.75,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6255784202367067,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5799375101923943,
+      "step": 86
+    },
+    {
+      "completion_length": 296.46875,
+      "epoch": 4.914285714285715,
+      "grad_norm": 1.8456752300262451,
+      "kl": 0.007626559119671583,
+      "learning_rate": 2.183372119961499e-08,
+      "loss": 0.0,
+      "reward": 2.681800600141287,
+      "reward_std": 1.8635689666261896,
+      "rewards/concensus_correctness_reward_func": 0.7971874834038317,
+      "rewards/consensus_reward_func": 0.3125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6776756662875414,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4569375063292682,
+      "step": 88
+    },
+    {
+      "completion_length": 290.5,
+      "epoch": 5.0,
+      "grad_norm": 2.047769546508789,
+      "kl": 0.006456721554665516,
+      "learning_rate": 1.5698323748414122e-08,
+      "loss": 0.0,
+      "reward": 2.1199893852074942,
+      "reward_std": 1.2634541131556034,
+      "rewards/concensus_correctness_reward_func": 0.1910833322132627,
+      "rewards/consensus_reward_func": 0.5833333333333334,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4166666666666667,
+      "rewards/question_recreation_reward_func": 0.5751561038196087,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3537500017943482,
+      "step": 90
+    },
+    {
+      "completion_length": 242.40625,
+      "epoch": 5.114285714285714,
+      "grad_norm": 2.607699394226074,
+      "kl": 0.007154520819312893,
+      "learning_rate": 1.054566895300324e-08,
+      "loss": 0.0,
+      "reward": 4.453570373356342,
+      "reward_std": 3.0469054598361254,
+      "rewards/concensus_correctness_reward_func": 2.308937451802194,
+      "rewards/consensus_reward_func": 0.4375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.8125,
+      "rewards/question_recreation_reward_func": 0.39563290192745626,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.48337499890476465,
+      "step": 92
+    },
+    {
+      "completion_length": 308.125,
+      "epoch": 5.228571428571429,
+      "grad_norm": 2.1194515228271484,
+      "kl": 0.006183088626130484,
+      "learning_rate": 6.397368838268496e-09,
+      "loss": 0.0,
+      "reward": 2.625663474202156,
+      "reward_std": 1.7279910603974713,
+      "rewards/concensus_correctness_reward_func": 0.7290624994784594,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.5077572092413902,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6232187608256936,
+      "step": 94
+    },
+    {
+      "completion_length": 274.21875,
+      "epoch": 5.3428571428571425,
+      "grad_norm": 2.2122576236724854,
+      "kl": 0.007067822094541043,
+      "learning_rate": 3.2708228165273244e-09,
+      "loss": 0.0,
+      "reward": 3.398947611451149,
+      "reward_std": 2.7149232206866145,
+      "rewards/concensus_correctness_reward_func": 1.5500000063329935,
+      "rewards/consensus_reward_func": 0.375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.5797913847491145,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.44103125110268593,
+      "step": 96
+    },
+    {
+      "completion_length": 313.75,
+      "epoch": 5.457142857142857,
+      "grad_norm": 1.9602198600769043,
+      "kl": 0.006589054362848401,
+      "learning_rate": 1.1791447083465133e-09,
+      "loss": 0.0,
+      "reward": 3.4984066113829613,
+      "reward_std": 2.1465693595819175,
+      "rewards/concensus_correctness_reward_func": 1.0073750019073486,
+      "rewards/consensus_reward_func": 0.4375,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.6875,
+      "rewards/question_recreation_reward_func": 0.6563441203907132,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.6940625086426735,
+      "step": 98
+    },
+    {
+      "completion_length": 299.125,
+      "epoch": 5.571428571428571,
+      "grad_norm": 2.126802682876587,
+      "kl": 0.007156282430514693,
+      "learning_rate": 1.3110773862126667e-10,
+      "loss": 0.0,
+      "reward": 3.7894926061853766,
+      "reward_std": 3.776049384730868,
+      "rewards/concensus_correctness_reward_func": 1.8608750000130385,
+      "rewards/consensus_reward_func": 0.5,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.665586419403553,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.49740625848062336,
+      "step": 100
+    },
+    {
+      "epoch": 5.571428571428571,
+      "step": 100,
+      "total_flos": 0.0,
+      "train_loss": 5.14280412971857e-06,
+      "train_runtime": 2155.4049,
+      "train_samples_per_second": 0.742,
+      "train_steps_per_second": 0.046
+    }
+  ],
+  "logging_steps": 2,
+  "max_steps": 100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 25,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd9497f02e3fb28ad58a4a416bd0548acc4b0efbe0cc9a1453eb443976f69957
+size 5944