Hugging Face's logo

Iedha
/
Qwen2.5-0.5B-Instruct-Gensyn-Swarm-lethal_tawny_deer

Text Generation
Transformers
Safetensors
qwen2
Generated from Trainer
rl-swarm
grpo
gensyn
I am lethal tawny deer
trl
genrl-swarm
I am lethal_tawny_deer
conversational
text-generation-inference
Model card Files Community
Iedha commited on
Commit
a1f2cf6
·
verified ·
1 Parent(s): 6a5d81f

End of training

Browse files
Files changed (10) hide show
  1. README.md +5 -5
  2. all_results.json +5 -5
  3. chat_template.jinja +54 -0
  4. config.json +1 -1
  5. generation_config.json +1 -1
  6. model.safetensors +1 -1
  7. tokenizer_config.json +0 -1
  8. train_results.json +5 -5
  9. trainer_state.json +203 -53
  10. training_args.bin +2 -2
README.md CHANGED
@@ -37,10 +37,10 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
37
 
38
  ### Framework versions
39
 
40
- - TRL: 0.15.2
41
- - Transformers: 4.51.2
42
- - Pytorch: 2.6.0
43
- - Datasets: 3.5.0
44
  - Tokenizers: 0.21.1
45
 
46
  ## Citations
@@ -62,7 +62,7 @@ Cite TRL as:
62
  ```bibtex
63
  @misc{vonwerra2022trl,
64
  title = {{TRL: Transformer Reinforcement Learning}},
65
- author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
66
  year = 2020,
67
  journal = {GitHub repository},
68
  publisher = {GitHub},
 
37
 
38
  ### Framework versions
39
 
40
+ - TRL: 0.18.2
41
+ - Transformers: 4.52.4
42
+ - Pytorch: 2.7.1
43
+ - Datasets: 3.6.0
44
  - Tokenizers: 0.21.1
45
 
46
  ## Citations
 
62
  ```bibtex
63
  @misc{vonwerra2022trl,
64
  title = {{TRL: Transformer Reinforcement Learning}},
65
+ author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
66
  year = 2020,
67
  journal = {GitHub repository},
68
  publisher = {GitHub},
all_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "total_flos": 0.0,
3
- "train_loss": 1.8887146507040597e-05,
4
- "train_runtime": 634.9749,
5
- "train_samples": 3,
6
- "train_samples_per_second": 0.031,
7
- "train_steps_per_second": 0.008
8
  }
 
1
  {
2
  "total_flos": 0.0,
3
+ "train_loss": 0.0012827153317630292,
4
+ "train_runtime": 3022.2304,
5
+ "train_samples": 2,
6
+ "train_samples_per_second": 0.013,
7
+ "train_steps_per_second": 0.003
8
  }
chat_template.jinja ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if tools %}
2
+ {{- '<|im_start|>system\n' }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {{- messages[0]['content'] }}
5
+ {%- else %}
6
+ {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
7
+ {%- endif %}
8
+ {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
9
+ {%- for tool in tools %}
10
+ {{- "\n" }}
11
+ {{- tool | tojson }}
12
+ {%- endfor %}
13
+ {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
14
+ {%- else %}
15
+ {%- if messages[0]['role'] == 'system' %}
16
+ {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
17
+ {%- else %}
18
+ {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
19
+ {%- endif %}
20
+ {%- endif %}
21
+ {%- for message in messages %}
22
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
23
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
24
+ {%- elif message.role == "assistant" %}
25
+ {{- '<|im_start|>' + message.role }}
26
+ {%- if message.content %}
27
+ {{- '\n' + message.content }}
28
+ {%- endif %}
29
+ {%- for tool_call in message.tool_calls %}
30
+ {%- if tool_call.function is defined %}
31
+ {%- set tool_call = tool_call.function %}
32
+ {%- endif %}
33
+ {{- '\n<tool_call>\n{"name": "' }}
34
+ {{- tool_call.name }}
35
+ {{- '", "arguments": ' }}
36
+ {{- tool_call.arguments | tojson }}
37
+ {{- '}\n</tool_call>' }}
38
+ {%- endfor %}
39
+ {{- '<|im_end|>\n' }}
40
+ {%- elif message.role == "tool" %}
41
+ {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
42
+ {{- '<|im_start|>user' }}
43
+ {%- endif %}
44
+ {{- '\n<tool_response>\n' }}
45
+ {{- message.content }}
46
+ {{- '\n</tool_response>' }}
47
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
48
+ {{- '<|im_end|>\n' }}
49
+ {%- endif %}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {%- if add_generation_prompt %}
53
+ {{- '<|im_start|>assistant\n' }}
54
+ {%- endif %}
config.json CHANGED
@@ -21,7 +21,7 @@
21
  "sliding_window": null,
22
  "tie_word_embeddings": true,
23
  "torch_dtype": "float32",
24
- "transformers_version": "4.51.2",
25
  "unsloth_fixed": true,
26
  "use_cache": true,
27
  "use_sliding_window": false,
 
21
  "sliding_window": null,
22
  "tie_word_embeddings": true,
23
  "torch_dtype": "float32",
24
+ "transformers_version": "4.52.4",
25
  "unsloth_fixed": true,
26
  "use_cache": true,
27
  "use_sliding_window": false,
generation_config.json CHANGED
@@ -11,5 +11,5 @@
11
  "temperature": 0.7,
12
  "top_k": 20,
13
  "top_p": 0.8,
14
- "transformers_version": "4.51.2"
15
  }
 
11
  "temperature": 0.7,
12
  "top_k": 20,
13
  "top_p": 0.8,
14
+ "transformers_version": "4.52.4"
15
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8fbb360e072f0e9fd844c62ece43a03593645246cc769019d60821d08916927b
3
  size 1976163472
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6347e2dfdbe7717ba9e9c6aadd257224c8d6cf8d9c313596d00d8ffa25d00dd6
3
  size 1976163472
tokenizer_config.json CHANGED
@@ -195,7 +195,6 @@
195
  "<|video_pad|>"
196
  ],
197
  "bos_token": null,
198
- "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n<tool_call>\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n</tool_call>' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- message.content }}\n {{- '\\n</tool_response>' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
199
  "clean_up_tokenization_spaces": false,
200
  "eos_token": "<|im_end|>",
201
  "errors": "replace",
 
195
  "<|video_pad|>"
196
  ],
197
  "bos_token": null,
 
198
  "clean_up_tokenization_spaces": false,
199
  "eos_token": "<|im_end|>",
200
  "errors": "replace",
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "total_flos": 0.0,
3
- "train_loss": 1.8887146507040597e-05,
4
- "train_runtime": 634.9749,
5
- "train_samples": 3,
6
- "train_samples_per_second": 0.031,
7
- "train_steps_per_second": 0.008
8
  }
 
1
  {
2
  "total_flos": 0.0,
3
+ "train_loss": 0.0012827153317630292,
4
+ "train_runtime": 3022.2304,
5
+ "train_samples": 2,
6
+ "train_samples_per_second": 0.013,
7
+ "train_steps_per_second": 0.003
8
  }
trainer_state.json CHANGED
@@ -2,77 +2,227 @@
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
- "epoch": 2.6666666666666665,
6
  "eval_steps": 500,
7
- "global_step": 5,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
- "completion_length": 51.833333333333336,
14
- "epoch": 1.0,
15
- "grad_norm": 13.95437240600586,
 
 
 
 
 
 
 
 
 
 
 
 
16
  "kl": 0.0,
17
  "learning_rate": 5e-07,
18
- "loss": -0.0,
19
- "reward": 1.5421740214029949,
20
- "reward_std": 0.13302947270373502,
21
- "rewards/concensus_correctness_reward_func": 0.0,
22
- "rewards/consensus_reward_func": 0.0,
23
- "rewards/cumulative_reward_2": 0.0,
24
- "rewards/final_correctness_reward_func": 1.3333333333333333,
25
- "rewards/question_recreation_reward_func": 0.20884071787198386,
26
- "rewards/soft_format_reward_func": 0.0,
27
- "rewards/strict_format_reward_func": 0.0,
28
- "rewards/xmlcount_reward_func": 0.0,
 
 
 
 
 
 
 
 
 
29
  "step": 2
30
  },
31
  {
32
- "completion_length": 44.833333333333336,
33
- "epoch": 2.0,
34
- "grad_norm": 0.002443525940179825,
35
- "kl": 0.009695128072053194,
36
- "learning_rate": 2.5e-07,
37
- "loss": 0.0,
38
- "reward": 1.5598274767398834,
39
- "reward_std": 0.021924182772636414,
40
- "rewards/concensus_correctness_reward_func": 0.0,
41
- "rewards/consensus_reward_func": 0.0,
42
- "rewards/cumulative_reward_2": 0.0,
43
- "rewards/final_correctness_reward_func": 1.3333333333333333,
44
- "rewards/question_recreation_reward_func": 0.22649407386779785,
45
- "rewards/soft_format_reward_func": 0.0,
46
- "rewards/strict_format_reward_func": 0.0,
47
- "rewards/xmlcount_reward_func": 0.0,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  "step": 4
49
  },
50
  {
51
- "completion_length": 46.75,
52
- "epoch": 2.6666666666666665,
53
- "kl": 0.07859085127711296,
54
- "reward": 0.7222268432378769,
55
- "reward_std": 0.8127293139696121,
56
- "rewards/concensus_correctness_reward_func": 0.0,
57
- "rewards/consensus_reward_func": 0.0,
58
- "rewards/cumulative_reward_2": 0.0,
59
- "rewards/final_correctness_reward_func": 0.5,
60
- "rewards/question_recreation_reward_func": 0.22222686558961868,
61
- "rewards/soft_format_reward_func": 0.0,
62
- "rewards/strict_format_reward_func": 0.0,
63
- "rewards/xmlcount_reward_func": 0.0,
64
- "step": 5,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  "total_flos": 0.0,
66
- "train_loss": 1.8887146507040597e-05,
67
- "train_runtime": 634.9749,
68
- "train_samples_per_second": 0.031,
69
- "train_steps_per_second": 0.008
70
  }
71
  ],
72
  "logging_steps": 2,
73
- "max_steps": 5,
74
- "num_input_tokens_seen": 0,
75
- "num_train_epochs": 5,
76
  "save_steps": 10,
77
  "stateful_callbacks": {
78
  "TrainerControl": {
 
2
  "best_global_step": null,
3
  "best_metric": null,
4
  "best_model_checkpoint": null,
5
+ "epoch": 10.0,
6
  "eval_steps": 500,
7
+ "global_step": 10,
8
  "is_hyper_param_search": false,
9
  "is_local_process_zero": true,
10
  "is_world_process_zero": true,
11
  "log_history": [
12
  {
13
+ "clip_ratio/high_max": 0.0,
14
+ "clip_ratio/high_mean": 0.0,
15
+ "clip_ratio/low_mean": 0.0,
16
+ "clip_ratio/low_min": 0.0,
17
+ "clip_ratio/region_mean": 0.0,
18
+ "completions/clipped_ratio": 0.5,
19
+ "completions/max_length": 256.0,
20
+ "completions/max_terminated_length": 199.0,
21
+ "completions/mean_length": 194.375,
22
+ "completions/mean_terminated_length": 172.5,
23
+ "completions/min_length": 151.5,
24
+ "completions/min_terminated_length": 151.5,
25
+ "epoch": 2.0,
26
+ "frac_reward_zero_std": 0.0,
27
+ "grad_norm": 8.42098331451416,
28
  "kl": 0.0,
29
  "learning_rate": 5e-07,
30
+ "loss": 0.0055,
31
+ "num_tokens": 2579.0,
32
+ "reward": 0.014096824452280998,
33
+ "reward_std": 0.00433365476783365,
34
+ "rewards/concensus_correctness_reward_func/mean": 0.0,
35
+ "rewards/concensus_correctness_reward_func/std": 0.0,
36
+ "rewards/consensus_reward_func/mean": 0.0,
37
+ "rewards/consensus_reward_func/std": 0.0,
38
+ "rewards/cumulative_reward_2/mean": 0.0,
39
+ "rewards/cumulative_reward_2/std": 0.0,
40
+ "rewards/final_correctness_reward_func/mean": 0.0,
41
+ "rewards/final_correctness_reward_func/std": 0.0,
42
+ "rewards/question_recreation_reward_func/mean": 0.014096824452280998,
43
+ "rewards/question_recreation_reward_func/std": 0.007435560924932361,
44
+ "rewards/soft_format_reward_func/mean": 0.0,
45
+ "rewards/soft_format_reward_func/std": 0.0,
46
+ "rewards/strict_format_reward_func/mean": 0.0,
47
+ "rewards/strict_format_reward_func/std": 0.0,
48
+ "rewards/xmlcount_reward_func/mean": 0.0,
49
+ "rewards/xmlcount_reward_func/std": 0.0,
50
  "step": 2
51
  },
52
  {
53
+ "clip_ratio/high_max": 0.0,
54
+ "clip_ratio/high_mean": 0.0,
55
+ "clip_ratio/low_mean": 0.0,
56
+ "clip_ratio/low_min": 0.0,
57
+ "clip_ratio/region_mean": 0.0,
58
+ "completions/clipped_ratio": 0.375,
59
+ "completions/max_length": 256.0,
60
+ "completions/max_terminated_length": 167.5,
61
+ "completions/mean_length": 182.375,
62
+ "completions/mean_terminated_length": 134.91666412353516,
63
+ "completions/min_length": 102.5,
64
+ "completions/min_terminated_length": 102.5,
65
+ "epoch": 4.0,
66
+ "frac_reward_zero_std": 0.0,
67
+ "grad_norm": 9.635068893432617,
68
+ "kl": 0.0003269764692959143,
69
+ "learning_rate": 4.415111107797445e-07,
70
+ "loss": 0.0833,
71
+ "num_tokens": 5062.0,
72
+ "reward": 0.012714702636003494,
73
+ "reward_std": 0.004921070067211986,
74
+ "rewards/concensus_correctness_reward_func/mean": 0.0,
75
+ "rewards/concensus_correctness_reward_func/std": 0.0,
76
+ "rewards/consensus_reward_func/mean": 0.0,
77
+ "rewards/consensus_reward_func/std": 0.0,
78
+ "rewards/cumulative_reward_2/mean": 0.0,
79
+ "rewards/cumulative_reward_2/std": 0.0,
80
+ "rewards/final_correctness_reward_func/mean": 0.0,
81
+ "rewards/final_correctness_reward_func/std": 0.0,
82
+ "rewards/question_recreation_reward_func/mean": 0.012714702636003494,
83
+ "rewards/question_recreation_reward_func/std": 0.011344576952978969,
84
+ "rewards/soft_format_reward_func/mean": 0.0,
85
+ "rewards/soft_format_reward_func/std": 0.0,
86
+ "rewards/strict_format_reward_func/mean": 0.0,
87
+ "rewards/strict_format_reward_func/std": 0.0,
88
+ "rewards/xmlcount_reward_func/mean": 0.0,
89
+ "rewards/xmlcount_reward_func/std": 0.0,
90
  "step": 4
91
  },
92
  {
93
+ "clip_ratio/high_max": 0.0,
94
+ "clip_ratio/high_mean": 0.0,
95
+ "clip_ratio/low_mean": 0.0,
96
+ "clip_ratio/low_min": 0.0,
97
+ "clip_ratio/region_mean": 0.0,
98
+ "completions/clipped_ratio": 0.125,
99
+ "completions/max_length": 237.5,
100
+ "completions/max_terminated_length": 190.5,
101
+ "completions/mean_length": 136.75,
102
+ "completions/mean_terminated_length": 116.41666793823242,
103
+ "completions/min_length": 30.0,
104
+ "completions/min_terminated_length": 30.0,
105
+ "epoch": 6.0,
106
+ "frac_reward_zero_std": 0.0,
107
+ "grad_norm": 14.077054977416992,
108
+ "kl": 0.0014197196724126115,
109
+ "learning_rate": 2.934120444167326e-07,
110
+ "loss": 0.0104,
111
+ "num_tokens": 7180.0,
112
+ "reward": 0.019166121259331703,
113
+ "reward_std": 0.013739365618675947,
114
+ "rewards/concensus_correctness_reward_func/mean": 0.0,
115
+ "rewards/concensus_correctness_reward_func/std": 0.0,
116
+ "rewards/consensus_reward_func/mean": 0.0,
117
+ "rewards/consensus_reward_func/std": 0.0,
118
+ "rewards/cumulative_reward_2/mean": 0.0,
119
+ "rewards/cumulative_reward_2/std": 0.0,
120
+ "rewards/final_correctness_reward_func/mean": 0.0,
121
+ "rewards/final_correctness_reward_func/std": 0.0,
122
+ "rewards/question_recreation_reward_func/mean": 0.019166121259331703,
123
+ "rewards/question_recreation_reward_func/std": 0.018057925160974264,
124
+ "rewards/soft_format_reward_func/mean": 0.0,
125
+ "rewards/soft_format_reward_func/std": 0.0,
126
+ "rewards/strict_format_reward_func/mean": 0.0,
127
+ "rewards/strict_format_reward_func/std": 0.0,
128
+ "rewards/xmlcount_reward_func/mean": 0.0,
129
+ "rewards/xmlcount_reward_func/std": 0.0,
130
+ "step": 6
131
+ },
132
+ {
133
+ "clip_ratio/high_max": 0.0,
134
+ "clip_ratio/high_mean": 0.0,
135
+ "clip_ratio/low_mean": 0.0,
136
+ "clip_ratio/low_min": 0.0,
137
+ "clip_ratio/region_mean": 0.0,
138
+ "completions/clipped_ratio": 0.125,
139
+ "completions/max_length": 239.5,
140
+ "completions/max_terminated_length": 151.0,
141
+ "completions/mean_length": 134.25,
142
+ "completions/mean_terminated_length": 108.08333396911621,
143
+ "completions/min_length": 39.0,
144
+ "completions/min_terminated_length": 39.0,
145
+ "epoch": 8.0,
146
+ "frac_reward_zero_std": 0.0,
147
+ "grad_norm": 10.22560977935791,
148
+ "kl": 0.004310431773774326,
149
+ "learning_rate": 1.2500000000000005e-07,
150
+ "loss": -0.054,
151
+ "num_tokens": 9278.0,
152
+ "reward": 0.013700914569199085,
153
+ "reward_std": 0.00266791571630165,
154
+ "rewards/concensus_correctness_reward_func/mean": 0.0,
155
+ "rewards/concensus_correctness_reward_func/std": 0.0,
156
+ "rewards/consensus_reward_func/mean": 0.0,
157
+ "rewards/consensus_reward_func/std": 0.0,
158
+ "rewards/cumulative_reward_2/mean": 0.0,
159
+ "rewards/cumulative_reward_2/std": 0.0,
160
+ "rewards/final_correctness_reward_func/mean": 0.0,
161
+ "rewards/final_correctness_reward_func/std": 0.0,
162
+ "rewards/question_recreation_reward_func/mean": 0.013700914569199085,
163
+ "rewards/question_recreation_reward_func/std": 0.006989771965891123,
164
+ "rewards/soft_format_reward_func/mean": 0.0,
165
+ "rewards/soft_format_reward_func/std": 0.0,
166
+ "rewards/strict_format_reward_func/mean": 0.0,
167
+ "rewards/strict_format_reward_func/std": 0.0,
168
+ "rewards/xmlcount_reward_func/mean": 0.0,
169
+ "rewards/xmlcount_reward_func/std": 0.0,
170
+ "step": 8
171
+ },
172
+ {
173
+ "clip_ratio/high_max": 0.0,
174
+ "clip_ratio/high_mean": 0.0,
175
+ "clip_ratio/low_mean": 0.0,
176
+ "clip_ratio/low_min": 0.0,
177
+ "clip_ratio/region_mean": 0.0,
178
+ "completions/clipped_ratio": 0.125,
179
+ "completions/max_length": 240.5,
180
+ "completions/max_terminated_length": 211.5,
181
+ "completions/mean_length": 145.5,
182
+ "completions/mean_terminated_length": 133.5,
183
+ "completions/min_length": 72.5,
184
+ "completions/min_terminated_length": 72.5,
185
+ "epoch": 10.0,
186
+ "frac_reward_zero_std": 0.0,
187
+ "grad_norm": 10.423093795776367,
188
+ "kl": 0.0022090721467975527,
189
+ "learning_rate": 1.507684480352292e-08,
190
+ "loss": -0.0387,
191
+ "num_tokens": 11466.0,
192
+ "reward": 0.02149433922022581,
193
+ "reward_std": 0.007891800487414002,
194
+ "rewards/concensus_correctness_reward_func/mean": 0.0,
195
+ "rewards/concensus_correctness_reward_func/std": 0.0,
196
+ "rewards/consensus_reward_func/mean": 0.0,
197
+ "rewards/consensus_reward_func/std": 0.0,
198
+ "rewards/cumulative_reward_2/mean": 0.0,
199
+ "rewards/cumulative_reward_2/std": 0.0,
200
+ "rewards/final_correctness_reward_func/mean": 0.0,
201
+ "rewards/final_correctness_reward_func/std": 0.0,
202
+ "rewards/question_recreation_reward_func/mean": 0.021494340151548386,
203
+ "rewards/question_recreation_reward_func/std": 0.010832870844751596,
204
+ "rewards/soft_format_reward_func/mean": 0.0,
205
+ "rewards/soft_format_reward_func/std": 0.0,
206
+ "rewards/strict_format_reward_func/mean": 0.0,
207
+ "rewards/strict_format_reward_func/std": 0.0,
208
+ "rewards/xmlcount_reward_func/mean": 0.0,
209
+ "rewards/xmlcount_reward_func/std": 0.0,
210
+ "step": 10
211
+ },
212
+ {
213
+ "epoch": 10.0,
214
+ "step": 10,
215
  "total_flos": 0.0,
216
+ "train_loss": 0.0012827153317630292,
217
+ "train_runtime": 3022.2304,
218
+ "train_samples_per_second": 0.013,
219
+ "train_steps_per_second": 0.003
220
  }
221
  ],
222
  "logging_steps": 2,
223
+ "max_steps": 10,
224
+ "num_input_tokens_seen": 11466,
225
+ "num_train_epochs": 10,
226
  "save_steps": 10,
227
  "stateful_callbacks": {
228
  "TrainerControl": {
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3aa9c8a176388e7c76ae75de6603d7296782923246e194a887bcf0c71fb987f4
3
- size 5944
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ace81fafb2f74343f27f2e3311dbbdfe8fbc4cc148781657ee0886855a89c4a
3
+ size 6865