Hugging Face's logo

QinLiuNLP
/
llama3-sudo-dpo-instruct-1epoch-new-key

PEFT
TensorBoard
Safetensors
trl
dpo
Generated from Trainer
Model card Files
xet
Metrics Community
Jackie999 commited on
Commit
2b3f05a
·
verified ·
1 Parent(s): cf67310

Model save

Browse files
Files changed (4) hide show
  1. README.md +62 -0
  2. all_results.json +9 -0
  3. train_results.json +9 -0
  4. trainer_state.json +387 -0
README.md ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: meta-llama/Meta-Llama-3-8B-Instruct
3
+ library_name: peft
4
+ license: llama3
5
+ tags:
6
+ - trl
7
+ - dpo
8
+ - generated_from_trainer
9
+ model-index:
10
+ - name: llama3-sudo-dpo-instruct-1epoch-new-key
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # llama3-sudo-dpo-instruct-1epoch-new-key
18
+
19
+ This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on an unknown dataset.
20
+
21
+ ## Model description
22
+
23
+ More information needed
24
+
25
+ ## Intended uses & limitations
26
+
27
+ More information needed
28
+
29
+ ## Training and evaluation data
30
+
31
+ More information needed
32
+
33
+ ## Training procedure
34
+
35
+ ### Training hyperparameters
36
+
37
+ The following hyperparameters were used during training:
38
+ - learning_rate: 5e-06
39
+ - train_batch_size: 2
40
+ - eval_batch_size: 8
41
+ - seed: 42
42
+ - distributed_type: multi-GPU
43
+ - num_devices: 8
44
+ - gradient_accumulation_steps: 4
45
+ - total_train_batch_size: 64
46
+ - total_eval_batch_size: 64
47
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
48
+ - lr_scheduler_type: cosine
49
+ - lr_scheduler_warmup_ratio: 0.1
50
+ - num_epochs: 1
51
+
52
+ ### Training results
53
+
54
+
55
+
56
+ ### Framework versions
57
+
58
+ - PEFT 0.12.0
59
+ - Transformers 4.44.0
60
+ - Pytorch 2.1.2
61
+ - Datasets 2.20.0
62
+ - Tokenizers 0.19.1
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.16486623261823405,
5
+ "train_runtime": 2148.0634,
6
+ "train_samples": 14642,
7
+ "train_samples_per_second": 6.816,
8
+ "train_steps_per_second": 0.107
9
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 1.0,
3
+ "total_flos": 0.0,
4
+ "train_loss": 0.16486623261823405,
5
+ "train_runtime": 2148.0634,
6
+ "train_samples": 14642,
7
+ "train_samples_per_second": 6.816,
8
+ "train_steps_per_second": 0.107
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,387 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.0,
5
+ "eval_steps": 1000,
6
+ "global_step": 229,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.004366812227074236,
13
+ "grad_norm": 0.4543283495867887,
14
+ "learning_rate": 2.173913043478261e-07,
15
+ "logits/chosen": -0.9449982047080994,
16
+ "logits/rejected": -0.915056049823761,
17
+ "logps/chosen": -328.08038330078125,
18
+ "logps/rejected": -274.1044921875,
19
+ "loss": 0.6931,
20
+ "rewards/accuracies": 0.0,
21
+ "rewards/chosen": 0.0,
22
+ "rewards/margins": 0.0,
23
+ "rewards/rejected": 0.0,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.043668122270742356,
28
+ "grad_norm": 0.47008608128274937,
29
+ "learning_rate": 2.173913043478261e-06,
30
+ "logits/chosen": -0.9651342034339905,
31
+ "logits/rejected": -1.0432785749435425,
32
+ "logps/chosen": -326.21295166015625,
33
+ "logps/rejected": -277.12603759765625,
34
+ "loss": 0.6928,
35
+ "rewards/accuracies": 0.4583333432674408,
36
+ "rewards/chosen": 0.000499483838211745,
37
+ "rewards/margins": 0.00042585484334267676,
38
+ "rewards/rejected": 7.362888572970405e-05,
39
+ "step": 10
40
+ },
41
+ {
42
+ "epoch": 0.08733624454148471,
43
+ "grad_norm": 0.5077455364034207,
44
+ "learning_rate": 4.347826086956522e-06,
45
+ "logits/chosen": -1.0618306398391724,
46
+ "logits/rejected": -1.009238600730896,
47
+ "logps/chosen": -334.39141845703125,
48
+ "logps/rejected": -346.700927734375,
49
+ "loss": 0.6882,
50
+ "rewards/accuracies": 0.699999988079071,
51
+ "rewards/chosen": -0.0003057918802369386,
52
+ "rewards/margins": 0.00985435489565134,
53
+ "rewards/rejected": -0.010160146281123161,
54
+ "step": 20
55
+ },
56
+ {
57
+ "epoch": 0.13100436681222707,
58
+ "grad_norm": 0.5720215285315815,
59
+ "learning_rate": 4.985768230048011e-06,
60
+ "logits/chosen": -1.0507558584213257,
61
+ "logits/rejected": -1.075430989265442,
62
+ "logps/chosen": -308.62359619140625,
63
+ "logps/rejected": -301.9061279296875,
64
+ "loss": 0.6651,
65
+ "rewards/accuracies": 0.887499988079071,
66
+ "rewards/chosen": 0.0059205591678619385,
67
+ "rewards/margins": 0.05983991548418999,
68
+ "rewards/rejected": -0.053919363766908646,
69
+ "step": 30
70
+ },
71
+ {
72
+ "epoch": 0.17467248908296942,
73
+ "grad_norm": 0.8551226198033497,
74
+ "learning_rate": 4.9164513914144005e-06,
75
+ "logits/chosen": -1.1152777671813965,
76
+ "logits/rejected": -1.1143274307250977,
77
+ "logps/chosen": -288.18585205078125,
78
+ "logps/rejected": -310.5125732421875,
79
+ "loss": 0.5986,
80
+ "rewards/accuracies": 0.9125000238418579,
81
+ "rewards/chosen": -0.003953046165406704,
82
+ "rewards/margins": 0.21825852990150452,
83
+ "rewards/rejected": -0.2222115695476532,
84
+ "step": 40
85
+ },
86
+ {
87
+ "epoch": 0.2183406113537118,
88
+ "grad_norm": 1.0672212168131259,
89
+ "learning_rate": 4.791042480696179e-06,
90
+ "logits/chosen": -1.1128385066986084,
91
+ "logits/rejected": -1.0449755191802979,
92
+ "logps/chosen": -317.44451904296875,
93
+ "logps/rejected": -390.5851135253906,
94
+ "loss": 0.4535,
95
+ "rewards/accuracies": 0.9750000238418579,
96
+ "rewards/chosen": 0.03481646627187729,
97
+ "rewards/margins": 0.6241441965103149,
98
+ "rewards/rejected": -0.5893277525901794,
99
+ "step": 50
100
+ },
101
+ {
102
+ "epoch": 0.26200873362445415,
103
+ "grad_norm": 0.6925238532589472,
104
+ "learning_rate": 4.612452562309975e-06,
105
+ "logits/chosen": -1.0924533605575562,
106
+ "logits/rejected": -1.0897819995880127,
107
+ "logps/chosen": -312.33837890625,
108
+ "logps/rejected": -432.6812438964844,
109
+ "loss": 0.2332,
110
+ "rewards/accuracies": 0.9750000238418579,
111
+ "rewards/chosen": 0.08799274265766144,
112
+ "rewards/margins": 1.521544098854065,
113
+ "rewards/rejected": -1.4335514307022095,
114
+ "step": 60
115
+ },
116
+ {
117
+ "epoch": 0.3056768558951965,
118
+ "grad_norm": 0.8913228040925837,
119
+ "learning_rate": 4.384827169085993e-06,
120
+ "logits/chosen": -1.0938880443572998,
121
+ "logits/rejected": -1.1012872457504272,
122
+ "logps/chosen": -322.24139404296875,
123
+ "logps/rejected": -572.3203125,
124
+ "loss": 0.114,
125
+ "rewards/accuracies": 0.987500011920929,
126
+ "rewards/chosen": -0.0598033182322979,
127
+ "rewards/margins": 2.8274989128112793,
128
+ "rewards/rejected": -2.8873023986816406,
129
+ "step": 70
130
+ },
131
+ {
132
+ "epoch": 0.34934497816593885,
133
+ "grad_norm": 1.025805090974843,
134
+ "learning_rate": 4.1134500737541026e-06,
135
+ "logits/chosen": -1.0882785320281982,
136
+ "logits/rejected": -0.9281471371650696,
137
+ "logps/chosen": -279.4626770019531,
138
+ "logps/rejected": -729.4642333984375,
139
+ "loss": 0.0632,
140
+ "rewards/accuracies": 1.0,
141
+ "rewards/chosen": 0.04784034937620163,
142
+ "rewards/margins": 4.184046745300293,
143
+ "rewards/rejected": -4.13620662689209,
144
+ "step": 80
145
+ },
146
+ {
147
+ "epoch": 0.3930131004366812,
148
+ "grad_norm": 0.7021281871664607,
149
+ "learning_rate": 3.8046206389447916e-06,
150
+ "logits/chosen": -0.9880226850509644,
151
+ "logits/rejected": -0.9430360794067383,
152
+ "logps/chosen": -297.018310546875,
153
+ "logps/rejected": -715.1465454101562,
154
+ "loss": 0.0491,
155
+ "rewards/accuracies": 1.0,
156
+ "rewards/chosen": 0.10021056234836578,
157
+ "rewards/margins": 4.483721733093262,
158
+ "rewards/rejected": -4.383510589599609,
159
+ "step": 90
160
+ },
161
+ {
162
+ "epoch": 0.4366812227074236,
163
+ "grad_norm": 0.586209743573467,
164
+ "learning_rate": 3.4655075927279576e-06,
165
+ "logits/chosen": -0.9809297323226929,
166
+ "logits/rejected": -0.8725347518920898,
167
+ "logps/chosen": -311.753662109375,
168
+ "logps/rejected": -830.88671875,
169
+ "loss": 0.0353,
170
+ "rewards/accuracies": 1.0,
171
+ "rewards/chosen": -0.36457541584968567,
172
+ "rewards/margins": 4.913618564605713,
173
+ "rewards/rejected": -5.278193950653076,
174
+ "step": 100
175
+ },
176
+ {
177
+ "epoch": 0.48034934497816595,
178
+ "grad_norm": 0.3979818436750899,
179
+ "learning_rate": 3.1039826239365754e-06,
180
+ "logits/chosen": -0.9446805119514465,
181
+ "logits/rejected": -0.8940761685371399,
182
+ "logps/chosen": -431.5677795410156,
183
+ "logps/rejected": -925.68017578125,
184
+ "loss": 0.0288,
185
+ "rewards/accuracies": 0.987500011920929,
186
+ "rewards/chosen": -1.047907829284668,
187
+ "rewards/margins": 5.1847357749938965,
188
+ "rewards/rejected": -6.232643127441406,
189
+ "step": 110
190
+ },
191
+ {
192
+ "epoch": 0.5240174672489083,
193
+ "grad_norm": 0.44864039181837617,
194
+ "learning_rate": 2.7284376599571776e-06,
195
+ "logits/chosen": -0.9638189077377319,
196
+ "logits/rejected": -0.8882731199264526,
197
+ "logps/chosen": -392.0179748535156,
198
+ "logps/rejected": -961.5611572265625,
199
+ "loss": 0.0207,
200
+ "rewards/accuracies": 1.0,
201
+ "rewards/chosen": -0.6613569259643555,
202
+ "rewards/margins": 6.055957794189453,
203
+ "rewards/rejected": -6.71731424331665,
204
+ "step": 120
205
+ },
206
+ {
207
+ "epoch": 0.5676855895196506,
208
+ "grad_norm": 0.15662373138721433,
209
+ "learning_rate": 2.3475900684411027e-06,
210
+ "logits/chosen": -0.97578364610672,
211
+ "logits/rejected": -0.8818238973617554,
212
+ "logps/chosen": -417.825439453125,
213
+ "logps/rejected": -989.6531372070312,
214
+ "loss": 0.0229,
215
+ "rewards/accuracies": 0.987500011920929,
216
+ "rewards/chosen": -0.840344250202179,
217
+ "rewards/margins": 5.920441627502441,
218
+ "rewards/rejected": -6.760786533355713,
219
+ "step": 130
220
+ },
221
+ {
222
+ "epoch": 0.611353711790393,
223
+ "grad_norm": 1.0644930727731667,
224
+ "learning_rate": 1.970280304707447e-06,
225
+ "logits/chosen": -1.0020447969436646,
226
+ "logits/rejected": -0.8326355814933777,
227
+ "logps/chosen": -390.6119384765625,
228
+ "logps/rejected": -1086.9949951171875,
229
+ "loss": 0.012,
230
+ "rewards/accuracies": 1.0,
231
+ "rewards/chosen": -0.8912075757980347,
232
+ "rewards/margins": 6.5178542137146,
233
+ "rewards/rejected": -7.40906286239624,
234
+ "step": 140
235
+ },
236
+ {
237
+ "epoch": 0.6550218340611353,
238
+ "grad_norm": 0.33838570745300933,
239
+ "learning_rate": 1.6052667019636462e-06,
240
+ "logits/chosen": -0.9777904748916626,
241
+ "logits/rejected": -0.9008253216743469,
242
+ "logps/chosen": -421.56658935546875,
243
+ "logps/rejected": -1008.5904541015625,
244
+ "loss": 0.0166,
245
+ "rewards/accuracies": 1.0,
246
+ "rewards/chosen": -1.0421282052993774,
247
+ "rewards/margins": 6.398242950439453,
248
+ "rewards/rejected": -7.440371513366699,
249
+ "step": 150
250
+ },
251
+ {
252
+ "epoch": 0.6986899563318777,
253
+ "grad_norm": 0.1209081830310832,
254
+ "learning_rate": 1.261022167792161e-06,
255
+ "logits/chosen": -1.0033915042877197,
256
+ "logits/rejected": -0.8852731585502625,
257
+ "logps/chosen": -402.15692138671875,
258
+ "logps/rejected": -1126.320556640625,
259
+ "loss": 0.011,
260
+ "rewards/accuracies": 1.0,
261
+ "rewards/chosen": -0.8826059103012085,
262
+ "rewards/margins": 6.96124267578125,
263
+ "rewards/rejected": -7.843849182128906,
264
+ "step": 160
265
+ },
266
+ {
267
+ "epoch": 0.74235807860262,
268
+ "grad_norm": 0.9669967772246254,
269
+ "learning_rate": 9.455375061024319e-07,
270
+ "logits/chosen": -0.9981080889701843,
271
+ "logits/rejected": -0.914253830909729,
272
+ "logps/chosen": -445.781982421875,
273
+ "logps/rejected": -1072.5477294921875,
274
+ "loss": 0.0161,
275
+ "rewards/accuracies": 0.987500011920929,
276
+ "rewards/chosen": -1.2199878692626953,
277
+ "rewards/margins": 6.577937126159668,
278
+ "rewards/rejected": -7.797924995422363,
279
+ "step": 170
280
+ },
281
+ {
282
+ "epoch": 0.7860262008733624,
283
+ "grad_norm": 0.09826043440610506,
284
+ "learning_rate": 6.661359299530626e-07,
285
+ "logits/chosen": -0.9915900230407715,
286
+ "logits/rejected": -0.8251992464065552,
287
+ "logps/chosen": -402.2227478027344,
288
+ "logps/rejected": -1161.5181884765625,
289
+ "loss": 0.0089,
290
+ "rewards/accuracies": 1.0,
291
+ "rewards/chosen": -1.1185193061828613,
292
+ "rewards/margins": 7.358689308166504,
293
+ "rewards/rejected": -8.477209091186523,
294
+ "step": 180
295
+ },
296
+ {
297
+ "epoch": 0.8296943231441049,
298
+ "grad_norm": 0.3084190328632994,
299
+ "learning_rate": 4.293030708802834e-07,
300
+ "logits/chosen": -1.0055569410324097,
301
+ "logits/rejected": -0.8623663187026978,
302
+ "logps/chosen": -418.1572265625,
303
+ "logps/rejected": -1129.2655029296875,
304
+ "loss": 0.0108,
305
+ "rewards/accuracies": 1.0,
306
+ "rewards/chosen": -1.1186723709106445,
307
+ "rewards/margins": 6.933996677398682,
308
+ "rewards/rejected": -8.0526704788208,
309
+ "step": 190
310
+ },
311
+ {
312
+ "epoch": 0.8733624454148472,
313
+ "grad_norm": 0.10794356987220223,
314
+ "learning_rate": 2.405364306547955e-07,
315
+ "logits/chosen": -0.8744913339614868,
316
+ "logits/rejected": -0.7203727960586548,
317
+ "logps/chosen": -407.0746765136719,
318
+ "logps/rejected": -1108.8079833984375,
319
+ "loss": 0.0118,
320
+ "rewards/accuracies": 1.0,
321
+ "rewards/chosen": -1.0695985555648804,
322
+ "rewards/margins": 7.07146692276001,
323
+ "rewards/rejected": -8.14106559753418,
324
+ "step": 200
325
+ },
326
+ {
327
+ "epoch": 0.9170305676855895,
328
+ "grad_norm": 1.7933618946506555,
329
+ "learning_rate": 1.0421777008019663e-07,
330
+ "logits/chosen": -1.047523856163025,
331
+ "logits/rejected": -0.8576782941818237,
332
+ "logps/chosen": -400.34527587890625,
333
+ "logps/rejected": -1216.9423828125,
334
+ "loss": 0.0123,
335
+ "rewards/accuracies": 1.0,
336
+ "rewards/chosen": -0.9318978190422058,
337
+ "rewards/margins": 7.647104740142822,
338
+ "rewards/rejected": -8.579002380371094,
339
+ "step": 210
340
+ },
341
+ {
342
+ "epoch": 0.9606986899563319,
343
+ "grad_norm": 1.0019715180278645,
344
+ "learning_rate": 2.351139701825267e-08,
345
+ "logits/chosen": -0.9511087536811829,
346
+ "logits/rejected": -0.8330629467964172,
347
+ "logps/chosen": -415.8382873535156,
348
+ "logps/rejected": -1095.948974609375,
349
+ "loss": 0.0086,
350
+ "rewards/accuracies": 1.0,
351
+ "rewards/chosen": -1.0705822706222534,
352
+ "rewards/margins": 7.189767360687256,
353
+ "rewards/rejected": -8.26034927368164,
354
+ "step": 220
355
+ },
356
+ {
357
+ "epoch": 1.0,
358
+ "step": 229,
359
+ "total_flos": 0.0,
360
+ "train_loss": 0.16486623261823405,
361
+ "train_runtime": 2148.0634,
362
+ "train_samples_per_second": 6.816,
363
+ "train_steps_per_second": 0.107
364
+ }
365
+ ],
366
+ "logging_steps": 10,
367
+ "max_steps": 229,
368
+ "num_input_tokens_seen": 0,
369
+ "num_train_epochs": 1,
370
+ "save_steps": 100,
371
+ "stateful_callbacks": {
372
+ "TrainerControl": {
373
+ "args": {
374
+ "should_epoch_stop": false,
375
+ "should_evaluate": false,
376
+ "should_log": false,
377
+ "should_save": true,
378
+ "should_training_stop": true
379
+ },
380
+ "attributes": {}
381
+ }
382
+ },
383
+ "total_flos": 0.0,
384
+ "train_batch_size": 2,
385
+ "trial_name": null,
386
+ "trial_params": null
387
+ }