Model save

README.md

@@ -1,19 +1,16 @@
 ---
-datasets:
-- argilla/dpo-mix-7k
 library_name: transformers
-model_name: /home/silvias/alignment-handbook/data/mistral-7b-sft-basic-5e-5-000-v132-full
+model_name: mistral-7b-sft-simpo-basic-5e-7-005-v142
 tags:
 - generated_from_trainer
-- alignment-handbook
 - trl
 - orpo
 licence: license
 ---
 
-# Model Card for /home/silvias/alignment-handbook/data/mistral-7b-sft-basic-5e-5-000-v132-full
+# Model Card for mistral-7b-sft-simpo-basic-5e-7-005-v142
 
-This model is a fine-tuned version of [None](https://huggingface.co/None) on the [['argilla/dpo-mix-7k']](https://huggingface.co/datasets/['argilla/dpo-mix-7k']) dataset.
+This model is a fine-tuned version of [None](https://huggingface.co/None).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -29,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/w11kzsbc) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/acoqxadt) 
 
 
 This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).
@@ -37,10 +34,10 @@ This model was trained with ORPO, a method introduced in [ORPO: Monolithic Prefe
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.49.0
-- Pytorch: 2.5.1
-- Datasets: 3.2.0
-- Tokenizers: 0.21.1
+- Transformers: 4.51.3
+- Pytorch: 2.4.0
+- Datasets: 3.0.0
+- Tokenizers: 0.21.0
 
 ## Citations
 

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9884169884169884,
+    "epoch": 0.9879931389365352,
     "total_flos": 0.0,
-    "train_loss": 21.922766897413467,
-    "train_runtime": 1196.0193,
+    "train_loss": 9.270023425420126,
+    "train_runtime": 759.6158,
     "train_samples": 4662,
-    "train_samples_per_second": 3.898,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 6.137,
+    "train_steps_per_second": 0.095
 }

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 0.9884169884169884,
+    "epoch": 0.9879931389365352,
     "total_flos": 0.0,
-    "train_loss": 21.922766897413467,
-    "train_runtime": 1196.0193,
+    "train_loss": 9.270023425420126,
+    "train_runtime": 759.6158,
     "train_samples": 4662,
-    "train_samples_per_second": 3.898,
-    "train_steps_per_second": 0.06
+    "train_samples_per_second": 6.137,
+    "train_steps_per_second": 0.095
 }

trainer_state.json

@@ -1,7 +1,8 @@
 {
+  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9884169884169884,
+  "epoch": 0.9879931389365352,
   "eval_steps": 500,
   "global_step": 72,
   "is_hyper_param_search": false,
@@ -9,237 +10,237 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.06864006864006864,
-      "grad_norm": 2.234375,
-      "learning_rate": 3.1249999999999997e-07,
-      "log_odds_chosen": 0.4900525212287903,
-      "log_odds_ratio": -0.5972136855125427,
-      "logps/chosen": -0.5807362794876099,
-      "logps/rejected": -0.7729060053825378,
-      "loss": 22.0294,
-      "nll_loss": 0.8785218000411987,
-      "rewards/accuracies": 0.653124988079071,
-      "rewards/chosen": -0.029036814346909523,
-      "rewards/margins": 0.009608486667275429,
-      "rewards/rejected": -0.03864530101418495,
+      "epoch": 0.0686106346483705,
+      "grad_norm": 19.247772216796875,
+      "learning_rate": 2.5e-07,
+      "log_odds_chosen": 1.202063798904419,
+      "log_odds_ratio": -0.3932521939277649,
+      "logps/chosen": -0.4192759096622467,
+      "logps/rejected": -0.8378359079360962,
+      "loss": 11.1199,
+      "nll_loss": 0.6684929728507996,
+      "rewards/accuracies": 0.828125,
+      "rewards/chosen": -0.020963797345757484,
+      "rewards/margins": 0.020928001031279564,
+      "rewards/rejected": -0.04189179837703705,
       "step": 5
     },
     {
-      "epoch": 0.13728013728013727,
-      "grad_norm": 2.0625,
-      "learning_rate": 4.987961816680492e-07,
-      "log_odds_chosen": 0.6504672169685364,
-      "log_odds_ratio": -0.5491082668304443,
-      "logps/chosen": -0.5200858116149902,
-      "logps/rejected": -0.7607679963111877,
-      "loss": 21.9908,
-      "nll_loss": 0.8031341433525085,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": -0.02600429579615593,
-      "rewards/margins": 0.012034102343022823,
-      "rewards/rejected": -0.03803839534521103,
+      "epoch": 0.137221269296741,
+      "grad_norm": 16.6769962310791,
+      "learning_rate": 4.996988640512931e-07,
+      "log_odds_chosen": 1.2801547050476074,
+      "log_odds_ratio": -0.3870637118816376,
+      "logps/chosen": -0.39408302307128906,
+      "logps/rejected": -0.8230869174003601,
+      "loss": 10.5716,
+      "nll_loss": 0.6343531608581543,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.019704151898622513,
+      "rewards/margins": 0.021450195461511612,
+      "rewards/rejected": -0.041154347360134125,
       "step": 10
     },
     {
-      "epoch": 0.2059202059202059,
-      "grad_norm": 2.234375,
-      "learning_rate": 4.853860162957551e-07,
-      "log_odds_chosen": 0.8738743662834167,
-      "log_odds_ratio": -0.4701429009437561,
-      "logps/chosen": -0.45837029814720154,
-      "logps/rejected": -0.7923842668533325,
-      "loss": 21.9168,
-      "nll_loss": 0.7228935956954956,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.022918514907360077,
-      "rewards/margins": 0.01670069992542267,
-      "rewards/rejected": -0.039619214832782745,
+      "epoch": 0.2058319039451115,
+      "grad_norm": 18.085586547851562,
+      "learning_rate": 4.892350839330522e-07,
+      "log_odds_chosen": 1.465559959411621,
+      "log_odds_ratio": -0.3448963165283203,
+      "logps/chosen": -0.3543047606945038,
+      "logps/rejected": -0.8525373339653015,
+      "loss": 9.6005,
+      "nll_loss": 0.5749582052230835,
+      "rewards/accuracies": 0.878125011920929,
+      "rewards/chosen": -0.01771523989737034,
+      "rewards/margins": 0.024911630898714066,
+      "rewards/rejected": -0.04262687265872955,
       "step": 15
     },
     {
-      "epoch": 0.27456027456027454,
-      "grad_norm": 2.625,
-      "learning_rate": 4.578674030756363e-07,
-      "log_odds_chosen": 0.7947912812232971,
-      "log_odds_ratio": -0.5199654698371887,
-      "logps/chosen": -0.5325753688812256,
-      "logps/rejected": -0.8283828496932983,
-      "loss": 21.9475,
-      "nll_loss": 0.7589122653007507,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -0.02662876807153225,
-      "rewards/margins": 0.014790371060371399,
-      "rewards/rejected": -0.0414191372692585,
+      "epoch": 0.274442538593482,
+      "grad_norm": 15.663309097290039,
+      "learning_rate": 4.64432152500068e-07,
+      "log_odds_chosen": 1.7299188375473022,
+      "log_odds_ratio": -0.30832645297050476,
+      "logps/chosen": -0.3413788974285126,
+      "logps/rejected": -0.897726833820343,
+      "loss": 8.7406,
+      "nll_loss": 0.5222011804580688,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.017068946734070778,
+      "rewards/margins": 0.027817392721772194,
+      "rewards/rejected": -0.04488633945584297,
       "step": 20
     },
     {
-      "epoch": 0.3432003432003432,
-      "grad_norm": 2.359375,
-      "learning_rate": 4.178897387117546e-07,
-      "log_odds_chosen": 0.7632033228874207,
-      "log_odds_ratio": -0.48927417397499084,
-      "logps/chosen": -0.477088987827301,
-      "logps/rejected": -0.761306881904602,
-      "loss": 21.9558,
-      "nll_loss": 0.7712680697441101,
-      "rewards/accuracies": 0.7906249761581421,
-      "rewards/chosen": -0.023854447528719902,
-      "rewards/margins": 0.014210891909897327,
-      "rewards/rejected": -0.038065336644649506,
+      "epoch": 0.34305317324185247,
+      "grad_norm": 18.158308029174805,
+      "learning_rate": 4.2677669529663686e-07,
+      "log_odds_chosen": 1.4837188720703125,
+      "log_odds_ratio": -0.3287079334259033,
+      "logps/chosen": -0.3441501259803772,
+      "logps/rejected": -0.8242276906967163,
+      "loss": 9.7102,
+      "nll_loss": 0.5818053483963013,
+      "rewards/accuracies": 0.8968750238418579,
+      "rewards/chosen": -0.01720750704407692,
+      "rewards/margins": 0.024003881961107254,
+      "rewards/rejected": -0.041211389005184174,
       "step": 25
     },
     {
-      "epoch": 0.4118404118404118,
-      "grad_norm": 2.75,
-      "learning_rate": 3.6784918420649944e-07,
-      "log_odds_chosen": 0.905913233757019,
-      "log_odds_ratio": -0.4420197904109955,
-      "logps/chosen": -0.4566729664802551,
-      "logps/rejected": -0.8100153207778931,
-      "loss": 21.9012,
-      "nll_loss": 0.6997581720352173,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.022833649069070816,
-      "rewards/margins": 0.017667118459939957,
-      "rewards/rejected": -0.04050076752901077,
+      "epoch": 0.411663807890223,
+      "grad_norm": 16.503480911254883,
+      "learning_rate": 3.7852568604830535e-07,
+      "log_odds_chosen": 1.5456057786941528,
+      "log_odds_ratio": -0.3088548183441162,
+      "logps/chosen": -0.3460735082626343,
+      "logps/rejected": -0.8717023134231567,
+      "loss": 9.158,
+      "nll_loss": 0.5480188727378845,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.017303675413131714,
+      "rewards/margins": 0.026281436905264854,
+      "rewards/rejected": -0.04358511418104172,
       "step": 30
     },
     {
-      "epoch": 0.4804804804804805,
-      "grad_norm": 2.65625,
-      "learning_rate": 3.10745044975816e-07,
-      "log_odds_chosen": 0.9323528409004211,
-      "log_odds_ratio": -0.44655704498291016,
-      "logps/chosen": -0.44178277254104614,
-      "logps/rejected": -0.7855373024940491,
-      "loss": 21.9085,
-      "nll_loss": 0.7086659669876099,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.022089136764407158,
-      "rewards/margins": 0.017187729477882385,
-      "rewards/rejected": -0.039276864379644394,
+      "epoch": 0.48027444253859347,
+      "grad_norm": 16.41570472717285,
+      "learning_rate": 3.2257116931361555e-07,
+      "log_odds_chosen": 1.6255455017089844,
+      "log_odds_ratio": -0.2997768521308899,
+      "logps/chosen": -0.3172294795513153,
+      "logps/rejected": -0.8485333323478699,
+      "loss": 8.8971,
+      "nll_loss": 0.531952977180481,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": -0.015861475840210915,
+      "rewards/margins": 0.026565194129943848,
+      "rewards/rejected": -0.04242666810750961,
       "step": 35
     },
     {
-      "epoch": 0.5491205491205491,
-      "grad_norm": 2.875,
-      "learning_rate": 2.5e-07,
-      "log_odds_chosen": 1.0263824462890625,
-      "log_odds_ratio": -0.4294084906578064,
-      "logps/chosen": -0.4194498062133789,
-      "logps/rejected": -0.791568398475647,
-      "loss": 21.8864,
-      "nll_loss": 0.701680064201355,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.020972490310668945,
-      "rewards/margins": 0.018605932593345642,
-      "rewards/rejected": -0.03957842290401459,
+      "epoch": 0.548885077186964,
+      "grad_norm": 16.295194625854492,
+      "learning_rate": 2.6226691858185454e-07,
+      "log_odds_chosen": 1.7775678634643555,
+      "log_odds_ratio": -0.2819564938545227,
+      "logps/chosen": -0.29198122024536133,
+      "logps/rejected": -0.8509271740913391,
+      "loss": 8.6232,
+      "nll_loss": 0.5151973962783813,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.014599060639739037,
+      "rewards/margins": 0.027947301045060158,
+      "rewards/rejected": -0.042546361684799194,
       "step": 40
     },
     {
-      "epoch": 0.6177606177606177,
-      "grad_norm": 2.640625,
-      "learning_rate": 1.8925495502418404e-07,
-      "log_odds_chosen": 0.9927960634231567,
-      "log_odds_ratio": -0.45052170753479004,
-      "logps/chosen": -0.44455814361572266,
-      "logps/rejected": -0.800442099571228,
-      "loss": 21.8996,
-      "nll_loss": 0.7001025080680847,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -0.022227909415960312,
-      "rewards/margins": 0.01779419369995594,
-      "rewards/rejected": -0.0400221012532711,
+      "epoch": 0.6174957118353345,
+      "grad_norm": 18.461917877197266,
+      "learning_rate": 2.0122741949596793e-07,
+      "log_odds_chosen": 1.683835744857788,
+      "log_odds_ratio": -0.30903160572052,
+      "logps/chosen": -0.31781280040740967,
+      "logps/rejected": -0.8516524434089661,
+      "loss": 8.6307,
+      "nll_loss": 0.5150163769721985,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.015890639275312424,
+      "rewards/margins": 0.02669198252260685,
+      "rewards/rejected": -0.04258262366056442,
       "step": 45
     },
     {
-      "epoch": 0.6864006864006864,
-      "grad_norm": 2.703125,
-      "learning_rate": 1.3215081579350056e-07,
-      "log_odds_chosen": 1.010258674621582,
-      "log_odds_ratio": -0.43662938475608826,
-      "logps/chosen": -0.45176443457603455,
-      "logps/rejected": -0.8273895978927612,
-      "loss": 21.8839,
-      "nll_loss": 0.7259224653244019,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.022588221356272697,
-      "rewards/margins": 0.018781261518597603,
-      "rewards/rejected": -0.04136947914958,
+      "epoch": 0.6861063464837049,
+      "grad_norm": 17.289121627807617,
+      "learning_rate": 1.4311122664242953e-07,
+      "log_odds_chosen": 1.7723395824432373,
+      "log_odds_ratio": -0.28731250762939453,
+      "logps/chosen": -0.31122511625289917,
+      "logps/rejected": -0.876534640789032,
+      "loss": 8.8422,
+      "nll_loss": 0.5288792848587036,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.015561257489025593,
+      "rewards/margins": 0.02826547622680664,
+      "rewards/rejected": -0.04382672905921936,
       "step": 50
     },
     {
-      "epoch": 0.7550407550407551,
-      "grad_norm": 2.546875,
-      "learning_rate": 8.211026128824538e-08,
-      "log_odds_chosen": 0.9382259249687195,
-      "log_odds_ratio": -0.4488070011138916,
-      "logps/chosen": -0.4725814759731293,
-      "logps/rejected": -0.8342872858047485,
-      "loss": 21.8959,
-      "nll_loss": 0.7326583862304688,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": -0.023629074916243553,
-      "rewards/margins": 0.018085282295942307,
-      "rewards/rejected": -0.04171435907483101,
+      "epoch": 0.7547169811320755,
+      "grad_norm": 17.134496688842773,
+      "learning_rate": 9.140167895908865e-08,
+      "log_odds_chosen": 1.7291252613067627,
+      "log_odds_ratio": -0.28900426626205444,
+      "logps/chosen": -0.3234766721725464,
+      "logps/rejected": -0.8822552561759949,
+      "loss": 8.9071,
+      "nll_loss": 0.5326792001724243,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": -0.0161738358438015,
+      "rewards/margins": 0.027938928455114365,
+      "rewards/rejected": -0.04411276429891586,
       "step": 55
     },
     {
-      "epoch": 0.8236808236808236,
-      "grad_norm": 2.65625,
-      "learning_rate": 4.213259692436366e-08,
-      "log_odds_chosen": 0.9388996958732605,
-      "log_odds_ratio": -0.4496310353279114,
-      "logps/chosen": -0.48969897627830505,
-      "logps/rejected": -0.8507472276687622,
-      "loss": 21.8952,
-      "nll_loss": 0.7445230484008789,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.024484951049089432,
-      "rewards/margins": 0.018052412196993828,
-      "rewards/rejected": -0.04253736510872841,
+      "epoch": 0.823327615780446,
+      "grad_norm": 16.877124786376953,
+      "learning_rate": 4.919811712983879e-08,
+      "log_odds_chosen": 1.698340654373169,
+      "log_odds_ratio": -0.29392507672309875,
+      "logps/chosen": -0.3388321101665497,
+      "logps/rejected": -0.8975754976272583,
+      "loss": 9.0244,
+      "nll_loss": 0.5402450561523438,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.016941606998443604,
+      "rewards/margins": 0.02793716825544834,
+      "rewards/rejected": -0.044878773391246796,
       "step": 60
     },
     {
-      "epoch": 0.8923208923208923,
-      "grad_norm": 2.8125,
-      "learning_rate": 1.4613983704244825e-08,
-      "log_odds_chosen": 0.9500824809074402,
-      "log_odds_ratio": -0.4538024067878723,
-      "logps/chosen": -0.47038406133651733,
-      "logps/rejected": -0.8408064842224121,
-      "loss": 21.8882,
-      "nll_loss": 0.7523521184921265,
-      "rewards/accuracies": 0.796875,
-      "rewards/chosen": -0.023519206792116165,
-      "rewards/margins": 0.01852111890912056,
-      "rewards/rejected": -0.042040325701236725,
+      "epoch": 0.8919382504288165,
+      "grad_norm": 18.37203598022461,
+      "learning_rate": 1.9030116872178314e-08,
+      "log_odds_chosen": 1.82688307762146,
+      "log_odds_ratio": -0.2703757882118225,
+      "logps/chosen": -0.30324336886405945,
+      "logps/rejected": -0.8961852788925171,
+      "loss": 8.723,
+      "nll_loss": 0.5218873023986816,
+      "rewards/accuracies": 0.9156249761581421,
+      "rewards/chosen": -0.015162169933319092,
+      "rewards/margins": 0.029647093266248703,
+      "rewards/rejected": -0.044809263199567795,
       "step": 65
     },
     {
-      "epoch": 0.960960960960961,
-      "grad_norm": 2.78125,
-      "learning_rate": 1.2038183319507956e-09,
-      "log_odds_chosen": 0.9230395555496216,
-      "log_odds_ratio": -0.45713886618614197,
-      "logps/chosen": -0.4812949299812317,
-      "logps/rejected": -0.8133236169815063,
-      "loss": 21.918,
-      "nll_loss": 0.7658084034919739,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": -0.024064745754003525,
-      "rewards/margins": 0.016601432114839554,
-      "rewards/rejected": -0.04066618159413338,
+      "epoch": 0.9605488850771869,
+      "grad_norm": 17.195167541503906,
+      "learning_rate": 2.7058725088047464e-09,
+      "log_odds_chosen": 1.7199735641479492,
+      "log_odds_ratio": -0.28608742356300354,
+      "logps/chosen": -0.32787787914276123,
+      "logps/rejected": -0.8620197176933289,
+      "loss": 9.1848,
+      "nll_loss": 0.5500085949897766,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.01639389432966709,
+      "rewards/margins": 0.02670709416270256,
+      "rewards/rejected": -0.0431009940803051,
       "step": 70
     },
     {
-      "epoch": 0.9884169884169884,
+      "epoch": 0.9879931389365352,
       "step": 72,
       "total_flos": 0.0,
-      "train_loss": 21.922766897413467,
-      "train_runtime": 1196.0193,
-      "train_samples_per_second": 3.898,
-      "train_steps_per_second": 0.06
+      "train_loss": 9.270023425420126,
+      "train_runtime": 759.6158,
+      "train_samples_per_second": 6.137,
+      "train_steps_per_second": 0.095
     }
   ],
   "logging_steps": 5,