Model save

README.md

@@ -0,0 +1,66 @@
+---
+library_name: transformers
+model_name: mistral-7b-sft-simpo-basic-5e-7-005-v141
+tags:
+- generated_from_trainer
+- trl
+- orpo
+licence: license
+---
+
+# Model Card for mistral-7b-sft-simpo-basic-5e-7-005-v141
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="silviasapora/mistral-7b-sft-simpo-basic-5e-7-005-v141", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/kk77jfrm) 
+
+
+This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).
+
+### Framework versions
+
+- TRL: 0.15.2
+- Transformers: 4.51.3
+- Pytorch: 2.4.0
+- Datasets: 3.0.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite ORPO as:
+
+```bibtex
+@article{hong2024orpo,
+    title        = {{ORPO: Monolithic Preference Optimization without Reference Model}},
+    author       = {Jiwoo Hong and Noah Lee and James Thorne},
+    year         = 2024,
+    eprint       = {arXiv:2403.07691}
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9879931389365352,
+    "total_flos": 0.0,
+    "train_loss": 9.269891473982069,
+    "train_runtime": 759.2455,
+    "train_samples": 4662,
+    "train_samples_per_second": 6.14,
+    "train_steps_per_second": 0.095
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9879931389365352,
+    "total_flos": 0.0,
+    "train_loss": 9.269891473982069,
+    "train_runtime": 759.2455,
+    "train_samples": 4662,
+    "train_samples_per_second": 6.14,
+    "train_steps_per_second": 0.095
+}

trainer_state.json

@@ -0,0 +1,267 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9879931389365352,
+  "eval_steps": 500,
+  "global_step": 72,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0686106346483705,
+      "grad_norm": 19.132543563842773,
+      "learning_rate": 2.5e-07,
+      "log_odds_chosen": 1.201877474784851,
+      "log_odds_ratio": -0.3934389352798462,
+      "logps/chosen": -0.41928571462631226,
+      "logps/rejected": -0.8378047943115234,
+      "loss": 11.1201,
+      "nll_loss": 0.6685064435005188,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.020964287221431732,
+      "rewards/margins": 0.02092595025897026,
+      "rewards/rejected": -0.04189024120569229,
+      "step": 5
+    },
+    {
+      "epoch": 0.137221269296741,
+      "grad_norm": 16.658292770385742,
+      "learning_rate": 4.996988640512931e-07,
+      "log_odds_chosen": 1.2810404300689697,
+      "log_odds_ratio": -0.38691967725753784,
+      "logps/chosen": -0.3941475749015808,
+      "logps/rejected": -0.8232101202011108,
+      "loss": 10.5726,
+      "nll_loss": 0.6344169974327087,
+      "rewards/accuracies": 0.840624988079071,
+      "rewards/chosen": -0.01970737986266613,
+      "rewards/margins": 0.021453125402331352,
+      "rewards/rejected": -0.04116050526499748,
+      "step": 10
+    },
+    {
+      "epoch": 0.2058319039451115,
+      "grad_norm": 18.118980407714844,
+      "learning_rate": 4.892350839330522e-07,
+      "log_odds_chosen": 1.4653966426849365,
+      "log_odds_ratio": -0.34464865922927856,
+      "logps/chosen": -0.35425886511802673,
+      "logps/rejected": -0.8524641990661621,
+      "loss": 9.5978,
+      "nll_loss": 0.5747923851013184,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.017712941393256187,
+      "rewards/margins": 0.02491026371717453,
+      "rewards/rejected": -0.042623210698366165,
+      "step": 15
+    },
+    {
+      "epoch": 0.274442538593482,
+      "grad_norm": 15.644585609436035,
+      "learning_rate": 4.64432152500068e-07,
+      "log_odds_chosen": 1.7301677465438843,
+      "log_odds_ratio": -0.30839115381240845,
+      "logps/chosen": -0.34137585759162903,
+      "logps/rejected": -0.8977292776107788,
+      "loss": 8.742,
+      "nll_loss": 0.5222873687744141,
+      "rewards/accuracies": 0.8843749761581421,
+      "rewards/chosen": -0.01706879213452339,
+      "rewards/margins": 0.027817675843834877,
+      "rewards/rejected": -0.04488646611571312,
+      "step": 20
+    },
+    {
+      "epoch": 0.34305317324185247,
+      "grad_norm": 18.07208824157715,
+      "learning_rate": 4.2677669529663686e-07,
+      "log_odds_chosen": 1.4824615716934204,
+      "log_odds_ratio": -0.32871100306510925,
+      "logps/chosen": -0.3440503478050232,
+      "logps/rejected": -0.8242480158805847,
+      "loss": 9.7095,
+      "nll_loss": 0.5817646384239197,
+      "rewards/accuracies": 0.8968750238418579,
+      "rewards/chosen": -0.01720252074301243,
+      "rewards/margins": 0.024009883403778076,
+      "rewards/rejected": -0.041212402284145355,
+      "step": 25
+    },
+    {
+      "epoch": 0.411663807890223,
+      "grad_norm": 16.565021514892578,
+      "learning_rate": 3.7852568604830535e-07,
+      "log_odds_chosen": 1.546881914138794,
+      "log_odds_ratio": -0.30869245529174805,
+      "logps/chosen": -0.3458595871925354,
+      "logps/rejected": -0.8716581463813782,
+      "loss": 9.1553,
+      "nll_loss": 0.5478529334068298,
+      "rewards/accuracies": 0.890625,
+      "rewards/chosen": -0.01729298010468483,
+      "rewards/margins": 0.0262899287045002,
+      "rewards/rejected": -0.04358290508389473,
+      "step": 30
+    },
+    {
+      "epoch": 0.48027444253859347,
+      "grad_norm": 16.609926223754883,
+      "learning_rate": 3.2257116931361555e-07,
+      "log_odds_chosen": 1.6251150369644165,
+      "log_odds_ratio": -0.29985013604164124,
+      "logps/chosen": -0.3172690272331238,
+      "logps/rejected": -0.848560631275177,
+      "loss": 8.8976,
+      "nll_loss": 0.5319867730140686,
+      "rewards/accuracies": 0.903124988079071,
+      "rewards/chosen": -0.01586345210671425,
+      "rewards/margins": 0.0265645794570446,
+      "rewards/rejected": -0.04242803156375885,
+      "step": 35
+    },
+    {
+      "epoch": 0.548885077186964,
+      "grad_norm": 16.254268646240234,
+      "learning_rate": 2.6226691858185454e-07,
+      "log_odds_chosen": 1.7786935567855835,
+      "log_odds_ratio": -0.28189554810523987,
+      "logps/chosen": -0.29200151562690735,
+      "logps/rejected": -0.8511508703231812,
+      "loss": 8.6234,
+      "nll_loss": 0.5152128338813782,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.014600073918700218,
+      "rewards/margins": 0.027957463636994362,
+      "rewards/rejected": -0.04255754128098488,
+      "step": 40
+    },
+    {
+      "epoch": 0.6174957118353345,
+      "grad_norm": 18.416759490966797,
+      "learning_rate": 2.0122741949596793e-07,
+      "log_odds_chosen": 1.6836531162261963,
+      "log_odds_ratio": -0.3088400065898895,
+      "logps/chosen": -0.31778112053871155,
+      "logps/rejected": -0.8519834280014038,
+      "loss": 8.6297,
+      "nll_loss": 0.5149655938148499,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -0.015889056026935577,
+      "rewards/margins": 0.026710111647844315,
+      "rewards/rejected": -0.04259916767477989,
+      "step": 45
+    },
+    {
+      "epoch": 0.6861063464837049,
+      "grad_norm": 17.294591903686523,
+      "learning_rate": 1.4311122664242953e-07,
+      "log_odds_chosen": 1.771561622619629,
+      "log_odds_ratio": -0.28731289505958557,
+      "logps/chosen": -0.31130000948905945,
+      "logps/rejected": -0.8764801025390625,
+      "loss": 8.844,
+      "nll_loss": 0.5289902687072754,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.015565002337098122,
+      "rewards/margins": 0.028259003534913063,
+      "rewards/rejected": -0.043824005872011185,
+      "step": 50
+    },
+    {
+      "epoch": 0.7547169811320755,
+      "grad_norm": 17.180084228515625,
+      "learning_rate": 9.140167895908865e-08,
+      "log_odds_chosen": 1.7292197942733765,
+      "log_odds_ratio": -0.28918027877807617,
+      "logps/chosen": -0.3234798014163971,
+      "logps/rejected": -0.8820427656173706,
+      "loss": 8.9077,
+      "nll_loss": 0.532711386680603,
+      "rewards/accuracies": 0.909375011920929,
+      "rewards/chosen": -0.016173990443348885,
+      "rewards/margins": 0.027928149327635765,
+      "rewards/rejected": -0.04410213977098465,
+      "step": 55
+    },
+    {
+      "epoch": 0.823327615780446,
+      "grad_norm": 16.852027893066406,
+      "learning_rate": 4.919811712983879e-08,
+      "log_odds_chosen": 1.6987159252166748,
+      "log_odds_ratio": -0.2939682900905609,
+      "logps/chosen": -0.3388054668903351,
+      "logps/rejected": -0.8977063298225403,
+      "loss": 9.023,
+      "nll_loss": 0.540162205696106,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.016940273344516754,
+      "rewards/margins": 0.02794504165649414,
+      "rewards/rejected": -0.044885315001010895,
+      "step": 60
+    },
+    {
+      "epoch": 0.8919382504288165,
+      "grad_norm": 18.404361724853516,
+      "learning_rate": 1.9030116872178314e-08,
+      "log_odds_chosen": 1.8277610540390015,
+      "log_odds_ratio": -0.27027684450149536,
+      "logps/chosen": -0.3031202554702759,
+      "logps/rejected": -0.896207332611084,
+      "loss": 8.722,
+      "nll_loss": 0.5218254327774048,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.015156012959778309,
+      "rewards/margins": 0.029654353857040405,
+      "rewards/rejected": -0.04481036588549614,
+      "step": 65
+    },
+    {
+      "epoch": 0.9605488850771869,
+      "grad_norm": 17.21025276184082,
+      "learning_rate": 2.7058725088047464e-09,
+      "log_odds_chosen": 1.7194201946258545,
+      "log_odds_ratio": -0.2862251400947571,
+      "logps/chosen": -0.32783252000808716,
+      "logps/rejected": -0.8619312047958374,
+      "loss": 9.1838,
+      "nll_loss": 0.5499452352523804,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.016391625627875328,
+      "rewards/margins": 0.02670494094491005,
+      "rewards/rejected": -0.04309656471014023,
+      "step": 70
+    },
+    {
+      "epoch": 0.9879931389365352,
+      "step": 72,
+      "total_flos": 0.0,
+      "train_loss": 9.269891473982069,
+      "train_runtime": 759.2455,
+      "train_samples_per_second": 6.14,
+      "train_steps_per_second": 0.095
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 72,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}