End of training

README.md

@@ -15,10 +15,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 # recreate_llama_68M_vanilla
 
-This model is a fine-tuned version of [JackFram/llama-68m](https://huggingface.co/JackFram/llama-68m) on an unknown dataset.
+This model is a fine-tuned version of [JackFram/llama-68m](https://huggingface.co/JackFram/llama-68m) on the anon8231489123/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.3603
-- Accuracy: 0.5811
+- Loss: 2.3558
+- Accuracy: 0.5820
 
 ## Model description
 

all_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 3.0,
-    "eval_accuracy": 0.45896433805569126,
-    "eval_loss": 4.308589935302734,
-    "eval_runtime": 2.8119,
-    "eval_samples": 10,
-    "eval_samples_per_second": 3.556,
-    "eval_steps_per_second": 0.356,
-    "perplexity": 74.33559700443098,
-    "total_flos": 1601895923712000.0,
-    "train_loss": 4.819753979879712,
-    "train_runtime": 153.0154,
-    "train_samples": 1000,
-    "train_samples_per_second": 19.606,
-    "train_steps_per_second": 1.647
+    "eval_accuracy": 0.5819606104373314,
+    "eval_loss": 2.3558192253112793,
+    "eval_runtime": 128.698,
+    "eval_samples": 1840,
+    "eval_samples_per_second": 14.297,
+    "eval_steps_per_second": 0.303,
+    "perplexity": 10.546765500786147,
+    "total_flos": 1.4536404559724544e+17,
+    "train_loss": 2.5941595100713495,
+    "train_runtime": 20556.3593,
+    "train_samples": 90745,
+    "train_samples_per_second": 13.243,
+    "train_steps_per_second": 0.552
 }

args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da0612c3c9d86d6249df50bee087bc8118d35203b4807ae34f25b74705525d44
+oid sha256:201594a79150cdbed9448595ad3b468d7c43b4a56fba8ab6f5d555f487e2bab9
 size 6036

eval_results.json

@@ -1,10 +1,10 @@
 {
     "epoch": 3.0,
-    "eval_accuracy": 0.45896433805569126,
-    "eval_loss": 4.308589935302734,
-    "eval_runtime": 2.8119,
-    "eval_samples": 10,
-    "eval_samples_per_second": 3.556,
-    "eval_steps_per_second": 0.356,
-    "perplexity": 74.33559700443098
+    "eval_accuracy": 0.5819606104373314,
+    "eval_loss": 2.3558192253112793,
+    "eval_runtime": 128.698,
+    "eval_samples": 1840,
+    "eval_samples_per_second": 14.297,
+    "eval_steps_per_second": 0.303,
+    "perplexity": 10.546765500786147
 }

events.out.tfevents.1717508814.isl-gpu33.2434801.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59238b0247d9a11cebfcaadb2151c6567d768aafe1085c694aab66aa014c757b
+size 411

log.txt

@@ -1018,3 +1018,44 @@ Training completed. Do not forget to share your model on huggingface.co/models =
 
 
 
+***** train metrics *****
+  epoch                    =         3.0
+  total_flos               = 135380817GF
+  train_loss               =      2.5942
+  train_runtime            =  5:42:36.35
+  train_samples            =       90745
+  train_samples_per_second =      13.243
+  train_steps_per_second   =       0.552
+06/04/2024 06:44:45 - INFO - __main__ - *** Evaluate ***
+[INFO|trainer.py:3662] 2024-06-04 06:44:45,746 >> ***** Running Evaluation *****
+[INFO|trainer.py:3664] 2024-06-04 06:44:45,746 >>   Num examples = 1840
+[INFO|trainer.py:3667] 2024-06-04 06:44:45,746 >>   Batch size = 48
+/home/dshteyma/miniconda3/lib/python3.9/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.
+  warnings.warn('Was asked to gather along dimension 0, but all '
+
  0%|          | 0/39 [00:00<?, ?it/s]
  5%|▌         | 2/39 [00:02<00:54,  1.48s/it]
  8%|▊         | 3/39 [00:05<01:15,  2.10s/it]
 10%|█         | 4/39 [00:08<01:24,  2.43s/it]
 13%|█▎        | 5/39 [00:11<01:28,  2.62s/it]
 15%|█▌        | 6/39 [00:14<01:30,  2.73s/it]
 18%|█▊        | 7/39 [00:17<01:29,  2.81s/it]
 21%|██        | 8/39 [00:20<01:28,  2.86s/it]
 23%|██▎       | 9/39 [00:23<01:26,  2.90s/it]
 26%|██▌       | 10/39 [00:26<01:24,  2.92s/it]
 28%|██▊       | 11/39 [00:29<01:22,  2.93s/it]
 31%|███       | 12/39 [00:32<01:19,  2.94s/it]
 33%|███▎      | 13/39 [00:35<01:16,  2.95s/it]
 36%|███▌      | 14/39 [00:38<01:13,  2.96s/it]
 38%|███▊      | 15/39 [00:41<01:11,  2.96s/it]
 41%|████      | 16/39 [00:44<01:08,  2.96s/it]
 44%|████▎     | 17/39 [00:47<01:05,  2.97s/it]
 46%|████▌     | 18/39 [00:50<01:02,  2.97s/it]
 49%|████▊     | 19/39 [00:53<00:59,  2.97s/it]
 51%|█████▏    | 20/39 [00:56<00:56,  2.97s/it]
 54%|█████▍    | 21/39 [00:59<00:53,  2.97s/it]
 56%|█████▋    | 22/39 [01:02<00:50,  2.97s/it]
 59%|█████▉    | 23/39 [01:05<00:47,  2.97s/it]
 62%|██████▏   | 24/39 [01:08<00:44,  2.97s/it]
 64%|██████▍   | 25/39 [01:11<00:41,  2.97s/it]
 67%|██████▋   | 26/39 [01:14<00:38,  2.97s/it]
 69%|██████▉   | 27/39 [01:17<00:35,  2.97s/it]
 72%|███████▏  | 28/39 [01:20<00:32,  2.97s/it]
 74%|███████▍  | 29/39 [01:23<00:29,  2.97s/it]
 77%|███████▋  | 30/39 [01:26<00:26,  2.97s/it]
 79%|███████▉  | 31/39 [01:29<00:23,  2.97s/it]
 82%|████████▏ | 32/39 [01:32<00:20,  2.97s/it]
 85%|████████▍ | 33/39 [01:35<00:17,  2.97s/it]
 87%|████████▋ | 34/39 [01:38<00:14,  2.97s/it]
 90%|████████▉ | 35/39 [01:41<00:11,  2.97s/it]
 92%|█████████▏| 36/39 [01:43<00:08,  2.97s/it]
 95%|█████████▍| 37/39 [01:46<00:05,  2.97s/it]
 97%|█████████▋| 38/39 [01:49<00:02,  2.94s/it]
+[INFO|trainer.py:3353] 2024-06-04 06:46:54,461 >> Saving model checkpoint to ./training_outputs_job_116987_1_04-06_01-01
+[INFO|configuration_utils.py:471] 2024-06-04 06:46:54,473 >> Configuration saved in ./training_outputs_job_116987_1_04-06_01-01/config.json
+[INFO|configuration_utils.py:705] 2024-06-04 06:46:54,478 >> Configuration saved in ./training_outputs_job_116987_1_04-06_01-01/generation_config.json
+[INFO|modeling_utils.py:2592] 2024-06-04 06:46:55,425 >> Model weights saved in ./training_outputs_job_116987_1_04-06_01-01/model.safetensors
+[INFO|tokenization_utils_base.py:2503] 2024-06-04 06:46:55,436 >> tokenizer config file saved in ./training_outputs_job_116987_1_04-06_01-01/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2512] 2024-06-04 06:46:55,440 >> Special tokens file saved in ./training_outputs_job_116987_1_04-06_01-01/special_tokens_map.json
+[INFO|modelcard.py:450] 2024-06-04 06:46:55,614 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}, 'metrics': [{'name': 'Accuracy', 'type': 'accuracy', 'value': 0.5819606104373314}]}
+***** eval metrics *****
+  epoch                   =        3.0
+  eval_accuracy           =      0.582
+  eval_loss               =     2.3558
+  eval_runtime            = 0:02:08.69
+  eval_samples            =       1840
+  eval_samples_per_second =     14.297
+  eval_steps_per_second   =      0.303
+  perplexity              =    10.5468
+
+
+
+
+
+
+
+
+

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 3.0,
-    "total_flos": 1601895923712000.0,
-    "train_loss": 4.819753979879712,
-    "train_runtime": 153.0154,
-    "train_samples": 1000,
-    "train_samples_per_second": 19.606,
-    "train_steps_per_second": 1.647
+    "total_flos": 1.4536404559724544e+17,
+    "train_loss": 2.5941595100713495,
+    "train_runtime": 20556.3593,
+    "train_samples": 90745,
+    "train_samples_per_second": 13.243,
+    "train_steps_per_second": 0.552
 }

trainer_state.json

@@ -3,40 +3,293 @@
   "best_model_checkpoint": null,
   "epoch": 3.0,
   "eval_steps": 1000,
-  "global_step": 252,
+  "global_step": 11346,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.13220518244315177,
+      "grad_norm": 0.8546377420425415,
+      "learning_rate": 8.816009873931059e-05,
+      "loss": 5.1118,
+      "step": 500
+    },
+    {
+      "epoch": 0.26441036488630354,
+      "grad_norm": 0.8593683838844299,
+      "learning_rate": 9.59831475011252e-05,
+      "loss": 3.406,
+      "step": 1000
+    },
+    {
+      "epoch": 0.26441036488630354,
+      "eval_accuracy": 0.5035308829464115,
+      "eval_loss": 3.23445987701416,
+      "eval_runtime": 128.969,
+      "eval_samples_per_second": 14.267,
+      "eval_steps_per_second": 0.302,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3966155473294553,
+      "grad_norm": 0.9617242217063904,
+      "learning_rate": 9.134314230431938e-05,
+      "loss": 3.0005,
+      "step": 1500
+    },
+    {
+      "epoch": 0.5288207297726071,
+      "grad_norm": 0.8956136107444763,
+      "learning_rate": 8.670313710751356e-05,
+      "loss": 2.8119,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5288207297726071,
+      "eval_accuracy": 0.5364783564495231,
+      "eval_loss": 2.821624517440796,
+      "eval_runtime": 128.8547,
+      "eval_samples_per_second": 14.28,
+      "eval_steps_per_second": 0.303,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6610259122157589,
+      "grad_norm": 1.2592207193374634,
+      "learning_rate": 8.206313191070773e-05,
+      "loss": 2.6861,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7932310946589106,
+      "grad_norm": 1.3978203535079956,
+      "learning_rate": 7.742312671390191e-05,
+      "loss": 2.6076,
+      "step": 3000
+    },
+    {
+      "epoch": 0.7932310946589106,
+      "eval_accuracy": 0.5501396529385527,
+      "eval_loss": 2.6552908420562744,
+      "eval_runtime": 129.1173,
+      "eval_samples_per_second": 14.251,
+      "eval_steps_per_second": 0.302,
+      "step": 3000
+    },
+    {
+      "epoch": 0.9254362771020624,
+      "grad_norm": 1.8927521705627441,
+      "learning_rate": 7.278312151709609e-05,
+      "loss": 2.5643,
+      "step": 3500
+    },
+    {
+      "epoch": 1.0576414595452142,
+      "grad_norm": 4.235791206359863,
+      "learning_rate": 6.814311632029027e-05,
+      "loss": 2.4729,
+      "step": 4000
+    },
+    {
+      "epoch": 1.0576414595452142,
+      "eval_accuracy": 0.5581473949151462,
+      "eval_loss": 2.5761468410491943,
+      "eval_runtime": 129.3204,
+      "eval_samples_per_second": 14.228,
+      "eval_steps_per_second": 0.302,
+      "step": 4000
+    },
+    {
+      "epoch": 1.189846641988366,
+      "grad_norm": 2.753159523010254,
+      "learning_rate": 6.350311112348446e-05,
+      "loss": 2.4531,
+      "step": 4500
+    },
+    {
+      "epoch": 1.3220518244315178,
+      "grad_norm": 3.542167901992798,
+      "learning_rate": 5.886310592667864e-05,
+      "loss": 2.4323,
+      "step": 5000
+    },
+    {
+      "epoch": 1.3220518244315178,
+      "eval_accuracy": 0.5616867738578195,
+      "eval_loss": 2.536275863647461,
+      "eval_runtime": 129.4842,
+      "eval_samples_per_second": 14.21,
+      "eval_steps_per_second": 0.301,
+      "step": 5000
+    },
+    {
+      "epoch": 1.4542570068746694,
+      "grad_norm": 3.678074598312378,
+      "learning_rate": 5.422310072987282e-05,
+      "loss": 2.3981,
+      "step": 5500
+    },
+    {
+      "epoch": 1.5864621893178212,
+      "grad_norm": 5.024658679962158,
+      "learning_rate": 4.9583095533066995e-05,
+      "loss": 2.3824,
+      "step": 6000
+    },
+    {
+      "epoch": 1.5864621893178212,
+      "eval_accuracy": 0.5660391134427901,
+      "eval_loss": 2.491274118423462,
+      "eval_runtime": 129.1418,
+      "eval_samples_per_second": 14.248,
+      "eval_steps_per_second": 0.302,
+      "step": 6000
+    },
+    {
+      "epoch": 1.718667371760973,
+      "grad_norm": 5.106564521789551,
+      "learning_rate": 4.4943090336261176e-05,
+      "loss": 2.3763,
+      "step": 6500
+    },
+    {
+      "epoch": 1.8508725542041247,
+      "grad_norm": 11.649778366088867,
+      "learning_rate": 4.030308513945535e-05,
+      "loss": 2.3719,
+      "step": 7000
+    },
+    {
+      "epoch": 1.8508725542041247,
+      "eval_accuracy": 0.5685828678235382,
+      "eval_loss": 2.466362237930298,
+      "eval_runtime": 128.8316,
+      "eval_samples_per_second": 14.282,
+      "eval_steps_per_second": 0.303,
+      "step": 7000
+    },
+    {
+      "epoch": 1.9830777366472767,
+      "grad_norm": 5.842829704284668,
+      "learning_rate": 3.566307994264953e-05,
+      "loss": 2.3411,
+      "step": 7500
+    },
+    {
+      "epoch": 2.1152829190904283,
+      "grad_norm": 14.795243263244629,
+      "learning_rate": 3.1023074745843715e-05,
+      "loss": 2.3021,
+      "step": 8000
+    },
+    {
+      "epoch": 2.1152829190904283,
+      "eval_accuracy": 0.5715702725090801,
+      "eval_loss": 2.4403789043426514,
+      "eval_runtime": 128.3806,
+      "eval_samples_per_second": 14.332,
+      "eval_steps_per_second": 0.304,
+      "step": 8000
+    },
+    {
+      "epoch": 2.24748810153358,
+      "grad_norm": 6.084632873535156,
+      "learning_rate": 2.6383069549037897e-05,
+      "loss": 2.2897,
+      "step": 8500
+    },
+    {
+      "epoch": 2.379693283976732,
+      "grad_norm": 17.67453956604004,
+      "learning_rate": 2.1743064352232075e-05,
+      "loss": 2.2848,
+      "step": 9000
+    },
+    {
+      "epoch": 2.379693283976732,
+      "eval_accuracy": 0.5754845372868036,
+      "eval_loss": 2.407994508743286,
+      "eval_runtime": 128.557,
+      "eval_samples_per_second": 14.313,
+      "eval_steps_per_second": 0.303,
+      "step": 9000
+    },
+    {
+      "epoch": 2.5118984664198836,
+      "grad_norm": 6.01920223236084,
+      "learning_rate": 1.7103059155426253e-05,
+      "loss": 2.2611,
+      "step": 9500
+    },
+    {
+      "epoch": 2.6441036488630356,
+      "grad_norm": 7.932415008544922,
+      "learning_rate": 1.2463053958620433e-05,
+      "loss": 2.2653,
+      "step": 10000
+    },
+    {
+      "epoch": 2.6441036488630356,
+      "eval_accuracy": 0.5784841549669718,
+      "eval_loss": 2.3834304809570312,
+      "eval_runtime": 128.7742,
+      "eval_samples_per_second": 14.289,
+      "eval_steps_per_second": 0.303,
+      "step": 10000
+    },
+    {
+      "epoch": 2.776308831306187,
+      "grad_norm": 9.934358596801758,
+      "learning_rate": 7.823048761814613e-06,
+      "loss": 2.2542,
+      "step": 10500
+    },
+    {
+      "epoch": 2.908514013749339,
+      "grad_norm": 8.533864974975586,
+      "learning_rate": 3.183043565008793e-06,
+      "loss": 2.2447,
+      "step": 11000
+    },
+    {
+      "epoch": 2.908514013749339,
+      "eval_accuracy": 0.5811017183152439,
+      "eval_loss": 2.3603451251983643,
+      "eval_runtime": 129.0604,
+      "eval_samples_per_second": 14.257,
+      "eval_steps_per_second": 0.302,
+      "step": 11000
+    },
     {
       "epoch": 3.0,
-      "step": 252,
-      "total_flos": 1601895923712000.0,
-      "train_loss": 4.819753979879712,
-      "train_runtime": 153.0154,
-      "train_samples_per_second": 19.606,
-      "train_steps_per_second": 1.647
+      "step": 11346,
+      "total_flos": 1.4536404559724544e+17,
+      "train_loss": 2.5941595100713495,
+      "train_runtime": 20556.3593,
+      "train_samples_per_second": 13.243,
+      "train_steps_per_second": 0.552
     }
   ],
   "logging_steps": 500,
-  "max_steps": 252,
+  "max_steps": 11346,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
-  "save_steps": 10000,
+  "save_steps": 1000,
   "stateful_callbacks": {
     "TrainerControl": {
       "args": {
         "should_epoch_stop": false,
         "should_evaluate": false,
         "should_log": false,
-        "should_save": false,
+        "should_save": true,
         "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 1601895923712000.0,
-  "train_batch_size": 12,
+  "total_flos": 1.4536404559724544e+17,
+  "train_batch_size": 24,
   "trial_name": null,
   "trial_params": null
 }