Use my fine-tuned model from huggingface

.gitignore

@@ -2,4 +2,5 @@ __pycache__
 flagged/
 gutenberg-dammit-files-v002.zip
 tmp_trainer
-*.gz
+*.gz
+gpt2-poetry-model

app.py

@@ -3,7 +3,7 @@ import gradio as gr
 from transformers import pipeline
 
 # Set up the generatove model transformer pipeline
-generator = pipeline("text-generation", model="tmp_trainer")
+generator = pipeline("text-generation", model="matthh/gpt2-poetry-model")
 
 # A sequence of lines both those typed in and the line so far
 # when save is clicked the txt file is downloaded

fine-tuning-for-casual-language-model.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 43,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -43,29 +43,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "ImportError",
-     "evalue": "This example requires a source install from HuggingFace Transformers (see `https://huggingface.co/transformers/installation.html#installing-from-source`), but the version found is 4.11.3.\nCheck out https://huggingface.co/transformers/examples.html for the examples corresponding to other versions of HuggingFace Transformers.",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn [4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mcheck_min_version\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m4.23.0.dev0\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/augmented_poetry/lib/python3.8/site-packages/transformers/utils/__init__.py:32\u001b[0m, in \u001b[0;36mcheck_min_version\u001b[0;34m(min_version)\u001b[0m\n\u001b[1;32m     30\u001b[0m     error_message \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mThis example requires a minimum version of \u001b[39m\u001b[39m{\u001b[39;00mmin_version\u001b[39m}\u001b[39;00m\u001b[39m,\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     31\u001b[0m error_message \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m but the version found is \u001b[39m\u001b[39m{\u001b[39;00m__version__\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m---> 32\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mImportError\u001b[39;00m(\n\u001b[1;32m     33\u001b[0m     error_message\n\u001b[1;32m     34\u001b[0m     \u001b[39m+\u001b[39m (\n\u001b[1;32m     35\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mCheck out https://huggingface.co/transformers/examples.html for the examples corresponding to other \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     36\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mversions of HuggingFace Transformers.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m     37\u001b[0m     )\n\u001b[1;32m     38\u001b[0m )\n",
-      "\u001b[0;31mImportError\u001b[0m: This example requires a source install from HuggingFace Transformers (see `https://huggingface.co/transformers/installation.html#installing-from-source`), but the version found is 4.11.3.\nCheck out https://huggingface.co/transformers/examples.html for the examples corresponding to other versions of HuggingFace Transformers."
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# check_min_version(\"4.23.0.dev0\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -74,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -90,90 +77,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Using custom data configuration merve--poetry-ca9a13ef5858cc3a\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Downloading and preparing dataset csv/merve--poetry to /Users/matth/.cache/huggingface/datasets/merve___csv/merve--poetry-ca9a13ef5858cc3a/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a...\n"
+      "/opt/homebrew/Caskroom/miniforge/base/envs/augmented_poetry/lib/python3.8/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n",
+      "  warnings.warn(message, FutureWarning)\n",
+      "Using custom data configuration merve--poetry-ca9a13ef5858cc3a\n",
+      "Found cached dataset csv (/Users/matth/.cache/huggingface/datasets/merve___csv/merve--poetry-ca9a13ef5858cc3a/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a)\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ed56ee6b324647798b19ac7bf5accc40",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "32c10441ff20404cb153f6b27f16a829",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading data:   0%|          | 0.00/606k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7ca47bc06937463e91d3948d7703ac64",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1631dbdc53d04b14a8a7733883bbd1cc",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "0 tables [00:00, ? tables/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Dataset csv downloaded and prepared to /Users/matth/.cache/huggingface/datasets/merve___csv/merve--poetry-ca9a13ef5858cc3a/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a. Subsequent calls will reuse this data.\n"
-     ]
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3c93229d66ad46d9a88da5f6a9528f2e",
+       "model_id": "67606d054e4a4b2f9ddf99f07c02c328",
        "version_major": 2,
        "version_minor": 0
       },
@@ -191,7 +111,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -200,16 +120,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
-    "config = AutoConfig.from_pretrained('gpt2')"
+    "config = AutoConfig.from_pretrained('gpt2')\n",
+    "\n",
+    "# max_seq_length"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -218,7 +140,7 @@
        "Embedding(50257, 768)"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -228,12 +150,13 @@
     "    \"gpt2\",\n",
     "    config=config\n",
     ")\n",
+    "model.max_seq_length = 128\n",
     "model.resize_token_embeddings(len(tokenizer))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -245,7 +168,7 @@
        "})"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -256,7 +179,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -265,7 +188,7 @@
        "'Mythology & Folklore'"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -276,7 +199,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -290,7 +213,7 @@
        "})"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -301,7 +224,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -312,7 +235,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -330,7 +253,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 33,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -341,29 +264,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 34,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "82c09dbdfa1a47d79607a4c9729fb286",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Running tokenizer on dataset:   0%|          | 0/1 [00:00<?, ?ba/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Token indices sequence length is longer than the specified maximum sequence length for this model (7725 > 1024). Running this sequence through the model will result in indexing errors\n",
-      "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n"
+      "Loading cached processed dataset at /Users/matth/.cache/huggingface/datasets/merve___csv/merve--poetry-ca9a13ef5858cc3a/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a/cache-62fd9c772e30c8d3.arrow\n"
      ]
     }
    ],
@@ -380,7 +288,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -389,7 +297,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -413,22 +321,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 44,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ca2f64461e304df6aecb16e8cfcd42ac",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Grouping texts in chunks of 1024:   0%|          | 0/1 [00:00<?, ?ba/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading cached processed dataset at /Users/matth/.cache/huggingface/datasets/merve___csv/merve--poetry-ca9a13ef5858cc3a/0.0.0/652c3096f041ee27b04d2232d41f10547a8fecda3e284a79a0ec4053c916ef7a/cache-88d7c64be469684a.arrow\n"
+     ]
     }
    ],
    "source": [
@@ -443,7 +344,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -459,14 +360,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "training_args = TrainingArguments(\n",
+    "    output_dir=\"gpt2-poetry-model\", \n",
+    "    overwrite_output_dir=True,\n",
+    "    # per_gpu_train_batch_size=256\n",
+    "    per_device_train_batch_size=16,\n",
+    "    push_to_hub=True,\n",
+    "    push_to_hub_token=\"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\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Initialize our Trainer\n",
     "trainer = Trainer(\n",
     "    model=model,\n",
-    "    # args=training_args,\n",
+    "    args=training_args,\n",
     "    train_dataset=train_dataset,\n",
     "    # eval_dataset=eval_dataset,\n",
     "    tokenizer=tokenizer,\n",
@@ -483,7 +400,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 48,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -558,18 +475,69 @@
    ],
    "source": [
     "# Training\n",
-    "checkpoint = None\n",
-    "train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
-    "trainer.save_model()  # Saves the tokenizer too for easy upload\n",
+    "# checkpoint = None\n",
+    "# train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
+    "# trainer.save_model()  # Saves the tokenizer too for easy upload\n",
     "\n",
-    "metrics = train_result.metrics\n",
+    "# metrics = train_result.metrics\n",
     "\n",
-    "max_train_samples = (len(train_dataset))\n",
-    "metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n",
+    "# max_train_samples = (len(train_dataset))\n",
+    "# metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n",
     "\n",
-    "trainer.log_metrics(\"train\", metrics)\n",
-    "trainer.save_metrics(\"train\", metrics)\n",
-    "trainer.save_state()"
+    "# trainer.log_metrics(\"train\", metrics)\n",
+    "# trainer.save_metrics(\"train\", metrics)\n",
+    "# trainer.save_state()\n",
+    "# # Upload the the hugging face hub for easy use in inference.\n",
+    "# trainer.push_to_hub()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2cec8af2b332409bb857695a7b099653",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Saving model checkpoint to gpt2-poetry-model\n",
+      "Configuration saved in gpt2-poetry-model/config.json\n",
+      "Model weights saved in gpt2-poetry-model/pytorch_model.bin\n",
+      "tokenizer config file saved in gpt2-poetry-model/tokenizer_config.json\n",
+      "Special tokens file saved in gpt2-poetry-model/special_tokens_map.json\n"
+     ]
+    },
+    {
+     "ename": "AttributeError",
+     "evalue": "'Trainer' object has no attribute 'repo'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn [27], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhuggingface_hub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m notebook_login\n\u001b[1;32m      2\u001b[0m notebook_login()\n\u001b[0;32m----> 3\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpush_to_hub\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/homebrew/Caskroom/miniforge/base/envs/augmented_poetry/lib/python3.8/site-packages/transformers/trainer.py:2677\u001b[0m, in \u001b[0;36mTrainer.push_to_hub\u001b[0;34m(self, commit_message, blocking, **kwargs)\u001b[0m\n\u001b[1;32m   2674\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mis_world_process_zero():\n\u001b[1;32m   2675\u001b[0m     \u001b[39mreturn\u001b[39;00m\n\u001b[0;32m-> 2677\u001b[0m git_head_commit_url \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrepo\u001b[39m.\u001b[39mpush_to_hub(commit_message\u001b[39m=\u001b[39mcommit_message, blocking\u001b[39m=\u001b[39mblocking)\n\u001b[1;32m   2678\u001b[0m \u001b[39m# push separately the model card to be independant from the rest of the model\u001b[39;00m\n\u001b[1;32m   2679\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mshould_save:\n",
+      "\u001b[0;31mAttributeError\u001b[0m: 'Trainer' object has no attribute 'repo'"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import notebook_login\n",
+    "notebook_login()\n",
+    "trainer.push_to_hub()"
    ]
   }
  ],