Upload folder using huggingface_hub

README.md

@@ -0,0 +1,89 @@
+---
+tags:
+- generated_from_trainer
+datasets:
+- roneneldan/TinyStories
+metrics:
+- accuracy
+model-index:
+- name: gpt2_u030_tiny-stories_1024_dpos
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: roneneldan/TinyStories
+      type: roneneldan/TinyStories
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.6788553562327562
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/scads-nlp/morph-gpt_gpt2_tiny-stories_dpos/runs/s3ht4a0t)
+# gpt2_u030_tiny-stories_1024_dpos
+
+This model is a fine-tuned version of [](https://huggingface.co/) on the roneneldan/TinyStories dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.2107
+- Accuracy: 0.6789
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 32
+- eval_batch_size: 32
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 1.0
+
+### Training results
+
+| Training Loss | Epoch  | Step  | Validation Loss | Accuracy |
+|:-------------:|:------:|:-----:|:---------------:|:--------:|
+| 2.92          | 0.0523 | 1000  | 2.4566          | 0.4470   |
+| 1.9818        | 0.1046 | 2000  | 1.8089          | 0.5672   |
+| 1.7321        | 0.1570 | 3000  | 1.6133          | 0.6021   |
+| 1.6137        | 0.2093 | 4000  | 1.5171          | 0.6195   |
+| 1.5353        | 0.2616 | 5000  | 1.4516          | 0.6312   |
+| 1.4845        | 0.3139 | 6000  | 1.4056          | 0.6400   |
+| 1.4443        | 0.3662 | 7000  | 1.3718          | 0.6466   |
+| 1.4118        | 0.4186 | 8000  | 1.3420          | 0.6525   |
+| 1.3878        | 0.4709 | 9000  | 1.3189          | 0.6569   |
+| 1.3661        | 0.5232 | 10000 | 1.2988          | 0.6608   |
+| 1.3485        | 0.5755 | 11000 | 1.2834          | 0.6639   |
+| 1.3326        | 0.6278 | 12000 | 1.2675          | 0.6669   |
+| 1.319         | 0.6802 | 13000 | 1.2555          | 0.6694   |
+| 1.3068        | 0.7325 | 14000 | 1.2440          | 0.6719   |
+| 1.2932        | 0.7848 | 15000 | 1.2350          | 0.6737   |
+| 1.2868        | 0.8371 | 16000 | 1.2263          | 0.6755   |
+| 1.2791        | 0.8894 | 17000 | 1.2193          | 0.6771   |
+| 1.2725        | 0.9418 | 18000 | 1.2141          | 0.6780   |
+| 1.2711        | 0.9941 | 19000 | 1.2108          | 0.6788   |
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.2.2+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_accuracy": 0.6788553562327562,
+    "eval_loss": 1.2106505632400513,
+    "eval_runtime": 53.3968,
+    "eval_samples": 6151,
+    "eval_samples_per_second": 115.194,
+    "eval_steps_per_second": 3.614,
+    "perplexity": 3.35566701619586,
+    "total_flos": 3.19615549046784e+17,
+    "train_loss": 1.5581016430754755,
+    "train_runtime": 7493.9423,
+    "train_samples": 611606,
+    "train_samples_per_second": 81.613,
+    "train_steps_per_second": 2.55
+}

config.json

@@ -0,0 +1,31 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 1024,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 1024,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 2048,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 50257
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 1.0,
+    "eval_accuracy": 0.6788553562327562,
+    "eval_loss": 1.2106505632400513,
+    "eval_runtime": 53.3968,
+    "eval_samples": 6151,
+    "eval_samples_per_second": 115.194,
+    "eval_steps_per_second": 3.614,
+    "perplexity": 3.35566701619586
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1024,
+  "eos_token_id": 1024,
+  "transformers_version": "4.42.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dc79872bad471e18a190a91f687e947f31102ddfa2472882038e0a482af24cb
+size 500919936

special_tokens_map.json

@@ -0,0 +1 @@
+{}

tokenizer.json

@@ -0,0 +1,1919 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 1024,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": false,
+    "use_regex": true
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "_": 61,
+      "`": 62,
+      "a": 63,
+      "b": 64,
+      "c": 65,
+      "d": 66,
+      "e": 67,
+      "f": 68,
+      "g": 69,
+      "h": 70,
+      "i": 71,
+      "j": 72,
+      "k": 73,
+      "l": 74,
+      "m": 75,
+      "n": 76,
+      "o": 77,
+      "p": 78,
+      "q": 79,
+      "r": 80,
+      "s": 81,
+      "t": 82,
+      "u": 83,
+      "v": 84,
+      "w": 85,
+      "x": 86,
+      "y": 87,
+      "z": 88,
+      "{": 89,
+      "|": 90,
+      "}": 91,
+      "~": 92,
+      "¡": 93,
+      "¢": 94,
+      "£": 95,
+      "¤": 96,
+      "¥": 97,
+      "¦": 98,
+      "§": 99,
+      "¨": 100,
+      "©": 101,
+      "ª": 102,
+      "«": 103,
+      "¬": 104,
+      "®": 105,
+      "¯": 106,
+      "°": 107,
+      "±": 108,
+      "³": 109,
+      "´": 110,
+      "µ": 111,
+      "¶": 112,
+      "·": 113,
+      "¸": 114,
+      "¹": 115,
+      "º": 116,
+      "»": 117,
+      "¼": 118,
+      "½": 119,
+      "¾": 120,
+      "¿": 121,
+      "Â": 122,
+      "Ã": 123,
+      "Ä": 124,
+      "Å": 125,
+      "É": 126,
+      "Ê": 127,
+      "Ñ": 128,
+      "Ò": 129,
+      "á": 130,
+      "â": 131,
+      "ã": 132,
+      "ä": 133,
+      "å": 134,
+      "æ": 135,
+      "ç": 136,
+      "è": 137,
+      "é": 138,
+      "î": 139,
+      "ï": 140,
+      "ð": 141,
+      "ĉ": 142,
+      "Ċ": 143,
+      "Ġ": 144,
+      "Ģ": 145,
+      "ģ": 146,
+      "Ĥ": 147,
+      "ĥ": 148,
+      "Ħ": 149,
+      "ħ": 150,
+      "Ĩ": 151,
+      "ĩ": 152,
+      "Ī": 153,
+      "ī": 154,
+      "Ĭ": 155,
+      "ĭ": 156,
+      "Į": 157,
+      "į": 158,
+      "İ": 159,
+      "ı": 160,
+      "IJ": 161,
+      "ij": 162,
+      "Ĵ": 163,
+      "ĵ": 164,
+      "Ķ": 165,
+      "ķ": 166,
+      "ĸ": 167,
+      "ĺ": 168,
+      "Ļ": 169,
+      "ļ": 170,
+      "Ľ": 171,
+      "ľ": 172,
+      "Ŀ": 173,
+      "ŀ": 174,
+      "Ł": 175,
+      "ł": 176,
+      "Ń": 177,
+      "Ġt": 178,
+      "Ġa": 179,
+      "he": 180,
+      "Ġs": 181,
+      "nd": 182,
+      "Ġw": 183,
+      "Ġthe": 184,
+      "ed": 185,
+      "Ġb": 186,
+      "ĠT": 187,
+      "Ġto": 188,
+      "Ġand": 189,
+      "Ġh": 190,
+      "Ġf": 191,
+      "in": 192,
+      "Ġwa": 193,
+      "re": 194,
+      "it": 195,
+      "ou": 196,
+      "Ġl": 197,
+      "Ġd": 198,
+      "Ġc": 199,
+      "Ġp": 200,
+      "ay": 201,
+      "Ġm": 202,
+      "er": 203,
+      "Ġwas": 204,
+      "om": 205,
+      "Ġhe": 206,
+      "is": 207,
+      "ĠThe": 208,
+      "im": 209,
+      "Ġn": 210,
+      "on": 211,
+      "ĠS": 212,
+      "ar": 213,
+      "Ġsa": 214,
+      "ll": 215,
+      "id": 216,
+      "Ġha": 217,
+      "at": 218,
+      "Ġg": 219,
+      "ing": 220,
+      "en": 221,
+      "ot": 222,
+      "an": 223,
+      "le": 224,
+      "end": 225,
+      "or": 226,
+      "Ġ\"": 227,
+      "of": 228,
+      "ĠH": 229,
+      "ir": 230,
+      "am": 231,
+      "et": 232,
+      "Ġit": 233,
+      "Ġth": 234,
+      "ig": 235,
+      "ĠHe": 236,
+      "Ġin": 237,
+      "Ġpl": 238,
+      "ĠO": 239,
+      "ly": 240,
+      "ri": 241,
+      "ver": 242,
+      "ut": 243,
+      "ow": 244,
+      "Ġbe": 245,
+      "Ġu": 246,
+      "Ġsaid": 247,
+      "Ġplay": 248,
+      "ith": 249,
+      "Ġwith": 250,
+      "Ġday": 251,
+      "Ġy": 252,
+      "ked": 253,
+      "pp": 254,
+      "ex": 255,
+      "Ġr": 256,
+      "ce": 257,
+      "ld": 258,
+      "oo": 259,
+      "ĠI": 260,
+      "ĠThey": 261,
+      "Ġher": 262,
+      "ĠL": 263,
+      "Ġhis": 264,
+      "Ġst": 265,
+      "es": 266,
+      "ĠOn": 267,
+      "ĠB": 268,
+      "ck": 269,
+      "Ġbig": 270,
+      "nt": 271,
+      "Ġyou": 272,
+      "ke": 273,
+      "ext": 274,
+      "very": 275,
+      "oft": 276,
+      "ĠM": 277,
+      "Ġon": 278,
+      "un": 279,
+      "Ġhapp": 280,
+      "st": 281,
+      "ve": 282,
+      "Ġfri": 283,
+      "Ġfriend": 284,
+      "ĠTim": 285,
+      "Ġli": 286,
+      "ĠLi": 287,
+      "Ġthey": 288,
+      "all": 289,
+      "Ġwe": 290,
+      "Ġhad": 291,
+      "Ġnot": 292,
+      "Ġup": 293,
+      "Ġwant": 294,
+      "Ġof": 295,
+      "her": 296,
+      "ĠShe": 297,
+      "ad": 298,
+      "Ġ<": 299,
+      "|>": 300,
+      "Ġ<|": 301,
+      "itt": 302,
+      "Ġdo": 303,
+      "Ġe": 304,
+      "se": 305,
+      "ĠA": 306,
+      "Ġhappy": 307,
+      "Ġvery": 308,
+      "ent": 309,
+      "Ġthat": 310,
+      "Ġsaw": 311,
+      "'s": 312,
+      "Ġfor": 313,
+      "Ġsh": 314,
+      "Ġmom": 315,
+      "ittle": 316,
+      "ould": 317,
+      "Ġlittle": 318,
+      "endoft": 319,
+      "Ġshe": 320,
+      "Ġk": 321,
+      "ime": 322,
+      "ch": 323,
+      ".\"": 324,
+      "Ġnam": 325,
+      "Ġtime": 326,
+      "Ġsm": 327,
+      "Ġso": 328,
+      "ound": 329,
+      "Ġthere": 330,
+      "Ġnamed": 331,
+      "Ġbo": 332,
+      "Ġwere": 333,
+      "Ġwanted": 334,
+      "Ġbut": 335,
+      "ved": 336,
+      "Ġfriends": 337,
+      "out": 338,
+      "ht": 339,
+      "!\"": 340,
+      "endoftext": 341,
+      "Ġne": 342,
+      "Ġan": 343,
+      "ird": 344,
+      "al": 345,
+      "Ġbird": 346,
+      "ue": 347,
+      "ĠIt": 348,
+      "Ġtoo": 349,
+      "ĠĊ": 350,
+      "il": 351,
+      "Ġhel": 352,
+      "el": 353,
+      "ide": 354,
+      "ome": 355,
+      "Ġwent": 356,
+      "Ġwh": 357,
+      "ry": 358,
+      "ĠLily": 359,
+      "ĠOne": 360,
+      "Ġis": 361,
+      "Ġall": 362,
+      "Ġlo": 363,
+      "ake": 364,
+      "Ġloo": 365,
+      "Ġupon": 366,
+      "ter": 367,
+      "ĠTom": 368,
+      "ug": 369,
+      "Ġhelp": 370,
+      "ore": 371,
+      "ame": 372,
+      "Ġfun": 373,
+      "ind": 374,
+      "Ġtoy": 375,
+      "Ġas": 376,
+      "get": 377,
+      "Ġj": 378,
+      "Ġat": 379,
+      "Ġre": 380,
+      "ra": 381,
+      "gether": 382,
+      "Ġo": 383,
+      "Ġse": 384,
+      "ill": 385,
+      "ur": 386,
+      "Ġdid": 387,
+      "Ġtogether": 388,
+      "Ġba": 389,
+      "Ġcat": 390,
+      "Ġtre": 391,
+      "ood": 392,
+      "ted": 393,
+      "ic": 394,
+      "my": 395,
+      "Ġdog": 396,
+      "Ġcan": 397,
+      "ec": 398,
+      "Ġcould": 399,
+      "Ġtheir": 400,
+      "ard": 401,
+      "hed": 402,
+      "ax": 403,
+      "Ġgir": 404,
+      "Ġhim": 405,
+      "Ġro": 406,
+      "Ġplayed": 407,
+      "Ġball": 408,
+      "ark": 409,
+      "Ġkn": 410,
+      "?\"": 411,
+      "Ġgirl": 412,
+      "ĠOnce": 413,
+      "way": 414,
+      "Ġgo": 415,
+      "Ġle": 416,
+      "Ġare": 417,
+      "Ġfr": 418,
+      "ĠW": 419,
+      "Ġout": 420,
+      "ain": 421,
+      "'t": 422,
+      "Ġthem": 423,
+      "Ġsad": 424,
+      "ul": 425,
+      "um": 426,
+      "Ġboy": 427,
+      "Ġtree": 428,
+      "Ġcl": 429,
+      "Ġhave": 430,
+      "one": 431,
+      "Ġman": 432,
+      "hen": 433,
+      "Ġlooked": 434,
+      "Ġloved": 435,
+      "Ġfound": 436,
+      "other": 437,
+      "Ġsp": 438,
+      "oug": 439,
+      "Ġsc": 440,
+      "Ġstar": 441,
+      "Ġbec": 442,
+      "hing": 443,
+      "Ġback": 444,
+      "ĠJ": 445,
+      "ful": 446,
+      "Ġla": 447,
+      "own": 448,
+      "side": 449,
+      "Ġme": 450,
+      "are": 451,
+      "Ġpark": 452,
+      "Ġcar": 453,
+      "ong": 454,
+      "ight": 455,
+      "elt": 456,
+      "Ġwould": 457,
+      "op": 458,
+      "round": 459,
+      "Ġfa": 460,
+      "Ġlike": 461,
+      "Ġfelt": 462,
+      "Ġsee": 463,
+      "Ġno": 464,
+      "ĠF": 465,
+      "Ġasked": 466,
+      "omet": 467,
+      "Ġnew": 468,
+      "ĠSp": 469,
+      "Ġstarted": 470,
+      "ared": 471,
+      "Ġcame": 472,
+      "ice": 473,
+      "ouse": 474,
+      "Ġal": 475,
+      "ĠBut": 476,
+      "ia": 477,
+      "ss": 478,
+      "Ġmake": 479,
+      "Ġsomet": 480,
+      "ought": 481,
+      "Ġother": 482,
+      "Ġag": 483,
+      "Ġbr": 484,
+      "Ġgood": 485,
+      "Ġsmall": 486,
+      "ĠSue": 487,
+      "ade": 488,
+      "ob": 489,
+      "ell": 490,
+      "ings": 491,
+      "ried": 492,
+      "ĠMax": 493,
+      "ened": 494,
+      "Ġwal": 495,
+      "Ġex": 496,
+      "Ġwor": 497,
+      "Ġco": 498,
+      "Ġfind": 499,
+      "ag": 500,
+      "ty": 501,
+      "Ġaway": 502,
+      "Ġsomething": 503,
+      "Ġwhat": 504,
+      "Ġfrom": 505,
+      "Ġmade": 506,
+      "Ġput": 507,
+      "iled": 508,
+      "Ġhome": 509,
+      "Ġthought": 510,
+      "Ġplaying": 511,
+      "Ġsay": 512,
+      "ĠY": 513,
+      "ĠBen": 514,
+      "Ġmu": 515,
+      "uc": 516,
+      "ach": 517,
+      "arn": 518,
+      "Ġran": 519,
+      "Ġfl": 520,
+      "ile": 521,
+      "ĠSam": 522,
+      "ie": 523,
+      "Ġevery": 524,
+      "ny": 525,
+      "Ġagain": 526,
+      "ook": 527,
+      "dd": 528,
+      "ew": 529,
+      "Ġdown": 530,
+      "Ġpr": 531,
+      "Ġsome": 532,
+      "Ġtook": 533,
+      "Ġscared": 534,
+      "Ġliked": 535,
+      "Ġtoys": 536,
+      "king": 537,
+      "Ġlearn": 538,
+      "Ġhouse": 539,
+      "ure": 540,
+      "Ġwill": 541,
+      "Ġbox": 542,
+      "ep": 543,
+      "ret": 544,
+      "if": 545,
+      "Ġbl": 546,
+      "Ġmy": 547,
+      "Ġthings": 548,
+      "Ġaround": 549,
+      "ble": 550,
+      "Ġyour": 551,
+      "Ġsw": 552,
+      "ish": 553,
+      "Ġwhen": 554,
+      "ĠSo": 555,
+      "Ġlived": 556,
+      "pped": 557,
+      "Ġsun": 558,
+      "Ġthen": 559,
+      "ank": 560,
+      "oud": 561,
+      "Ġch": 562,
+      ",\"": 563,
+      "Ġlaug": 564,
+      "us": 565,
+      "ick": 566,
+      "Th": 567,
+      "uck": 568,
+      "ĠD": 569,
+      "Ġta": 570,
+      "Ġget": 571,
+      "ucy": 572,
+      "ist": 573,
+      "Ġsmiled": 574,
+      "Ġdec": 575,
+      "ĠLucy": 576,
+      "Ġfe": 577,
+      "Ġtried": 578,
+      "oth": 579,
+      "ily": 580,
+      "Ġgot": 581,
+      "Ġknow": 582,
+      "Ġwho": 583,
+      "ave": 584,
+      "qu": 585,
+      "Ġab": 586,
+      "Ġmany": 587,
+      "Ġany": 588,
+      "au": 589,
+      "ause": 590,
+      "na": 591,
+      "Ġint": 592,
+      "ited": 593,
+      "Ġabout": 594,
+      "ous": 595,
+      "ĠSpot": 596,
+      "Ġpret": 597,
+      "Ġlot": 598,
+      "Ġv": 599,
+      "ers": 600,
+      "Ġred": 601,
+      "ĠE": 602,
+      "ise": 603,
+      "ust": 604,
+      "Ġmore": 605,
+      "ided": 606,
+      "Ġpic": 607,
+      "nder": 608,
+      "Ġcare": 609,
+      "ace": 610,
+      "Ġpo": 611,
+      "as": 612,
+      "urp": 613,
+      "Ġlearned": 614,
+      "Ġun": 615,
+      "Ġwater": 616,
+      "Ġhug": 617,
+      "Ġexc": 618,
+      "Ġbest": 619,
+      "ump": 620,
+      "Ġop": 621,
+      "ap": 622,
+      "Ġgre": 623,
+      "ways": 624,
+      "âĢ": 625,
+      "Ġoutside": 626,
+      "ĠMia": 627,
+      "fe": 628,
+      "Ġalways": 629,
+      "ant": 630,
+      "Ġroom": 631,
+      "Ġho": 632,
+      "Ġinto": 633,
+      "ive": 634,
+      "Ġeat": 635,
+      "ĠMom": 636,
+      "ĠAn": 637,
+      "Ġpe": 638,
+      "ite": 639,
+      "Ġboth": 640,
+      "ĠFr": 641,
+      "ĠYou": 642,
+      "Ġdad": 643,
+      "Ġke": 644,
+      "ally": 645,
+      "Ġlook": 646,
+      "Ġone": 647,
+      "Ġshow": 648,
+      "Ġthis": 649,
+      "Ġexcited": 650,
+      "Ġfast": 651,
+      "Ġnice": 652,
+      "Ġsurp": 653,
+      "Ġrun": 654,
+      "Ġlong": 655,
+      "ĠC": 656,
+      "Ġam": 657,
+      "fter": 658,
+      "als": 659,
+      "Ġsk": 660,
+      "Ġjump": 661,
+      "Ġtold": 662,
+      "Ġfeel": 663,
+      "Ġmo": 664,
+      "Ġinside": 665,
+      "Ġtr": 666,
+      "our": 667,
+      "ull": 668,
+      "Ġsl": 669,
+      "Ġpretty": 670,
+      "iny": 671,
+      "ĠBob": 672,
+      "Ġgra": 673,
+      "udd": 674,
+      "ged": 675,
+      "Ġeach": 676,
+      "Ġgave": 677,
+      "Ġmuch": 678,
+      "bb": 679,
+      "Ġsays": 680,
+      "Ġstr": 681,
+      "Ġhow": 682,
+      "ink": 683,
+      "og": 684,
+      "Ġtow": 685,
+      "Ġor": 686,
+      "ĠAs": 687,
+      "lew": 688,
+      "Ġneed": 689,
+      "etter": 690,
+      "Ġunder": 691,
+      "ised": 692,
+      "ven": 693,
+      "Ġold": 694,
+      "Ġrock": 695,
+      "Ġtake": 696,
+      "ess": 697,
+      "Ġknew": 698,
+      "ro": 699,
+      "ĠTh": 700,
+      "Ġca": 701,
+      "Ġcle": 702,
+      "Ġfish": 703,
+      "here": 704,
+      "Ġlist": 705,
+      "Ġclo": 706,
+      "urt": 707,
+      "ĠAnd": 708,
+      "Ġbear": 709,
+      "ched": 710,
+      "Ġhand": 711,
+      "ara": 712,
+      "Ġtry": 713,
+      "Ġbecame": 714,
+      "Ġthan": 715,
+      "ĠWe": 716,
+      "Ġkind": 717,
+      "est": 718,
+      "urn": 719,
+      "Ġlaughed": 720,
+      "Ġjust": 721,
+      "orry": 722,
+      "Ġfood": 723,
+      "more": 724,
+      "ge": 725,
+      "Ġhig": 726,
+      "Ġfi": 727,
+      "Ġwat": 728,
+      "ĠIn": 729,
+      "ine": 730,
+      "Ġsurpr": 731,
+      "ak": 732,
+      "Ġdecided": 733,
+      "Ġsky": 734,
+      "Ġbecause": 735,
+      "Ġide": 736,
+      "Ġus": 737,
+      "ving": 738,
+      "��near": 739,
+      "Ġtw": 740,
+      "Ġidea": 741,
+      "Ġbetter": 742,
+      "Ġbug": 743,
+      "Ġheard": 744,
+      "gry": 745,
+      "Ġits": 746,
+      "ase": 747,
+      "ĠThen": 748,
+      "Ġen": 749,
+      "ack": 750,
+      "ate": 751,
+      "Wh": 752,
+      "lf": 753,
+      "Ġlet": 754,
+      "ort": 755,
+      "ĠK": 756,
+      "by": 757,
+      "Ġanim": 758,
+      "ĠWhen": 759,
+      "ff": 760,
+      "Ġshare": 761,
+      "Ġcareful": 762,
+      "Ġcol": 763,
+      "Ġif": 764,
+      "Ġte": 765,
+      "eci": 766,
+      "Ġgr": 767,
+      "Ġfly": 768,
+      "Ġfo": 769,
+      "Ġstor": 770,
+      "Ġflew": 771,
+      "ves": 772,
+      "Ġdan": 773,
+      "Yes": 774,
+      "Ġcom": 775,
+      "ecial": 776,
+      "Ġspecial": 777,
+      "ion": 778,
+      "Ġflow": 779,
+      "Ġnever": 780,
+      "Ġby": 781,
+      "ected": 782,
+      "Ġbu": 783,
+      "Ġdon": 784,
+      "Ġra": 785,
+      "Ġwind": 786,
+      "Ġtal": 787,
+      "Ġclean": 788,
+      "rm": 789,
+      "Ġqu": 790,
+      "Ġdre": 791,
+      "ĠHis": 792,
+      "Ġend": 793,
+      "Ġeven": 794,
+      "ople": 795,
+      "Ġsafe": 796,
+      "Ġmag": 797,
+      "Ġunex": 798,
+      "Ġunexp": 799,
+      "Ġshiny": 800,
+      "Ġhard": 801,
+      "Ġover": 802,
+      "Ġbook": 803,
+      "ĠWh": 804,
+      "Ġturn": 805,
+      "ĠSara": 806,
+      "Ġfam": 807,
+      "Ġfore": 808,
+      "Ġbad": 809,
+      "Ġcake": 810,
+      "Ġafter": 811,
+      "ady": 812,
+      "Ġpeople": 813,
+      "Ġcu": 814,
+      "pl": 815,
+      "ĠP": 816,
+      "Ġproud": 817,
+      "ber": 818,
+      "Ġhigh": 819,
+      "Ġgl": 820,
+      "Ġche": 821,
+      "ĠAnna": 822,
+      "ĠAmy": 823,
+      "Ġcome": 824,
+      "Ġgard": 825,
+      "Ġdoor": 826,
+      "Ġopened": 827,
+      "Ġground": 828,
+      "Ġgarden": 829,
+      "Ġpicked": 830,
+      "Ġim": 831,
+      "Ġhurt": 832,
+      "Ġloud": 833,
+      "ĠN": 834,
+      "Ġsto": 835,
+      "ild": 836,
+      "Ġsorry": 837,
+      "Ġway": 838,
+      "ip": 839,
+      "'m": 840,
+      "Ġblue": 841,
+      "Ġstill": 842,
+      "Ġever": 843,
+      "Ġhugged": 844,
+      "Ġfar": 845,
+      "Ġcall": 846,
+      "Ġpa": 847,
+      "Ġlove": 848,
+      "age": 849,
+      "hes": 850,
+      "Ġoff": 851,
+      "Ġmagic": 852,
+      "ough": 853,
+      "Ġwalked": 854,
+      "Ġsn": 855,
+      "udden": 856,
+      "Ġpar": 857,
+      "Ġfamily": 858,
+      "Ġshould": 859,
+      "ool": 860,
+      "be": 861,
+      "Ġclim": 862,
+      "iz": 863,
+      "Ġkid": 864,
+      "ies": 865,
+      "dy": 866,
+      "Ġma": 867,
+      "Ġnow": 868,
+      "Ġhappened": 869,
+      "Ġgreat": 870,
+      "ĠHer": 871,
+      "Ġdidn": 872,
+      "em": 873,
+      "Ġplace": 874,
+      "Ġanimals": 875,
+      "Ġstrong": 876,
+      "uff": 877,
+      "Ġunt": 878,
+      "uddenly": 879,
+      "ĠJo": 880,
+      "Ġwalk": 881,
+      "ct": 882,
+      "Ġforest": 883,
+      "Ġbra": 884,
+      "No": 885,
+      "Ġclimb": 886,
+      "aut": 887,
+      "ĠFrom": 888,
+      "Ġgive": 889,
+      "Ġbro": 890,
+      "Ġwo": 891,
+      "Ġsqu": 892,
+      "Ġuntil": 893,
+      "Ġbeaut": 894,
+      "Ġapp": 895,
+      "ĠEvery": 896,
+      "Ġnext": 897,
+      "Ġfrog": 898,
+      "Ġboat": 899,
+      "ning": 900,
+      "Ġstick": 901,
+      "Ġjo": 902,
+      "Ġtra": 903,
+      "Ġimp": 904,
+      "Ġpict": 905,
+      "Ġkids": 906,
+      "Ġstay": 907,
+      "Ġbeauti": 908,
+      "Ġbeautiful": 909,
+      "Ġclos": 910,
+      "Thank": 911,
+      "ĠR": 912,
+      "Ġdra": 913,
+      "Ġlea": 914,
+      "Ġlisten": 915,
+      "Ġstory": 916,
+      "Ġanymore": 917,
+      "ummy": 918,
+      "Ġwhile": 919,
+      "pt": 920,
+      "rel": 921,
+      "Ġrem": 922,
+      "Ġtown": 923,
+      "ane": 924,
+      "Ġcre": 925,
+      "Ġlots": 926,
+      "ĠAt": 927,
+      "kay": 928,
+      "Ġad": 929,
+      "cks": 930,
+      "ary": 931,
+      "Ġbeing": 932,
+      "Ġrain": 933,
+      "Ġmor": 934,
+      "unny": 935,
+      "ors": 936,
+      "Ġcry": 937,
+      "Ġcalled": 938,
+      "th": 939,
+      "and": 940,
+      "Ġmet": 941,
+      "les": 942,
+      "Ġgame": 943,
+      "Ġangry": 944,
+      "ger": 945,
+      "Ġhat": 946,
+      "lease": 947,
+      "Ġthre": 948,
+      "self": 949,
+      "Ġsoft": 950,
+      "ma": 951,
+      "uffy": 952,
+      "Ġkeep": 953,
+      "Ġbed": 954,
+      "Ġlost": 955,
+      "Ġtruck": 956,
+      "Ġopen": 957,
+      "Ġwarm": 958,
+      "Ġmouse": 959,
+      "oon": 960,
+      "Ġate": 961,
+      "It": 962,
+      "fore": 963,
+      "Ġbrave": 964,
+      "so": 965,
+      "Ġwatch": 966,
+      "ĠG": 967,
+      "Ġsurprised": 968,
+      "Let": 969,
+      "ect": 970,
+      "Ġalso": 971,
+      "Ġfell": 972,
+      "Ġgreen": 973,
+      "Ġface": 974,
+      "Ġtwo": 975,
+      "ĠFl": 976,
+      "Ġunexpected": 977,
+      "ĠKitt": 978,
+      "You": 979,
+      "Ġpicture": 980,
+      "ĠâĢ": 981,
+      "vent": 982,
+      "Ġnoise": 983,
+      "bbit": 984,
+      "gan": 985,
+      "irst": 986,
+      "Ġwhere": 987,
+      "led": 988,
+      "Ġthanked": 989,
+      "Ġsoon": 990,
+      "kes": 991,
+      "Ġslide": 992,
+      "Ġsmile": 993,
+      "Ġhole": 994,
+      "Ġpus": 995,
+      "day": 996,
+      "Ġbefore": 997,
+      "Ġrabbit": 998,
+      "ired": 999,
+      "Ġsat": 1000,
+      "Ġdoll": 1001,
+      "ĠFluffy": 1002,
+      "irrel": 1003,
+      "Ġkept": 1004,
+      "ĠâĢľ": 1005,
+      "Hi": 1006,
+      "âĢĿ": 1007,
+      "iss": 1008,
+      "owl": 1009,
+      "Ġfin": 1010,
+      "ear": 1011,
+      "Ġsquirrel": 1012,
+      "Ġjumped": 1013,
+      "Ġblo": 1014,
+      "Ġuse": 1015,
+      "able": 1016,
+      "Ġstore": 1017,
+      "Ġhands": 1018,
+      "Ġbirds": 1019,
+      "Ġbre": 1020,
+      "Ġpain": 1021,
+      "Ġgoing": 1022,
+      "Ġfunny": 1023
+    },
+    "merges": [
+      "Ġ t",
+      "Ġ a",
+      "h e",
+      "Ġ s",
+      "n d",
+      "Ġ w",
+      "Ġt he",
+      "e d",
+      "Ġ b",
+      "Ġ T",
+      "Ġt o",
+      "Ġa nd",
+      "Ġ h",
+      "Ġ f",
+      "i n",
+      "Ġw a",
+      "r e",
+      "i t",
+      "o u",
+      "Ġ l",
+      "Ġ d",
+      "Ġ c",
+      "Ġ p",
+      "a y",
+      "Ġ m",
+      "e r",
+      "Ġwa s",
+      "o m",
+      "Ġ he",
+      "i s",
+      "ĠT he",
+      "i m",
+      "Ġ n",
+      "o n",
+      "Ġ S",
+      "a r",
+      "Ġs a",
+      "l l",
+      "i d",
+      "Ġh a",
+      "a t",
+      "Ġ g",
+      "in g",
+      "e n",
+      "o t",
+      "a n",
+      "l e",
+      "e nd",
+      "o r",
+      "Ġ \"",
+      "o f",
+      "Ġ H",
+      "i r",
+      "a m",
+      "e t",
+      "Ġ it",
+      "Ġt h",
+      "i g",
+      "ĠH e",
+      "Ġ in",
+      "Ġp l",
+      "Ġ O",
+      "l y",
+      "r i",
+      "v er",
+      "u t",
+      "o w",
+      "Ġb e",
+      "Ġ u",
+      "Ġsa id",
+      "Ġpl ay",
+      "it h",
+      "Ġw ith",
+      "Ġd ay",
+      "Ġ y",
+      "k ed",
+      "p p",
+      "e x",
+      "Ġ r",
+      "c e",
+      "l d",
+      "o o",
+      "Ġ I",
+      "ĠThe y",
+      "Ġhe r",
+      "Ġ L",
+      "Ġh is",
+      "Ġs t",
+      "e s",
+      "ĠO n",
+      "Ġ B",
+      "c k",
+      "Ġb ig",
+      "n t",
+      "Ġy ou",
+      "k e",
+      "ex t",
+      "ver y",
+      "of t",
+      "Ġ M",
+      "Ġ on",
+      "u n",
+      "Ġha pp",
+      "s t",
+      "v e",
+      "Ġf ri",
+      "Ġfri end",
+      "ĠT im",
+      "Ġl i",
+      "ĠL i",
+      "Ġthe y",
+      "a ll",
+      "Ġw e",
+      "Ġha d",
+      "Ġn ot",
+      "Ġu p",
+      "Ġwa nt",
+      "Ġ of",
+      "he r",
+      "ĠS he",
+      "a d",
+      "Ġ <",
+      "| >",
+      "Ġ< |",
+      "it t",
+      "Ġd o",
+      "Ġ e",
+      "s e",
+      "Ġ A",
+      "Ġhapp y",
+      "Ġ very",
+      "en t",
+      "Ġth at",
+      "Ġsa w",
+      "' s",
+      "Ġf or",
+      "Ġs h",
+      "Ġm om",
+      "itt le",
+      "ou ld",
+      "Ġl ittle",
+      "end oft",
+      "Ġs he",
+      "Ġ k",
+      "im e",
+      "c h",
+      ". \"",
+      "Ġn am",
+      "Ġt ime",
+      "Ġs m",
+      "Ġs o",
+      "ou nd",
+      "Ġthe re",
+      "Ġnam ed",
+      "Ġb o",
+      "Ġwe re",
+      "Ġwant ed",
+      "Ġb ut",
+      "v ed",
+      "Ġfriend s",
+      "ou t",
+      "h t",
+      "! \"",
+      "endoft ext",
+      "Ġn e",
+      "Ġa n",
+      "ir d",
+      "a l",
+      "Ġb ird",
+      "u e",
+      "ĠI t",
+      "Ġto o",
+      "Ġ Ċ",
+      "i l",
+      "Ġhe l",
+      "e l",
+      "id e",
+      "om e",
+      "Ġw ent",
+      "Ġw h",
+      "r y",
+      "ĠLi ly",
+      "ĠOn e",
+      "Ġ is",
+      "Ġa ll",
+      "Ġl o",
+      "a ke",
+      "Ġl oo",
+      "Ġup on",
+      "t er",
+      "ĠT om",
+      "u g",
+      "Ġhel p",
+      "o re",
+      "am e",
+      "Ġf un",
+      "i nd",
+      "Ġto y",
+      "Ġa s",
+      "g et",
+      "Ġ j",
+      "Ġa t",
+      "Ġ re",
+      "r a",
+      "get her",
+      "Ġ o",
+      "Ġs e",
+      "i ll",
+      "u r",
+      "Ġd id",
+      "Ġto gether",
+      "Ġb a",
+      "Ġc at",
+      "Ġt re",
+      "oo d",
+      "t ed",
+      "i c",
+      "m y",
+      "Ġdo g",
+      "Ġc an",
+      "e c",
+      "Ġc ould",
+      "Ġthe ir",
+      "ar d",
+      "he d",
+      "a x",
+      "Ġg ir",
+      "Ġh im",
+      "Ġr o",
+      "Ġplay ed",
+      "Ġb all",
+      "ar k",
+      "Ġk n",
+      "? \"",
+      "Ġgir l",
+      "ĠOn ce",
+      "w ay",
+      "Ġg o",
+      "Ġl e",
+      "Ġa re",
+      "Ġf r",
+      "Ġ W",
+      "Ġ out",
+      "a in",
+      "' t",
+      "Ġthe m",
+      "Ġsa d",
+      "u l",
+      "u m",
+      "Ġbo y",
+      "Ġtre e",
+      "Ġc l",
+      "Ġha ve",
+      "on e",
+      "Ġm an",
+      "he n",
+      "Ġloo ked",
+      "Ġlo ved",
+      "Ġf ound",
+      "ot her",
+      "Ġs p",
+      "ou g",
+      "Ġs c",
+      "Ġst ar",
+      "Ġbe c",
+      "h ing",
+      "Ġba ck",
+      "Ġ J",
+      "f ul",
+      "Ġl a",
+      "ow n",
+      "s ide",
+      "Ġm e",
+      "a re",
+      "Ġp ark",
+      "Ġc ar",
+      "on g",
+      "ig ht",
+      "el t",
+      "Ġw ould",
+      "o p",
+      "r ound",
+      "Ġf a",
+      "Ġli ke",
+      "Ġf elt",
+      "Ġse e",
+      "Ġn o",
+      "Ġ F",
+      "Ġas ked",
+      "om et",
+      "Ġne w",
+      "ĠS p",
+      "Ġstar ted",
+      "ar ed",
+      "Ġc ame",
+      "i ce",
+      "ou se",
+      "Ġa l",
+      "ĠB ut",
+      "i a",
+      "s s",
+      "Ġm ake",
+      "Ġs omet",
+      "oug ht",
+      "Ġ other",
+      "Ġa g",
+      "Ġb r",
+      "Ġg ood",
+      "Ġsm all",
+      "ĠS ue",
+      "ad e",
+      "o b",
+      "e ll",
+      "ing s",
+      "ri ed",
+      "ĠM ax",
+      "en ed",
+      "Ġwa l",
+      "Ġ ex",
+      "Ġw or",
+      "Ġc o",
+      "Ġf ind",
+      "a g",
+      "t y",
+      "Ġa way",
+      "Ġsomet hing",
+      "Ġwh at",
+      "Ġfr om",
+      "Ġm ade",
+      "Ġp ut",
+      "il ed",
+      "Ġh ome",
+      "Ġth ought",
+      "Ġplay ing",
+      "Ġs ay",
+      "Ġ Y",
+      "ĠB en",
+      "Ġm u",
+      "u c",
+      "a ch",
+      "ar n",
+      "Ġr an",
+      "Ġf l",
+      "i le",
+      "ĠS am",
+      "i e",
+      "Ġe very",
+      "n y",
+      "Ġag ain",
+      "oo k",
+      "d d",
+      "e w",
+      "Ġd own",
+      "Ġp r",
+      "Ġs ome",
+      "Ġtoo k",
+      "Ġsc ared",
+      "Ġli ked",
+      "Ġtoy s",
+      "k ing",
+      "Ġle arn",
+      "Ġh ouse",
+      "u re",
+      "Ġw ill",
+      "Ġbo x",
+      "e p",
+      "re t",
+      "i f",
+      "Ġb l",
+      "Ġm y",
+      "Ġth ings",
+      "Ġa round",
+      "b le",
+      "Ġyou r",
+      "Ġs w",
+      "is h",
+      "Ġw hen",
+      "ĠS o",
+      "Ġli ved",
+      "pp ed",
+      "Ġs un",
+      "Ġthe n",
+      "an k",
+      "ou d",
+      "Ġc h",
+      ", \"",
+      "Ġla ug",
+      "u s",
+      "i ck",
+      "T h",
+      "u ck",
+      "Ġ D",
+      "Ġt a",
+      "Ġg et",
+      "uc y",
+      "is t",
+      "Ġsm iled",
+      "Ġd ec",
+      "ĠL ucy",
+      "Ġf e",
+      "Ġt ried",
+      "ot h",
+      "i ly",
+      "Ġg ot",
+      "Ġkn ow",
+      "Ġwh o",
+      "a ve",
+      "q u",
+      "Ġa b",
+      "Ġman y",
+      "Ġan y",
+      "a u",
+      "au se",
+      "n a",
+      "Ġin t",
+      "it ed",
+      "Ġab out",
+      "ou s",
+      "ĠSp ot",
+      "Ġp ret",
+      "Ġl ot",
+      "Ġ v",
+      "er s",
+      "Ġr ed",
+      "Ġ E",
+      "is e",
+      "u st",
+      "Ġm ore",
+      "id ed",
+      "Ġp ic",
+      "nd er",
+      "Ġc are",
+      "a ce",
+      "Ġp o",
+      "a s",
+      "ur p",
+      "Ġlearn ed",
+      "Ġu n",
+      "Ġwa ter",
+      "Ġh ug",
+      "Ġex c",
+      "Ġbe st",
+      "um p",
+      "Ġo p",
+      "a p",
+      "Ġg re",
+      "way s",
+      "â Ģ",
+      "Ġout side",
+      "ĠM ia",
+      "f e",
+      "Ġal ways",
+      "an t",
+      "Ġro om",
+      "Ġh o",
+      "Ġint o",
+      "i ve",
+      "Ġe at",
+      "ĠM om",
+      "ĠA n",
+      "Ġp e",
+      "it e",
+      "Ġb oth",
+      "ĠF r",
+      "ĠY ou",
+      "Ġd ad",
+      "Ġ ke",
+      "all y",
+      "Ġloo k",
+      "Ġon e",
+      "Ġsh ow",
+      "Ġth is",
+      "Ġexc ited",
+      "Ġfa st",
+      "Ġn ice",
+      "Ġs urp",
+      "Ġr un",
+      "Ġl ong",
+      "Ġ C",
+      "Ġa m",
+      "f ter",
+      "al s",
+      "Ġs k",
+      "Ġj ump",
+      "Ġto ld",
+      "Ġfe el",
+      "Ġm o",
+      "Ġin side",
+      "Ġt r",
+      "ou r",
+      "u ll",
+      "Ġs l",
+      "Ġpret ty",
+      "in y",
+      "ĠB ob",
+      "Ġg ra",
+      "u dd",
+      "g ed",
+      "Ġe ach",
+      "Ġg ave",
+      "Ġmu ch",
+      "b b",
+      "Ġsay s",
+      "Ġst r",
+      "Ġh ow",
+      "in k",
+      "o g",
+      "Ġto w",
+      "Ġ or",
+      "ĠA s",
+      "le w",
+      "Ġne ed",
+      "et ter",
+      "Ġu nder",
+      "is ed",
+      "v en",
+      "Ġo ld",
+      "Ġro ck",
+      "Ġt ake",
+      "es s",
+      "Ġkn ew",
+      "r o",
+      "ĠT h",
+      "Ġc a",
+      "Ġc le",
+      "Ġf ish",
+      "he re",
+      "Ġl ist",
+      "Ġcl o",
+      "ur t",
+      "ĠA nd",
+      "Ġbe ar",
+      "c hed",
+      "Ġha nd",
+      "ar a",
+      "Ġt ry",
+      "Ġbec ame",
+      "Ġth an",
+      "ĠW e",
+      "Ġk ind",
+      "es t",
+      "ur n",
+      "Ġlaug hed",
+      "Ġj ust",
+      "or ry",
+      "Ġf ood",
+      "m ore",
+      "g e",
+      "Ġh ig",
+      "Ġf i",
+      "Ġwa t",
+      "ĠI n",
+      "in e",
+      "Ġsurp r",
+      "a k",
+      "Ġdec ided",
+      "Ġsk y",
+      "Ġbec ause",
+      "Ġ ide",
+      "Ġu s",
+      "v ing",
+      "Ġne ar",
+      "Ġt w",
+      "Ġide a",
+      "Ġb etter",
+      "Ġb ug",
+      "Ġhe ard",
+      "g ry",
+      "Ġit s",
+      "a se",
+      "ĠThe n",
+      "Ġ en",
+      "a ck",
+      "at e",
+      "W h",
+      "l f",
+      "Ġl et",
+      "or t",
+      "Ġ K",
+      "b y",
+      "Ġan im",
+      "ĠW hen",
+      "f f",
+      "Ġsh are",
+      "Ġcare ful",
+      "Ġco l",
+      "Ġ if",
+      "Ġt e",
+      "ec i",
+      "Ġg r",
+      "Ġf ly",
+      "Ġf o",
+      "Ġst or",
+      "Ġf lew",
+      "v es",
+      "Ġd an",
+      "Y es",
+      "Ġc om",
+      "eci al",
+      "Ġsp ecial",
+      "i on",
+      "Ġfl ow",
+      "Ġne ver",
+      "Ġb y",
+      "ec ted",
+      "Ġb u",
+      "Ġd on",
+      "Ġr a",
+      "Ġw ind",
+      "Ġt al",
+      "Ġcle an",
+      "r m",
+      "Ġ qu",
+      "Ġd re",
+      "ĠH is",
+      "Ġ end",
+      "Ġe ven",
+      "op le",
+      "Ġsa fe",
+      "Ġm ag",
+      "Ġun ex",
+      "Ġunex p",
+      "Ġsh iny",
+      "Ġh ard",
+      "Ġo ver",
+      "Ġb ook",
+      "ĠW h",
+      "Ġt urn",
+      "ĠS ara",
+      "Ġf am",
+      "Ġf ore",
+      "Ġb ad",
+      "Ġc ake",
+      "Ġa fter",
+      "ad y",
+      "Ġpe ople",
+      "Ġc u",
+      "p l",
+      "Ġ P",
+      "Ġpr oud",
+      "b er",
+      "Ġhig h",
+      "Ġg l",
+      "Ġc he",
+      "ĠAn na",
+      "ĠA my",
+      "Ġc ome",
+      "Ġg ard",
+      "Ġdo or",
+      "Ġop ened",
+      "Ġg round",
+      "Ġgard en",
+      "Ġpic ked",
+      "Ġ im",
+      "Ġh urt",
+      "Ġl oud",
+      "Ġ N",
+      "Ġst o",
+      "i ld",
+      "Ġs orry",
+      "Ġwa y",
+      "i p",
+      "' m",
+      "Ġbl ue",
+      "Ġst ill",
+      "Ġe ver",
+      "Ġhug ged",
+      "Ġf ar",
+      "Ġc all",
+      "Ġp a",
+      "Ġlo ve",
+      "ag e",
+      "he s",
+      "Ġof f",
+      "Ġmag ic",
+      "oug h",
+      "Ġwal ked",
+      "Ġs n",
+      "udd en",
+      "Ġp ar",
+      "Ġfam ily",
+      "Ġsh ould",
+      "oo l",
+      "b e",
+      "Ġcl im",
+      "i z",
+      "Ġk id",
+      "i es",
+      "d y",
+      "Ġm a",
+      "Ġn ow",
+      "Ġhapp ened",
+      "Ġgre at",
+      "ĠH er",
+      "Ġdid n",
+      "e m",
+      "Ġpl ace",
+      "Ġanim als",
+      "Ġstr ong",
+      "u ff",
+      "Ġu nt",
+      "udden ly",
+      "ĠJ o",
+      "Ġwal k",
+      "c t",
+      "Ġfore st",
+      "Ġb ra",
+      "N o",
+      "Ġclim b",
+      "a ut",
+      "ĠFr om",
+      "Ġg ive",
+      "Ġbr o",
+      "Ġw o",
+      "Ġs qu",
+      "Ġunt il",
+      "Ġbe aut",
+      "Ġa pp",
+      "ĠE very",
+      "Ġn ext",
+      "Ġfr og",
+      "Ġbo at",
+      "n ing",
+      "Ġst ick",
+      "Ġj o",
+      "Ġt ra",
+      "Ġim p",
+      "Ġpic t",
+      "Ġkid s",
+      "Ġst ay",
+      "Ġbeaut i",
+      "Ġbeauti ful",
+      "Ġclo s",
+      "Th ank",
+      "Ġ R",
+      "Ġd ra",
+      "Ġle a",
+      "Ġlist en",
+      "Ġstor y",
+      "Ġany more",
+      "um my",
+      "Ġwh ile",
+      "p t",
+      "re l",
+      "Ġre m",
+      "Ġtow n",
+      "an e",
+      "Ġc re",
+      "Ġlot s",
+      "ĠA t",
+      "k ay",
+      "Ġa d",
+      "ck s",
+      "ar y",
+      "Ġbe ing",
+      "Ġr ain",
+      "Ġm or",
+      "un ny",
+      "or s",
+      "Ġc ry",
+      "Ġcall ed",
+      "t h",
+      "a nd",
+      "Ġm et",
+      "le s",
+      "Ġg ame",
+      "Ġan gry",
+      "g er",
+      "Ġha t",
+      "le ase",
+      "Ġth re",
+      "se lf",
+      "Ġs oft",
+      "m a",
+      "uff y",
+      "Ġke ep",
+      "Ġb ed",
+      "Ġlo st",
+      "Ġtr uck",
+      "Ġop en",
+      "Ġwa rm",
+      "Ġm ouse",
+      "o on",
+      "Ġat e",
+      "I t",
+      "f ore",
+      "Ġbra ve",
+      "s o",
+      "Ġwat ch",
+      "Ġ G",
+      "Ġsurpr ised",
+      "L et",
+      "ec t",
+      "Ġal so",
+      "Ġf ell",
+      "Ġgre en",
+      "Ġfa ce",
+      "Ġtw o",
+      "ĠF l",
+      "Ġunexp ected",
+      "ĠK itt",
+      "Y ou",
+      "Ġpict ure",
+      "Ġ âĢ",
+      "v ent",
+      "Ġno ise",
+      "bb it",
+      "g an",
+      "ir st",
+      "Ġw here",
+      "l ed",
+      "Ġthan ked",
+      "Ġso on",
+      "k es",
+      "Ġsl ide",
+      "Ġsm ile",
+      "Ġho le",
+      "Ġp us",
+      "d ay",
+      "Ġbe fore",
+      "Ġra bbit",
+      "ir ed",
+      "Ġsa t",
+      "Ġdo ll",
+      "ĠFl uffy",
+      "ir rel",
+      "Ġke pt",
+      "ĠâĢ ľ",
+      "H i",
+      "âĢ Ŀ",
+      "is s",
+      "ow l",
+      "Ġf in",
+      "e ar",
+      "Ġsqu irrel",
+      "Ġjump ed",
+      "Ġbl o",
+      "Ġu se",
+      "a ble",
+      "Ġst ore",
+      "Ġhand s",
+      "Ġbird s",
+      "Ġb re",
+      "Ġp ain",
+      "Ġgo ing",
+      "Ġfun ny"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "added_tokens_decoder": {
+    "1024": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.19615549046784e+17,
+    "train_loss": 1.5581016430754755,
+    "train_runtime": 7493.9423,
+    "train_samples": 611606,
+    "train_samples_per_second": 81.613,
+    "train_steps_per_second": 2.55
+}

trainer_state.json

@@ -0,0 +1,479 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 1000,
+  "global_step": 19113,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.026160205096007954,
+      "grad_norm": 2.7528529167175293,
+      "learning_rate": 4.869198974519961e-05,
+      "loss": 4.403,
+      "step": 500
+    },
+    {
+      "epoch": 0.05232041019201591,
+      "grad_norm": 2.5874102115631104,
+      "learning_rate": 4.7383979490399206e-05,
+      "loss": 2.92,
+      "step": 1000
+    },
+    {
+      "epoch": 0.05232041019201591,
+      "eval_accuracy": 0.4470485610347474,
+      "eval_loss": 2.4565796852111816,
+      "eval_runtime": 53.659,
+      "eval_samples_per_second": 114.631,
+      "eval_steps_per_second": 3.597,
+      "step": 1000
+    },
+    {
+      "epoch": 0.07848061528802386,
+      "grad_norm": 2.103119134902954,
+      "learning_rate": 4.607596923559881e-05,
+      "loss": 2.2955,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10464082038403182,
+      "grad_norm": 2.2540347576141357,
+      "learning_rate": 4.476795898079841e-05,
+      "loss": 1.9818,
+      "step": 2000
+    },
+    {
+      "epoch": 0.10464082038403182,
+      "eval_accuracy": 0.5672200738882789,
+      "eval_loss": 1.8089067935943604,
+      "eval_runtime": 53.7114,
+      "eval_samples_per_second": 114.52,
+      "eval_steps_per_second": 3.593,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13080102548003977,
+      "grad_norm": 1.4521691799163818,
+      "learning_rate": 4.345994872599801e-05,
+      "loss": 1.83,
+      "step": 2500
+    },
+    {
+      "epoch": 0.15696123057604772,
+      "grad_norm": 1.3930741548538208,
+      "learning_rate": 4.2151938471197614e-05,
+      "loss": 1.7321,
+      "step": 3000
+    },
+    {
+      "epoch": 0.15696123057604772,
+      "eval_accuracy": 0.6020537553359386,
+      "eval_loss": 1.613312005996704,
+      "eval_runtime": 53.6022,
+      "eval_samples_per_second": 114.753,
+      "eval_steps_per_second": 3.601,
+      "step": 3000
+    },
+    {
+      "epoch": 0.18312143567205566,
+      "grad_norm": 1.4211032390594482,
+      "learning_rate": 4.084392821639722e-05,
+      "loss": 1.6624,
+      "step": 3500
+    },
+    {
+      "epoch": 0.20928164076806363,
+      "grad_norm": 1.275313138961792,
+      "learning_rate": 3.953591796159682e-05,
+      "loss": 1.6137,
+      "step": 4000
+    },
+    {
+      "epoch": 0.20928164076806363,
+      "eval_accuracy": 0.6195058763065014,
+      "eval_loss": 1.5170681476593018,
+      "eval_runtime": 53.3002,
+      "eval_samples_per_second": 115.403,
+      "eval_steps_per_second": 3.621,
+      "step": 4000
+    },
+    {
+      "epoch": 0.23544184586407158,
+      "grad_norm": 1.2738176584243774,
+      "learning_rate": 3.8227907706796424e-05,
+      "loss": 1.5721,
+      "step": 4500
+    },
+    {
+      "epoch": 0.26160205096007955,
+      "grad_norm": 1.2026641368865967,
+      "learning_rate": 3.691989745199603e-05,
+      "loss": 1.5353,
+      "step": 5000
+    },
+    {
+      "epoch": 0.26160205096007955,
+      "eval_accuracy": 0.6311744206133264,
+      "eval_loss": 1.4516141414642334,
+      "eval_runtime": 53.8002,
+      "eval_samples_per_second": 114.33,
+      "eval_steps_per_second": 3.587,
+      "step": 5000
+    },
+    {
+      "epoch": 0.28776225605608746,
+      "grad_norm": 1.1667968034744263,
+      "learning_rate": 3.561188719719563e-05,
+      "loss": 1.5091,
+      "step": 5500
+    },
+    {
+      "epoch": 0.31392246115209543,
+      "grad_norm": 1.4095014333724976,
+      "learning_rate": 3.430387694239523e-05,
+      "loss": 1.4845,
+      "step": 6000
+    },
+    {
+      "epoch": 0.31392246115209543,
+      "eval_accuracy": 0.6400102153795495,
+      "eval_loss": 1.4056029319763184,
+      "eval_runtime": 53.6423,
+      "eval_samples_per_second": 114.667,
+      "eval_steps_per_second": 3.598,
+      "step": 6000
+    },
+    {
+      "epoch": 0.3400826662481034,
+      "grad_norm": 1.2620683908462524,
+      "learning_rate": 3.299586668759483e-05,
+      "loss": 1.4606,
+      "step": 6500
+    },
+    {
+      "epoch": 0.3662428713441113,
+      "grad_norm": 1.2207545042037964,
+      "learning_rate": 3.168785643279443e-05,
+      "loss": 1.4443,
+      "step": 7000
+    },
+    {
+      "epoch": 0.3662428713441113,
+      "eval_accuracy": 0.646570593151532,
+      "eval_loss": 1.371826171875,
+      "eval_runtime": 53.6024,
+      "eval_samples_per_second": 114.752,
+      "eval_steps_per_second": 3.601,
+      "step": 7000
+    },
+    {
+      "epoch": 0.3924030764401193,
+      "grad_norm": 1.1445685625076294,
+      "learning_rate": 3.0379846177994036e-05,
+      "loss": 1.4255,
+      "step": 7500
+    },
+    {
+      "epoch": 0.41856328153612726,
+      "grad_norm": 1.1666637659072876,
+      "learning_rate": 2.907183592319364e-05,
+      "loss": 1.4118,
+      "step": 8000
+    },
+    {
+      "epoch": 0.41856328153612726,
+      "eval_accuracy": 0.6524860734404423,
+      "eval_loss": 1.3419677019119263,
+      "eval_runtime": 53.4631,
+      "eval_samples_per_second": 115.051,
+      "eval_steps_per_second": 3.61,
+      "step": 8000
+    },
+    {
+      "epoch": 0.4447234866321352,
+      "grad_norm": 1.2419095039367676,
+      "learning_rate": 2.776382566839324e-05,
+      "loss": 1.4015,
+      "step": 8500
+    },
+    {
+      "epoch": 0.47088369172814315,
+      "grad_norm": 1.1277480125427246,
+      "learning_rate": 2.6455815413592845e-05,
+      "loss": 1.3878,
+      "step": 9000
+    },
+    {
+      "epoch": 0.47088369172814315,
+      "eval_accuracy": 0.6569193066064805,
+      "eval_loss": 1.3188564777374268,
+      "eval_runtime": 53.2918,
+      "eval_samples_per_second": 115.421,
+      "eval_steps_per_second": 3.622,
+      "step": 9000
+    },
+    {
+      "epoch": 0.4970438968241511,
+      "grad_norm": 1.0969843864440918,
+      "learning_rate": 2.5147805158792447e-05,
+      "loss": 1.3773,
+      "step": 9500
+    },
+    {
+      "epoch": 0.5232041019201591,
+      "grad_norm": 1.1453758478164673,
+      "learning_rate": 2.383979490399205e-05,
+      "loss": 1.3661,
+      "step": 10000
+    },
+    {
+      "epoch": 0.5232041019201591,
+      "eval_accuracy": 0.6608493989564993,
+      "eval_loss": 1.2987806797027588,
+      "eval_runtime": 53.4371,
+      "eval_samples_per_second": 115.107,
+      "eval_steps_per_second": 3.612,
+      "step": 10000
+    },
+    {
+      "epoch": 0.549364307016167,
+      "grad_norm": 1.1494508981704712,
+      "learning_rate": 2.253178464919165e-05,
+      "loss": 1.3563,
+      "step": 10500
+    },
+    {
+      "epoch": 0.5755245121121749,
+      "grad_norm": 1.1540991067886353,
+      "learning_rate": 2.1223774394391253e-05,
+      "loss": 1.3485,
+      "step": 11000
+    },
+    {
+      "epoch": 0.5755245121121749,
+      "eval_accuracy": 0.6638841358556485,
+      "eval_loss": 1.2833938598632812,
+      "eval_runtime": 53.55,
+      "eval_samples_per_second": 114.865,
+      "eval_steps_per_second": 3.604,
+      "step": 11000
+    },
+    {
+      "epoch": 0.601684717208183,
+      "grad_norm": 1.1493678092956543,
+      "learning_rate": 1.9915764139590855e-05,
+      "loss": 1.3393,
+      "step": 11500
+    },
+    {
+      "epoch": 0.6278449223041909,
+      "grad_norm": 1.1292744874954224,
+      "learning_rate": 1.8607753884790457e-05,
+      "loss": 1.3326,
+      "step": 12000
+    },
+    {
+      "epoch": 0.6278449223041909,
+      "eval_accuracy": 0.6669099732330993,
+      "eval_loss": 1.2675042152404785,
+      "eval_runtime": 54.1653,
+      "eval_samples_per_second": 113.56,
+      "eval_steps_per_second": 3.563,
+      "step": 12000
+    },
+    {
+      "epoch": 0.6540051274001988,
+      "grad_norm": 1.1094636917114258,
+      "learning_rate": 1.729974362999006e-05,
+      "loss": 1.3249,
+      "step": 12500
+    },
+    {
+      "epoch": 0.6801653324962068,
+      "grad_norm": 1.1196491718292236,
+      "learning_rate": 1.5991733375189664e-05,
+      "loss": 1.319,
+      "step": 13000
+    },
+    {
+      "epoch": 0.6801653324962068,
+      "eval_accuracy": 0.6694385498356529,
+      "eval_loss": 1.2554900646209717,
+      "eval_runtime": 53.6683,
+      "eval_samples_per_second": 114.611,
+      "eval_steps_per_second": 3.596,
+      "step": 13000
+    },
+    {
+      "epoch": 0.7063255375922147,
+      "grad_norm": 1.1508065462112427,
+      "learning_rate": 1.4683723120389265e-05,
+      "loss": 1.3118,
+      "step": 13500
+    },
+    {
+      "epoch": 0.7324857426882226,
+      "grad_norm": 1.0860583782196045,
+      "learning_rate": 1.3375712865588865e-05,
+      "loss": 1.3068,
+      "step": 14000
+    },
+    {
+      "epoch": 0.7324857426882226,
+      "eval_accuracy": 0.6718871896629512,
+      "eval_loss": 1.2440038919448853,
+      "eval_runtime": 53.2979,
+      "eval_samples_per_second": 115.408,
+      "eval_steps_per_second": 3.621,
+      "step": 14000
+    },
+    {
+      "epoch": 0.7586459477842307,
+      "grad_norm": 1.126051902770996,
+      "learning_rate": 1.2067702610788469e-05,
+      "loss": 1.3014,
+      "step": 14500
+    },
+    {
+      "epoch": 0.7848061528802386,
+      "grad_norm": 1.100677490234375,
+      "learning_rate": 1.075969235598807e-05,
+      "loss": 1.2932,
+      "step": 15000
+    },
+    {
+      "epoch": 0.7848061528802386,
+      "eval_accuracy": 0.6737417864168825,
+      "eval_loss": 1.2349998950958252,
+      "eval_runtime": 53.6159,
+      "eval_samples_per_second": 114.723,
+      "eval_steps_per_second": 3.6,
+      "step": 15000
+    },
+    {
+      "epoch": 0.8109663579762465,
+      "grad_norm": 1.1026638746261597,
+      "learning_rate": 9.451682101187674e-06,
+      "loss": 1.2913,
+      "step": 15500
+    },
+    {
+      "epoch": 0.8371265630722545,
+      "grad_norm": 1.152256727218628,
+      "learning_rate": 8.143671846387275e-06,
+      "loss": 1.2868,
+      "step": 16000
+    },
+    {
+      "epoch": 0.8371265630722545,
+      "eval_accuracy": 0.6755100895943456,
+      "eval_loss": 1.2262605428695679,
+      "eval_runtime": 53.3362,
+      "eval_samples_per_second": 115.325,
+      "eval_steps_per_second": 3.619,
+      "step": 16000
+    },
+    {
+      "epoch": 0.8632867681682624,
+      "grad_norm": 1.1048023700714111,
+      "learning_rate": 6.835661591586878e-06,
+      "loss": 1.2809,
+      "step": 16500
+    },
+    {
+      "epoch": 0.8894469732642704,
+      "grad_norm": 1.1195833683013916,
+      "learning_rate": 5.527651336786481e-06,
+      "loss": 1.2791,
+      "step": 17000
+    },
+    {
+      "epoch": 0.8894469732642704,
+      "eval_accuracy": 0.6771291668633302,
+      "eval_loss": 1.219257116317749,
+      "eval_runtime": 53.5262,
+      "eval_samples_per_second": 114.916,
+      "eval_steps_per_second": 3.606,
+      "step": 17000
+    },
+    {
+      "epoch": 0.9156071783602784,
+      "grad_norm": 1.1175094842910767,
+      "learning_rate": 4.219641081986083e-06,
+      "loss": 1.2743,
+      "step": 17500
+    },
+    {
+      "epoch": 0.9417673834562863,
+      "grad_norm": 1.1169252395629883,
+      "learning_rate": 2.911630827185685e-06,
+      "loss": 1.2725,
+      "step": 18000
+    },
+    {
+      "epoch": 0.9417673834562863,
+      "eval_accuracy": 0.6780440694779302,
+      "eval_loss": 1.214146614074707,
+      "eval_runtime": 53.7753,
+      "eval_samples_per_second": 114.383,
+      "eval_steps_per_second": 3.589,
+      "step": 18000
+    },
+    {
+      "epoch": 0.9679275885522942,
+      "grad_norm": 1.1183288097381592,
+      "learning_rate": 1.6036205723852876e-06,
+      "loss": 1.2693,
+      "step": 18500
+    },
+    {
+      "epoch": 0.9940877936483022,
+      "grad_norm": 1.10493803024292,
+      "learning_rate": 2.956103175848899e-07,
+      "loss": 1.2711,
+      "step": 19000
+    },
+    {
+      "epoch": 0.9940877936483022,
+      "eval_accuracy": 0.6788121299845069,
+      "eval_loss": 1.2107646465301514,
+      "eval_runtime": 53.4355,
+      "eval_samples_per_second": 115.111,
+      "eval_steps_per_second": 3.612,
+      "step": 19000
+    },
+    {
+      "epoch": 1.0,
+      "step": 19113,
+      "total_flos": 3.19615549046784e+17,
+      "train_loss": 1.5581016430754755,
+      "train_runtime": 7493.9423,
+      "train_samples_per_second": 81.613,
+      "train_steps_per_second": 2.55
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 19113,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.19615549046784e+17,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9b51e06a31bb1c9757b6e1a4500835749b8cf1c45052dfc464fd94dd0134833
+size 5176