Upload 7 files

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03927bb31bd78bdb4fca4b75c1d18766154ccf2e687de03a30e926c6fea8f784
+size 96709

token_fraction_describe.txt

@@ -0,0 +1,16 @@
+              token          text    token/text
+count  41573.000000  41573.000000  41573.000000
+mean      43.040579     91.721574      0.488299
+std       29.328293     69.693056      0.113421
+min        7.000000     41.000000      0.159420
+10%       20.000000     45.000000      0.360000
+25%       25.000000     51.000000      0.410667
+33%       28.000000     56.000000      0.430769
+50%       34.000000     67.000000      0.471264
+67%       43.000000     86.000000      0.518519
+75%       50.000000    103.000000      0.546218
+80%       56.000000    118.000000      0.569444
+90%       77.000000    170.000000      0.636364
+95%      100.000000    226.000000      0.704545
+99%      158.280000    371.000000      0.844444
+max      602.000000   1819.000000      1.086420

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5235": {
+      "content": "…",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6000": {
+      "content": "<0x{i:02X}>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "mask_token": "…",
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}