Upload 5 files

special_tokens_map.json

@@ -1,13 +1,13 @@
 {
   "bos_token": {
-    "content": "<|begin_of_text|>",
+    "content": "[gMASK]<sop>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|end_of_text|>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 128000,
-      "content": "<|begin_of_text|>",
+      "content": "[gMASK]<sop>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 128001,
-      "content": "<|end_of_text|>",
+      "content": "<|endoftext|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 128011,
-      "content": "<|reserved_special_token_3|>",
+      "content": "<|system|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 128012,
-      "content": "<|reserved_special_token_4|>",
+      "content": "<|assistant|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 128013,
-      "content": "<|reserved_special_token_5|>",
+      "content": "<|user|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,7 +131,7 @@
     },
     {
       "id": 128014,
-      "content": "<|reserved_special_token_6|>",
+      "content": "<|observation|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -2342,7 +2342,7 @@
         "single": [
           {
             "SpecialToken": {
-              "id": "<|begin_of_text|>",
+              "id": "[gMASK]<sop>",
               "type_id": 0
             }
           },
@@ -2356,7 +2356,7 @@
         "pair": [
           {
             "SpecialToken": {
-              "id": "<|begin_of_text|>",
+              "id": "[gMASK]<sop>",
               "type_id": 0
             }
           },
@@ -2368,7 +2368,7 @@
           },
           {
             "SpecialToken": {
-              "id": "<|begin_of_text|>",
+              "id": "[gMASK]<sop>",
               "type_id": 1
             }
           },
@@ -2380,13 +2380,13 @@
           }
         ],
         "special_tokens": {
-          "<|begin_of_text|>": {
-            "id": "<|begin_of_text|>",
+          "[gMASK]<sop>": {
+            "id": "[gMASK]<sop>",
             "ids": [
               128000
             ],
             "tokens": [
-              "<|begin_of_text|>"
+              "[gMASK]<sop>"
             ]
           }
         }

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "128000": {
-      "content": "<|begin_of_text|>",
+      "content": "[gMASK]<sop>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "128001": {
-      "content": "<|end_of_text|>",
+      "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -89,7 +89,7 @@
       "special": true
     },
     "128011": {
-      "content": "<|reserved_special_token_3|>",
+      "content": "<|system|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -97,7 +97,7 @@
       "special": true
     },
     "128012": {
-      "content": "<|reserved_special_token_4|>",
+      "content": "<|assistant|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -105,7 +105,7 @@
       "special": true
     },
     "128013": {
-      "content": "<|reserved_special_token_5|>",
+      "content": "<|user|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
       "special": true
     },
     "128014": {
-      "content": "<|reserved_special_token_6|>",
+      "content": "<|observation|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -2049,13 +2049,11 @@
       "special": true
     }
   },
-  "bos_token": "<|begin_of_text|>",
+  "bos_token": "[gMASK]<sop>",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ],
+  "eos_token": "<|endoftext|>",
+  "model_input_names": ["input_ids", "attention_mask"],
+  "chat_template": "{{ bos_token }}{%- set loop_messages = messages %}\n{%- for message in loop_messages %}\n    {%- set content = '<|' + message['role'] + '|>'+ message['content'] | trim %}\n    {%- if loop.index0 == 0 %}\n        {%- set content = content %}\n    {%- endif %}\n    {%- if not (loop.last and message['role'] == 'assistant') %}\n        {%- set content = content + '<|endoftext|>' %}\n    {%- endif %}\n    {{- content }}\n{%- endfor %}\n{%- if messages[-1]['role'] != 'assistant' %}\n  {{- '<|assistant|>' }}\n{%- endif %}",
   "model_max_length": 131072,
   "pad_token": "<|finetune_right_pad_id|>",
   "padding_side": "left",