Upload with huggingface_hub

cctokenizer.py

@@ -1,12 +1,11 @@
-"""Tokenization classes for THUBert."""
-
-from typing import List, Optional, Tuple, Union
+"""Tokenization classes for ChineseCharTokenizer."""
 
+from typing import Optional, Tuple, Union
 from transformers import BertTokenizer
 import numpy as np
 import os
 import re
-
+import shutil
 
 # https://www.ling.upenn.edu/courses/Spring_2003/ling538/UnicodeRanges.html
 # https://www.microfocus.com/documentation/idol/IDOL/Servers/IDOLServer/11.2/Guides/html/English/expert/Content/IDOLExpert/Languages/Script_Ranges.htm
@@ -259,7 +258,8 @@ class ChineseCharTokenizer(BertTokenizer):
             [token for token in self.enclosed_tokens if len(token) == 6],
             [token for token in self.enclosed_tokens if len(token) == 7]
         ]
-        self.replace_map = load_json(os.path.join(os.path.dirname(vocab_file), 'replace.json'))
+        self.dir = os.path.join(os.path.dirname(vocab_file))
+        self.replace_map = load_json(os.path.join(self.dir, 'replace.json'))
 
     # # [EOS]相当于逗号、换行，不用看作special token
     def convert_token_to_representative(self, token: str) -> str:
@@ -267,8 +267,8 @@ class ChineseCharTokenizer(BertTokenizer):
         if token in self.vocab:
             return token
         else:
-            assert len(token) == 1
-            if re.match(r'\s', token): # 匹配\u2003, \t等
+            assert len(token) == 1, token
+            if re.match(r'\s', token):  # 匹配\u2003, \t等
                 return ' '
             v = ord(token)
             if _is_chinese_char(v):
@@ -279,6 +279,7 @@ class ChineseCharTokenizer(BertTokenizer):
             else:
                 return '[UNK]'
 
+    # bert的tokenize会加上CLS?
     def _tokenize(self, text):
         # 如果没有人为加的特殊赋好，可以不用这个tokenize，list(text)就是tokenize的结果
         split_tokens = []
@@ -299,24 +300,13 @@ class ChineseCharTokenizer(BertTokenizer):
         return split_tokens
 
     def _convert_token_to_id(self, token):
-        return self.vocab.get(token, self.convert_token_to_representative(token))
+        return self.vocab.get(self.convert_token_to_representative(token), self.vocab.get(self.unk_token))  # BUG: convert_token_to_representative 不是 id!
 
     def convert_tokens_to_string(self, tokens):
         return ''.join(tokens)
 
-
-if __name__ == '__main__':
-    tokenizer: ChineseCharTokenizer = ChineseCharTokenizer.from_pretrained("/data03/private/chengzhili/pretrain/bert/tokenizer/bert-base-chinese-char-cm")
-    for c in '\tｔ \nのᄌབོяا ㄞ∥∩①₁我🪐㓨 im Ok O走吧鏍𩐏':
-        print(c, tokenizer.convert_token_to_representative(c))
-    print(tokenizer.tokenize('ｔの 我བོя🪐㓨 im Ok  O[MASK][SEP]'))
-    print('')
-    print(tokenizer.enclosed_tokens_by_len)
-
-    text = 'བོ 我[MASK]я🪐'
-    tokens = tokenizer.tokenize(text)
-    print(tokens)  # བོ(藏语，两个unicode), 空格, 我, [MASK], я(俄语), 🪐
-    input_ids = tokenizer.convert_tokens_to_ids(tokens)
-    print(input_ids)  # [UNK],  , 我, [MASK], [U_RUS], [U_EMO]
-    inputs = tokenizer(text)
-    print(inputs)
+    def save_pretrained(self, save_directory: Union[str, os.PathLike], legacy_format: Optional[bool] = None, filename_prefix: Optional[str] = None, push_to_hub: bool = False, **kwargs) -> Tuple[str]:
+        ret = super().save_pretrained(save_directory, legacy_format, filename_prefix, push_to_hub, **kwargs)
+        shutil.copyfile(os.path.join(self.dir, 'replace.json'), f'{save_directory}/replace.json')
+        shutil.copyfile(os.path.join(self.dir, 'cctokenizer.py'), f'{save_directory}/cctokenizer.py')
+        return ret

config.json

@@ -1,4 +1,5 @@
 {
+  "_name_or_path": "output/2023-04-07_05-25-49/save/step_480000",
   "architectures": [
     "BertForMaskedLM"
   ],

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:914b38d56f0084daeb0388f7a3282b780f9fd199162188471cad63952678f77f
+oid sha256:6779545362cf64328e7074b88796295be61c620c2bbede8e03847c472f1add97
 size 382042873

special_tokens_map.json

@@ -1,5 +1,6 @@
 {
   "cls_token": "[CLS]",
+  "eos_token": "[EOS]",
   "mask_token": "[MASK]",
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",

tokenizer_config.json

@@ -1,10 +1,16 @@
 {
+  "auto_map": {
+    "AutoTokenizer": [
+      "cctokenizer.ChineseCharTokenizer",
+      null
+    ]
+  },
   "cls_token": "[CLS]",
   "do_basic_tokenize": true,
   "do_lower_case": true,
   "mask_token": "[MASK]",
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "chengzl18/bert-base-chinese-char-cm",
+  "name_or_path": "chengzl18/cctokenizer",
   "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
@@ -12,11 +18,5 @@
   "strip_accents": null,
   "tokenize_chinese_chars": true,
   "tokenizer_class": "ChineseCharTokenizer",
-  "auto_map": {
-    "AutoTokenizer": [
-      "cctokenizer.ChineseCharTokenizer",
-      null
-      ]
-  },
   "unk_token": "[UNK]"
 }