make `tokenizer_class` same as the actual class name (#6)
Browse files- make `tokenizer_class` same as the actual class name (c2feee1d9bac6f667935dcb5bd03ef3c79eae8d9)
Co-authored-by: Tomohiro Manabe <[email protected]>
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
@@ -17,7 +17,7 @@
|
|
17 |
"__type":"AddedToken"
|
18 |
},
|
19 |
"tokenize_chinese_chars":false,
|
20 |
-
"tokenizer_class": "
|
21 |
"word_tokenizer_type": "mecab",
|
22 |
"subword_tokenizer_type": "sentencepiece",
|
23 |
"mecab_kwargs": {
|
|
|
17 |
"__type":"AddedToken"
|
18 |
},
|
19 |
"tokenize_chinese_chars":false,
|
20 |
+
"tokenizer_class": "DistilBertJapaneseTokenizer",
|
21 |
"word_tokenizer_type": "mecab",
|
22 |
"subword_tokenizer_type": "sentencepiece",
|
23 |
"mecab_kwargs": {
|