AnnyNguyen commited on
Commit
51ac3d6
·
verified ·
1 Parent(s): 2788e35

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -3
tokenizer_config.json CHANGED
@@ -32,10 +32,10 @@
32
  "single_word": false,
33
  "special": true
34
  },
35
- "15001": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
- "normalized": false,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
@@ -51,6 +51,6 @@
51
  "pad_token": "<pad>",
52
  "sep_token": "</s>",
53
  "sp_model_kwargs": {},
54
- "tokenizer_class": "XLMRobertaTokenizer",
55
  "unk_token": "<unk>"
56
  }
 
32
  "single_word": false,
33
  "special": true
34
  },
35
+ "40029": {
36
  "content": "<mask>",
37
  "lstrip": true,
38
+ "normalized": true,
39
  "rstrip": false,
40
  "single_word": false,
41
  "special": true
 
51
  "pad_token": "<pad>",
52
  "sep_token": "</s>",
53
  "sp_model_kwargs": {},
54
+ "tokenizer_class": "BartphoTokenizer",
55
  "unk_token": "<unk>"
56
  }