fix lstrip on mask token
Browse files- special_tokens_map.json +2 -2
- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
special_tokens_map.json
CHANGED
@@ -22,9 +22,9 @@
|
|
22 |
},
|
23 |
"mask_token": {
|
24 |
"content": "<mask>",
|
25 |
-
"lstrip":
|
26 |
"normalized": true,
|
27 |
-
"rstrip":
|
28 |
"single_word": false
|
29 |
},
|
30 |
"pad_token": {
|
|
|
22 |
},
|
23 |
"mask_token": {
|
24 |
"content": "<mask>",
|
25 |
+
"lstrip": true,
|
26 |
"normalized": true,
|
27 |
+
"rstrip": false,
|
28 |
"single_word": false
|
29 |
},
|
30 |
"pad_token": {
|
tokenizer.json
CHANGED
@@ -86,8 +86,8 @@
|
|
86 |
"id": 65003,
|
87 |
"content": "<mask>",
|
88 |
"single_word": false,
|
89 |
-
"lstrip":
|
90 |
-
"rstrip":
|
91 |
"normalized": true,
|
92 |
"special": true
|
93 |
}
|
|
|
86 |
"id": 65003,
|
87 |
"content": "<mask>",
|
88 |
"single_word": false,
|
89 |
+
"lstrip": true,
|
90 |
+
"rstrip": false,
|
91 |
"normalized": true,
|
92 |
"special": true
|
93 |
}
|
tokenizer_config.json
CHANGED
@@ -67,9 +67,9 @@
|
|
67 |
},
|
68 |
"65003": {
|
69 |
"content": "<mask>",
|
70 |
-
"lstrip":
|
71 |
"normalized": true,
|
72 |
-
"rstrip":
|
73 |
"single_word": false,
|
74 |
"special": true
|
75 |
}
|
|
|
67 |
},
|
68 |
"65003": {
|
69 |
"content": "<mask>",
|
70 |
+
"lstrip": true,
|
71 |
"normalized": true,
|
72 |
+
"rstrip": false,
|
73 |
"single_word": false,
|
74 |
"special": true
|
75 |
}
|