Upload tokenization_splinter.py
Browse files- tokenization_splinter.py +4 -1
tokenization_splinter.py
CHANGED
@@ -215,5 +215,8 @@ class SplinterBertTokenizerFast(BertTokenizerFast):
|
|
215 |
])
|
216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
217 |
|
218 |
-
def
|
|
|
|
|
|
|
219 |
print('Cannot save SplinterBertTokenizerFast, please copy the files directly from the repository')
|
|
|
215 |
])
|
216 |
self._tokenizer.decoder = Decoder.custom(SplinterDecoder(self.splinter))
|
217 |
|
218 |
+
def save_pretrained(self, *args, **kwargs):
|
219 |
+
self._save_pretrained(*args, **kwargs)
|
220 |
+
|
221 |
+
def _save_pretrained(self, *args, **kwargs):
|
222 |
print('Cannot save SplinterBertTokenizerFast, please copy the files directly from the repository')
|