Commit
·
d8411bd
1
Parent(s):
f0e19c5
Upload processor (#2)
Browse files- Upload processor (b634ee66a93a2f270ba8bfab680050a66dd096e4)
Co-authored-by: Sanchit Gandhi <[email protected]>
- special_tokens_map.json +1 -0
- tokenizer_config.json +2 -1
special_tokens_map.json
CHANGED
|
@@ -122,6 +122,7 @@
|
|
| 122 |
"rstrip": false,
|
| 123 |
"single_word": false
|
| 124 |
},
|
|
|
|
| 125 |
"unk_token": {
|
| 126 |
"content": "<|endoftext|>",
|
| 127 |
"lstrip": false,
|
|
|
|
| 122 |
"rstrip": false,
|
| 123 |
"single_word": false
|
| 124 |
},
|
| 125 |
+
"pad_token": "<|endoftext|>",
|
| 126 |
"unk_token": {
|
| 127 |
"content": "<|endoftext|>",
|
| 128 |
"lstrip": false,
|
tokenizer_config.json
CHANGED
|
@@ -12988,8 +12988,9 @@
|
|
| 12988 |
"eos_token": "<|endoftext|>",
|
| 12989 |
"errors": "replace",
|
| 12990 |
"model_max_length": 1000000000000000019884624838656,
|
| 12991 |
-
"pad_token":
|
| 12992 |
"processor_class": "WhisperProcessor",
|
| 12993 |
"tokenizer_class": "WhisperTokenizer",
|
|
|
|
| 12994 |
"unk_token": "<|endoftext|>"
|
| 12995 |
}
|
|
|
|
| 12988 |
"eos_token": "<|endoftext|>",
|
| 12989 |
"errors": "replace",
|
| 12990 |
"model_max_length": 1000000000000000019884624838656,
|
| 12991 |
+
"pad_token": "<|endoftext|>",
|
| 12992 |
"processor_class": "WhisperProcessor",
|
| 12993 |
"tokenizer_class": "WhisperTokenizer",
|
| 12994 |
+
"trust_remote_code": false,
|
| 12995 |
"unk_token": "<|endoftext|>"
|
| 12996 |
}
|