speed commited on
Commit
46258a9
·
verified ·
1 Parent(s): e7f11e7

Upload tokenizer

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
special_tokens_map.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|reserved_special_token_248|>",
4
+ "<|reserved_special_token_249|>",
5
+ "<|reserved_special_token_250|>",
6
+ "<|reserved_special_token_251|>",
7
+ "<|reserved_special_token_252|>",
8
+ "<|reserved_special_token_253|>",
9
+ "<|reserved_special_token_254|>",
10
+ "<|reserved_special_token_255|>",
11
+ "<|reserved_special_token_256|>",
12
+ "<|reserved_special_token_257|>",
13
+ "<|reserved_special_token_258|>",
14
+ "<|reserved_special_token_259|>",
15
+ "<|reserved_special_token_260|>",
16
+ "<|reserved_special_token_261|>",
17
+ "<|reserved_special_token_262|>",
18
+ "<|reserved_special_token_263|>",
19
+ "<|reserved_special_token_264|>",
20
+ "<|reserved_special_token_265|>",
21
+ "<|reserved_special_token_266|>",
22
+ "<|reserved_special_token_267|>",
23
+ "<|reserved_special_token_268|>",
24
+ "<|reserved_special_token_269|>",
25
+ "<|reserved_special_token_270|>",
26
+ "<|reserved_special_token_271|>",
27
+ "<|reserved_special_token_272|>",
28
+ "<|reserved_special_token_273|>",
29
+ "<|reserved_special_token_274|>",
30
+ "<|reserved_special_token_275|>",
31
+ "<|reserved_special_token_276|>",
32
+ "<|reserved_special_token_277|>"
33
+ ],
34
+ "bos_token": {
35
+ "content": "<|begin_of_text|>",
36
+ "lstrip": false,
37
+ "normalized": false,
38
+ "rstrip": false,
39
+ "single_word": false
40
+ },
41
+ "eos_token": {
42
+ "content": "<|end_of_text|>",
43
+ "lstrip": false,
44
+ "normalized": false,
45
+ "rstrip": false,
46
+ "single_word": false
47
+ },
48
+ "pad_token": "!"
49
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d62e27ffbfa24d9281f432757b1abca3e118b3dac7e1dce146f64aea37153e3
3
+ size 18727752
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff