Hugging Face's logo

pretraining
/
pico-tokenizer-32k

Transformers
Model card Files
xet
Community
minpeter commited on
Commit
e788be0
·
verified ·
1 Parent(s): e67cb7f

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +0 -0
  2. tokenizer_config.json +11 -11
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -2,7 +2,7 @@
2
  "add_bos_token": false,
3
  "add_prefix_space": false,
4
  "added_tokens_decoder": {
5
- "15675": {
6
  "content": "<|boundary|>",
7
  "lstrip": false,
8
  "normalized": true,
@@ -10,7 +10,7 @@
10
  "single_word": false,
11
  "special": false
12
  },
13
- "15676": {
14
  "content": "<|im_start|>",
15
  "lstrip": false,
16
  "normalized": true,
@@ -18,7 +18,7 @@
18
  "single_word": false,
19
  "special": false
20
  },
21
- "15677": {
22
  "content": "<|im_end|>",
23
  "lstrip": false,
24
  "normalized": true,
@@ -26,7 +26,7 @@
26
  "single_word": false,
27
  "special": false
28
  },
29
- "15678": {
30
  "content": "<tool_call>",
31
  "lstrip": false,
32
  "normalized": false,
@@ -34,7 +34,7 @@
34
  "single_word": false,
35
  "special": false
36
  },
37
- "15679": {
38
  "content": "</tool_call>",
39
  "lstrip": false,
40
  "normalized": false,
@@ -42,7 +42,7 @@
42
  "single_word": false,
43
  "special": false
44
  },
45
- "15680": {
46
  "content": "<think>",
47
  "lstrip": false,
48
  "normalized": false,
@@ -50,7 +50,7 @@
50
  "single_word": false,
51
  "special": false
52
  },
53
- "15681": {
54
  "content": "</think>",
55
  "lstrip": false,
56
  "normalized": false,
@@ -58,7 +58,7 @@
58
  "single_word": false,
59
  "special": false
60
  },
61
- "15682": {
62
  "content": "<|unused_special_token_0|>",
63
  "lstrip": false,
64
  "normalized": false,
@@ -66,7 +66,7 @@
66
  "single_word": false,
67
  "special": true
68
  },
69
- "15683": {
70
  "content": "<|unused_special_token_1|>",
71
  "lstrip": false,
72
  "normalized": false,
@@ -74,7 +74,7 @@
74
  "single_word": false,
75
  "special": true
76
  },
77
- "15684": {
78
  "content": "<|unused_special_token_2|>",
79
  "lstrip": false,
80
  "normalized": false,
@@ -82,7 +82,7 @@
82
  "single_word": false,
83
  "special": true
84
  },
85
- "15685": {
86
  "content": "<|unused_special_token_3|>",
87
  "lstrip": false,
88
  "normalized": false,
 
2
  "add_bos_token": false,
3
  "add_prefix_space": false,
4
  "added_tokens_decoder": {
5
+ "31989": {
6
  "content": "<|boundary|>",
7
  "lstrip": false,
8
  "normalized": true,
 
10
  "single_word": false,
11
  "special": false
12
  },
13
+ "31990": {
14
  "content": "<|im_start|>",
15
  "lstrip": false,
16
  "normalized": true,
 
18
  "single_word": false,
19
  "special": false
20
  },
21
+ "31991": {
22
  "content": "<|im_end|>",
23
  "lstrip": false,
24
  "normalized": true,
 
26
  "single_word": false,
27
  "special": false
28
  },
29
+ "31992": {
30
  "content": "<tool_call>",
31
  "lstrip": false,
32
  "normalized": false,
 
34
  "single_word": false,
35
  "special": false
36
  },
37
+ "31993": {
38
  "content": "</tool_call>",
39
  "lstrip": false,
40
  "normalized": false,
 
42
  "single_word": false,
43
  "special": false
44
  },
45
+ "31994": {
46
  "content": "<think>",
47
  "lstrip": false,
48
  "normalized": false,
 
50
  "single_word": false,
51
  "special": false
52
  },
53
+ "31995": {
54
  "content": "</think>",
55
  "lstrip": false,
56
  "normalized": false,
 
58
  "single_word": false,
59
  "special": false
60
  },
61
+ "31996": {
62
  "content": "<|unused_special_token_0|>",
63
  "lstrip": false,
64
  "normalized": false,
 
66
  "single_word": false,
67
  "special": true
68
  },
69
+ "31997": {
70
  "content": "<|unused_special_token_1|>",
71
  "lstrip": false,
72
  "normalized": false,
 
74
  "single_word": false,
75
  "special": true
76
  },
77
+ "31998": {
78
  "content": "<|unused_special_token_2|>",
79
  "lstrip": false,
80
  "normalized": false,
 
82
  "single_word": false,
83
  "special": true
84
  },
85
+ "31999": {
86
  "content": "<|unused_special_token_3|>",
87
  "lstrip": false,
88
  "normalized": false,