Xenova HF Staff commited on
Commit
ec9d1a8
·
verified ·
1 Parent(s): 69145ca

Upload 2 files

Browse files
Files changed (2) hide show
  1. tokenizer.json +1 -1
  2. tokenizer_config.json +6 -6
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:74bc832c094b6170fc19e50feec75b3c026d5b0f1102c6357550799e2f312353
3
  size 12603078
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e18748766f9c939f2ec40506e922b46e89f5aef8bc6ce13ec12b1a11c597550
3
  size 12603078
tokenizer_config.json CHANGED
@@ -140,7 +140,7 @@
140
  "special": true
141
  },
142
  "17": {
143
- "content": "<SPECIAL_17>",
144
  "lstrip": false,
145
  "normalized": false,
146
  "rstrip": false,
@@ -148,7 +148,7 @@
148
  "special": true
149
  },
150
  "18": {
151
- "content": "<SPECIAL_18>",
152
  "lstrip": false,
153
  "normalized": false,
154
  "rstrip": false,
@@ -156,7 +156,7 @@
156
  "special": true
157
  },
158
  "19": {
159
- "content": "<SPECIAL_19>",
160
  "lstrip": false,
161
  "normalized": false,
162
  "rstrip": false,
@@ -196,7 +196,7 @@
196
  "special": true
197
  },
198
  "24": {
199
- "content": "<SPECIAL_24>",
200
  "lstrip": false,
201
  "normalized": false,
202
  "rstrip": false,
@@ -204,7 +204,7 @@
204
  "special": true
205
  },
206
  "25": {
207
- "content": "<SPECIAL_25>",
208
  "lstrip": false,
209
  "normalized": false,
210
  "rstrip": false,
@@ -276,7 +276,7 @@
276
  "special": true
277
  },
278
  "34": {
279
- "content": "<SPECIAL_34>",
280
  "lstrip": false,
281
  "normalized": false,
282
  "rstrip": false,
 
140
  "special": true
141
  },
142
  "17": {
143
+ "content": "[SYSTEM_PROMPT]",
144
  "lstrip": false,
145
  "normalized": false,
146
  "rstrip": false,
 
148
  "special": true
149
  },
150
  "18": {
151
+ "content": "[/SYSTEM_PROMPT]",
152
  "lstrip": false,
153
  "normalized": false,
154
  "rstrip": false,
 
156
  "special": true
157
  },
158
  "19": {
159
+ "content": "[TOOL_CONTENT]",
160
  "lstrip": false,
161
  "normalized": false,
162
  "rstrip": false,
 
196
  "special": true
197
  },
198
  "24": {
199
+ "content": "[AUDIO]",
200
  "lstrip": false,
201
  "normalized": false,
202
  "rstrip": false,
 
204
  "special": true
205
  },
206
  "25": {
207
+ "content": "[BEGIN_AUDIO]",
208
  "lstrip": false,
209
  "normalized": false,
210
  "rstrip": false,
 
276
  "special": true
277
  },
278
  "34": {
279
+ "content": "[TRANSCRIBE]",
280
  "lstrip": false,
281
  "normalized": false,
282
  "rstrip": false,