hardmaru cat-state commited on
Commit
21956a2
1 Parent(s): b097992

Upload tokenizer (#3)

Browse files

- Upload tokenizer (063c856bdba176a4341b6aedba759caa4669d5d3)


Co-authored-by: Full name <[email protected]>

Files changed (1) hide show
  1. tokenizer.json +27 -0
tokenizer.json CHANGED
@@ -227,6 +227,33 @@
227
  "rstrip": false,
228
  "normalized": true,
229
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
230
  }
231
  ],
232
  "normalizer": {
 
227
  "rstrip": false,
228
  "normalized": true,
229
  "special": false
230
+ },
231
+ {
232
+ "id": 50277,
233
+ "content": "<|SYSTEM|>",
234
+ "single_word": false,
235
+ "lstrip": false,
236
+ "rstrip": false,
237
+ "normalized": false,
238
+ "special": true
239
+ },
240
+ {
241
+ "id": 50278,
242
+ "content": "<|USER|>",
243
+ "single_word": false,
244
+ "lstrip": false,
245
+ "rstrip": false,
246
+ "normalized": false,
247
+ "special": true
248
+ },
249
+ {
250
+ "id": 50279,
251
+ "content": "<|ASSISTANT|>",
252
+ "single_word": false,
253
+ "lstrip": false,
254
+ "rstrip": false,
255
+ "normalized": false,
256
+ "special": true
257
  }
258
  ],
259
  "normalizer": {