0x7o commited on
Commit
ceea28d
1 Parent(s): fdd141f

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -27
tokenizer.json CHANGED
@@ -2324,33 +2324,6 @@
2324
  "rstrip": false,
2325
  "normalized": false,
2326
  "special": true
2327
- },
2328
- {
2329
- "id": 128000,
2330
- "content": "<|user|>",
2331
- "single_word": false,
2332
- "lstrip": false,
2333
- "rstrip": false,
2334
- "normalized": false,
2335
- "special": true
2336
- },
2337
- {
2338
- "id": 128001,
2339
- "content": "<|system|>",
2340
- "single_word": false,
2341
- "lstrip": false,
2342
- "rstrip": false,
2343
- "normalized": false,
2344
- "special": true
2345
- },
2346
- {
2347
- "id": 128002,
2348
- "content": "<|assistant|>",
2349
- "single_word": false,
2350
- "lstrip": false,
2351
- "rstrip": false,
2352
- "normalized": false,
2353
- "special": true
2354
  }
2355
  ],
2356
  "normalizer": null,
@@ -2393,6 +2366,7 @@
2393
  "end_of_word_suffix": null,
2394
  "fuse_unk": false,
2395
  "byte_fallback": false,
 
2396
  "vocab": {
2397
  "<|begin_of_text|>": 0,
2398
  "<|end_of_text|>": 1,
 
2324
  "rstrip": false,
2325
  "normalized": false,
2326
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2327
  }
2328
  ],
2329
  "normalizer": null,
 
2366
  "end_of_word_suffix": null,
2367
  "fuse_unk": false,
2368
  "byte_fallback": false,
2369
+ "ignore_merges": true,
2370
  "vocab": {
2371
  "<|begin_of_text|>": 0,
2372
  "<|end_of_text|>": 1,