JustinLin610 commited on
Commit
cbaac2a
1 Parent(s): 3e2a9bf

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. config.json +1 -1
  2. tokenizer.json +6 -10
config.json CHANGED
@@ -26,4 +26,4 @@
26
  "use_cache": true,
27
  "use_sliding_window": false,
28
  "vocab_size": 92416
29
- }
 
26
  "use_cache": true,
27
  "use_sliding_window": false,
28
  "vocab_size": 92416
29
+ }
tokenizer.json CHANGED
@@ -12556,21 +12556,17 @@
12556
  }
12557
  ]
12558
  },
12559
- "pre_tokenizer": {
12560
- "type": "Metaspace",
12561
- "replacement": "▁",
12562
- "add_prefix_space": false,
12563
- "prepend_scheme": "first"
12564
- },
12565
  "post_processor": null,
12566
  "decoder": {
12567
  "type": "Sequence",
12568
  "decoders": [
12569
  {
12570
- "type": "Metaspace",
12571
- "replacement": "▁",
12572
- "add_prefix_space": false,
12573
- "prepend_scheme": "first"
 
12574
  },
12575
  {
12576
  "type": "ByteFallback"
 
12556
  }
12557
  ]
12558
  },
12559
+ "pre_tokenizer": null,
 
 
 
 
 
12560
  "post_processor": null,
12561
  "decoder": {
12562
  "type": "Sequence",
12563
  "decoders": [
12564
  {
12565
+ "type": "Replace",
12566
+ "pattern": {
12567
+ "String": "▁"
12568
+ },
12569
+ "content": " "
12570
  },
12571
  {
12572
  "type": "ByteFallback"