Commit
•
848ffdc
1
Parent(s):
530025d
model improved
Browse files- tokenizer.json +14 -0
tokenizer.json
CHANGED
|
@@ -784,6 +784,20 @@
|
|
| 784 |
},
|
| 785 |
"content": ","
|
| 786 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 787 |
{
|
| 788 |
"type": "Replace",
|
| 789 |
"pattern": {
|
|
|
|
| 784 |
},
|
| 785 |
"content": ","
|
| 786 |
},
|
| 787 |
+
{
|
| 788 |
+
"type": "Replace",
|
| 789 |
+
"pattern": {
|
| 790 |
+
"String": "“"
|
| 791 |
+
},
|
| 792 |
+
"content": "\""
|
| 793 |
+
},
|
| 794 |
+
{
|
| 795 |
+
"type": "Replace",
|
| 796 |
+
"pattern": {
|
| 797 |
+
"String": "”"
|
| 798 |
+
},
|
| 799 |
+
"content": "\""
|
| 800 |
+
},
|
| 801 |
{
|
| 802 |
"type": "Replace",
|
| 803 |
"pattern": {
|