File size: 4,137 Bytes
3f926e2 362d2d8 3f926e2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
{
"additional_special_tokens": [
"<|reserved_token_1|>",
"<|reserved_token_2|>",
"<|reserved_token_3|>",
"<|reserved_token_4|>",
"<|reserved_token_5|>",
"<|reserved_token_6|>",
"<|reserved_token_7|>",
"<|reserved_token_8|>",
"<|reserved_token_9|>",
"<|reserved_token_10|>",
"<|reserved_token_11|>",
"<|reserved_token_12|>",
"<|reserved_token_13|>",
"<|reserved_token_14|>",
"<|reserved_token_15|>",
"<|reserved_token_16|>",
"<|reserved_token_17|>",
"<|reserved_token_18|>",
"<|reserved_token_19|>",
"<|reserved_token_20|>",
"<|reserved_token_21|>",
"<|reserved_token_22|>",
"<|reserved_token_23|>",
"<|reserved_token_24|>",
"<|reserved_token_25|>",
"<|reserved_token_26|>",
"<|reserved_token_27|>",
"<|reserved_token_28|>",
"<|reserved_token_29|>",
"<|reserved_token_30|>",
"<|reserved_token_31|>",
"<|reserved_token_32|>",
"<|reserved_token_33|>",
"<|reserved_token_34|>",
"<|reserved_token_35|>",
"<|reserved_token_36|>",
"<|reserved_token_37|>",
"<|reserved_token_38|>",
"<|reserved_token_39|>",
"<|reserved_token_40|>",
"<|reserved_token_41|>",
"<|reserved_token_42|>",
"<|reserved_token_43|>",
"<|reserved_token_44|>",
"<|reserved_token_45|>",
"<|reserved_token_46|>",
"<|reserved_token_47|>",
"<|reserved_token_48|>",
"<|reserved_token_49|>",
"<|reserved_token_50|>",
"<|reserved_token_51|>",
"<|reserved_token_52|>",
"<|reserved_token_53|>",
"<|reserved_token_54|>",
"<|reserved_token_55|>",
"<|reserved_token_56|>",
"<|reserved_token_57|>",
"<|reserved_token_58|>",
"<|reserved_token_59|>",
"<|reserved_token_60|>",
"<|reserved_token_61|>",
"<|reserved_token_62|>",
"<|reserved_token_63|>",
"<|reserved_token_64|>",
"<|reserved_token_65|>",
"<|reserved_token_66|>",
"<|reserved_token_67|>",
"<|reserved_token_68|>",
"<|reserved_token_69|>",
"<|reserved_token_70|>",
"<|reserved_token_71|>",
"<|reserved_token_72|>",
"<|reserved_token_73|>",
"<|reserved_token_74|>",
"<|reserved_token_75|>",
"<|reserved_token_76|>",
"<|reserved_token_77|>",
"<|reserved_token_78|>",
"<|reserved_token_79|>",
"<|reserved_token_80|>",
"<|reserved_token_81|>",
"<|reserved_token_82|>",
"<|reserved_token_83|>",
"<|reserved_token_84|>",
"<|reserved_token_85|>",
"<|reserved_token_86|>",
"<|reserved_token_87|>",
"<|reserved_token_88|>",
"<|reserved_token_89|>",
"<|reserved_token_90|>",
"<|reserved_token_91|>",
"<|reserved_token_92|>",
"<|reserved_token_93|>",
"<|reserved_token_94|>",
"<|reserved_token_95|>",
"<|reserved_token_96|>",
"<|reserved_token_97|>",
"<|reserved_token_98|>",
"<|reserved_token_99|>",
"<|reserved_token_100|>",
"<|reserved_token_101|>",
"<|reserved_token_102|>",
"<|reserved_token_103|>",
"<|reserved_token_104|>",
"<|reserved_token_105|>",
"<|reserved_token_106|>",
"<|reserved_token_107|>",
"<|reserved_token_108|>",
"<|reserved_token_109|>",
"<|reserved_token_110|>",
"<|reserved_token_111|>",
"<|reserved_token_112|>",
"<|reserved_token_113|>",
"<|reserved_token_114|>",
"<|reserved_token_115|>",
"<|reserved_token_116|>",
"<|reserved_token_117|>",
"<|reserved_token_118|>",
"<|reserved_token_119|>",
"<|reserved_token_120|>",
"<|reserved_token_121|>"
],
"bos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": {
"content": "<|pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}
|