{
"add_bos_token": true,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92352": {
"content": "Ｅ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92353": {
"content": "Ｆ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92354": {
"content": "Ｇ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92355": {
"content": "Ｈ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92356": {
"content": "Ｉ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92357": {
"content": "Ｊ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92358": {
"content": "Ｋ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92359": {
"content": "Ｌ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92360": {
"content": "Ｍ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92361": {
"content": "Ｎ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92362": {
"content": "Ｒ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92363": {
"content": "Ｕ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92364": {
"content": "Ｖ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92365": {
"content": "Ｗ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92366": {
"content": "Ｘ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92367": {
"content": "Ｙ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92368": {
"content": "Ｚ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92369": {
"content": "ａ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92370": {
"content": "ｂ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92371": {
"content": "ｃ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92372": {
"content": "ｄ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92373": {
"content": "ｅ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92374": {
"content": "ｆ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92375": {
"content": "ｇ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92376": {
"content": "ｈ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92377": {
"content": "ｉ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92378": {
"content": "ｊ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92379": {
"content": "ｋ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92380": {
"content": "ｌ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92381": {
"content": "ｍ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92382": {
"content": "ｎ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92383": {
"content": "ｏ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92384": {
"content": "ｐ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92385": {
"content": "ｑ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92386": {
"content": "ｒ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92387": {
"content": "ｓ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92388": {
"content": "ｔ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92389": {
"content": "ｕ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92390": {
"content": "ｖ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92391": {
"content": "ｗ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92392": {
"content": "ｘ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92393": {
"content": "ｙ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92394": {
"content": "ｚ",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92395": {
"content": "——",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92396": {
"content": "……",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92397": {
"content": "[UNUSED_TOKEN_0]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92398": {
"content": "[UNUSED_TOKEN_1]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92399": {
"content": "[UNUSED_TOKEN_2]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92400": {
"content": "[UNUSED_TOKEN_3]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92401": {
"content": "[UNUSED_TOKEN_4]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92402": {
"content": "[UNUSED_TOKEN_5]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92403": {
"content": "[UNUSED_TOKEN_6]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92404": {
"content": "[UNUSED_TOKEN_7]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92405": {
"content": "[UNUSED_TOKEN_8]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92406": {
"content": "[UNUSED_TOKEN_9]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92407": {
"content": "[UNUSED_TOKEN_10]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92408": {
"content": "[UNUSED_TOKEN_11]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92409": {
"content": "[UNUSED_TOKEN_12]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92410": {
"content": "[UNUSED_TOKEN_13]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92411": {
"content": "[UNUSED_TOKEN_14]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92412": {
"content": "[UNUSED_TOKEN_15]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92413": {
"content": "[UNUSED_TOKEN_16]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92414": {
"content": "[UNUSED_TOKEN_17]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92415": {
"content": "[UNUSED_TOKEN_18]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92416": {
"content": "[UNUSED_TOKEN_19]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92417": {
"content": "[UNUSED_TOKEN_20]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92418": {
"content": "[UNUSED_TOKEN_21]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92419": {
"content": "[UNUSED_TOKEN_22]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92420": {
"content": "[UNUSED_TOKEN_23]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92421": {
"content": "[UNUSED_TOKEN_24]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92422": {
"content": "[UNUSED_TOKEN_25]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92423": {
"content": "[UNUSED_TOKEN_26]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92424": {
"content": "[UNUSED_TOKEN_27]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92425": {
"content": "[UNUSED_TOKEN_28]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92426": {
"content": "[UNUSED_TOKEN_29]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92427": {
"content": "[UNUSED_TOKEN_30]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92428": {
"content": "[UNUSED_TOKEN_31]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92429": {
"content": "[UNUSED_TOKEN_32]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92430": {
"content": "[UNUSED_TOKEN_33]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92431": {
"content": "[UNUSED_TOKEN_34]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92432": {
"content": "[UNUSED_TOKEN_35]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92433": {
"content": "[UNUSED_TOKEN_36]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92434": {
"content": "[UNUSED_TOKEN_37]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92435": {
"content": "[UNUSED_TOKEN_38]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92436": {
"content": "[UNUSED_TOKEN_39]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92437": {
"content": "[UNUSED_TOKEN_40]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92438": {
"content": "[UNUSED_TOKEN_41]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92439": {
"content": "[UNUSED_TOKEN_42]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92440": {
"content": "[UNUSED_TOKEN_43]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92441": {
"content": "[UNUSED_TOKEN_44]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92442": {
"content": "[UNUSED_TOKEN_45]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92443": {
"content": "[UNUSED_TOKEN_46]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92444": {
"content": "[UNUSED_TOKEN_47]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92445": {
"content": "[UNUSED_TOKEN_48]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92446": {
"content": "[UNUSED_TOKEN_49]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92447": {
"content": "[UNUSED_TOKEN_50]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92448": {
"content": "[UNUSED_TOKEN_51]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92449": {
"content": "[UNUSED_TOKEN_52]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92450": {
"content": "[UNUSED_TOKEN_53]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92451": {
"content": "[UNUSED_TOKEN_54]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92452": {
"content": "[UNUSED_TOKEN_55]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92453": {
"content": "[UNUSED_TOKEN_56]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92454": {
"content": "[UNUSED_TOKEN_57]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92455": {
"content": "[UNUSED_TOKEN_58]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92456": {
"content": "[UNUSED_TOKEN_59]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92457": {
"content": "[UNUSED_TOKEN_60]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92458": {
"content": "[UNUSED_TOKEN_61]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92459": {
"content": "[UNUSED_TOKEN_62]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92460": {
"content": "[UNUSED_TOKEN_63]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92461": {
"content": "[UNUSED_TOKEN_64]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92462": {
"content": "[UNUSED_TOKEN_65]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92463": {
"content": "[UNUSED_TOKEN_66]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92464": {
"content": "[UNUSED_TOKEN_67]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92465": {
"content": "[UNUSED_TOKEN_68]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92466": {
"content": "[UNUSED_TOKEN_69]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92467": {
"content": "[UNUSED_TOKEN_70]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92468": {
"content": "[UNUSED_TOKEN_71]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92469": {
"content": "[UNUSED_TOKEN_72]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92470": {
"content": "[UNUSED_TOKEN_73]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92471": {
"content": "[UNUSED_TOKEN_74]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92472": {
"content": "[UNUSED_TOKEN_75]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92473": {
"content": "[UNUSED_TOKEN_76]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92474": {
"content": "[UNUSED_TOKEN_77]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92475": {
"content": "[UNUSED_TOKEN_78]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92476": {
"content": "[UNUSED_TOKEN_79]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92477": {
"content": "[UNUSED_TOKEN_80]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92478": {
"content": "[UNUSED_TOKEN_81]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92479": {
"content": "[UNUSED_TOKEN_82]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92480": {
"content": "[UNUSED_TOKEN_83]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92481": {
"content": "[UNUSED_TOKEN_84]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92482": {
"content": "[UNUSED_TOKEN_85]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92483": {
"content": "[UNUSED_TOKEN_86]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92484": {
"content": "[UNUSED_TOKEN_87]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92485": {
"content": "[UNUSED_TOKEN_88]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92486": {
"content": "[UNUSED_TOKEN_89]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92487": {
"content": "[UNUSED_TOKEN_90]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92488": {
"content": "[UNUSED_TOKEN_91]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92489": {
"content": "[UNUSED_TOKEN_92]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92490": {
"content": "[UNUSED_TOKEN_93]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92491": {
"content": "[UNUSED_TOKEN_94]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92492": {
"content": "[UNUSED_TOKEN_95]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92493": {
"content": "[UNUSED_TOKEN_96]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92494": {
"content": "[UNUSED_TOKEN_97]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92495": {
"content": "[UNUSED_TOKEN_98]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92496": {
"content": "[UNUSED_TOKEN_99]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92497": {
"content": "[UNUSED_TOKEN_100]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92498": {
"content": "[UNUSED_TOKEN_101]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92499": {
"content": "[UNUSED_TOKEN_102]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92500": {
"content": "[UNUSED_TOKEN_103]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92501": {
"content": "[UNUSED_TOKEN_104]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92502": {
"content": "[UNUSED_TOKEN_105]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92503": {
"content": "[UNUSED_TOKEN_106]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92504": {
"content": "[UNUSED_TOKEN_107]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92505": {
"content": "[UNUSED_TOKEN_108]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92506": {
"content": "[UNUSED_TOKEN_109]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92507": {
"content": "[UNUSED_TOKEN_110]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92508": {
"content": "[UNUSED_TOKEN_111]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92509": {
"content": "[UNUSED_TOKEN_112]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92510": {
"content": "[UNUSED_TOKEN_113]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92511": {
"content": "[UNUSED_TOKEN_114]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92512": {
"content": "[UNUSED_TOKEN_115]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92513": {
"content": "[UNUSED_TOKEN_116]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92514": {
"content": "[UNUSED_TOKEN_117]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92515": {
"content": "[UNUSED_TOKEN_118]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92516": {
"content": "[UNUSED_TOKEN_119]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92517": {
"content": "[UNUSED_TOKEN_120]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92518": {
"content": "[UNUSED_TOKEN_121]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92519": {
"content": "[UNUSED_TOKEN_122]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92520": {
"content": "[UNUSED_TOKEN_123]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92521": {
"content": "[UNUSED_TOKEN_124]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92522": {
"content": "[UNUSED_TOKEN_125]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92523": {
"content": "[UNUSED_TOKEN_126]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92524": {
"content": "[UNUSED_TOKEN_127]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92525": {
"content": "[UNUSED_TOKEN_128]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92526": {
"content": "[UNUSED_TOKEN_129]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92527": {
"content": "[UNUSED_TOKEN_130]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92528": {
"content": "[UNUSED_TOKEN_131]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92529": {
"content": "[UNUSED_TOKEN_132]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92530": {
"content": "[UNUSED_TOKEN_133]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92531": {
"content": "[UNUSED_TOKEN_134]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92532": {
"content": "[UNUSED_TOKEN_135]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92533": {
"content": "[UNUSED_TOKEN_136]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92534": {
"content": "[UNUSED_TOKEN_137]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92535": {
"content": "[UNUSED_TOKEN_138]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92536": {
"content": "[UNUSED_TOKEN_139]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92537": {
"content": "[UNUSED_TOKEN_140]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92538": {
"content": "<|plugin|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92539": {
"content": "<|interpreter|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92540": {
"content": "<|action_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92541": {
"content": "<|action_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92542": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92543": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"92544": {
"content": "[UNUSED_TOKEN_141]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92545": {
"content": "[UNUSED_TOKEN_142]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92546": {
"content": "[UNUSED_TOKEN_143]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92547": {
"content": "[UNUSED_TOKEN_144]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92548": {
"content": "[UNUSED_TOKEN_145]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
},
"92549": {
"content": "[UNUSED_TOKEN_146]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|action_start|>",
"<|action_end|>",
"<|interpreter|>",
"<|plugin|>"
],
"auto_map": {
"AutoTokenizer": [
"tokenization_internlm2.InternLM2Tokenizer",
"tokenization_internlm2_fast.InternLM2TokenizerFast"
]
},
"bos_token": "<s>",
"chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"decode_with_prefix_space": false,
"eos_token": "</s>",
"model_max_length": 1000000000000000019884624838656,
"pad_token": "</s>",
"sp_model_kwargs": null,
"tokenizer_class": "InternLM2Tokenizer",
"unk_token": "<unk>"
}