{ "add_bos_token": true, "add_eos_token": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92352": { "content": "Ｅ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92353": { "content": "Ｆ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92354": { "content": "Ｇ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92355": { "content": "Ｈ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92356": { "content": "Ｉ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92357": { "content": "Ｊ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92358": { "content": "Ｋ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92359": { "content": "Ｌ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92360": { "content": "Ｍ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92361": { "content": "Ｎ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92362": { "content": "Ｒ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92363": { "content": "Ｕ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92364": { "content": "Ｖ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92365": { "content": "Ｗ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92366": { "content": "Ｘ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92367": { "content": "Ｙ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92368": { "content": "Ｚ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92369": { "content": "ａ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92370": { "content": "ｂ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92371": { "content": "ｃ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92372": { "content": "ｄ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92373": { "content": "ｅ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92374": { "content": "ｆ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92375": { "content": "ｇ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92376": { "content": "ｈ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92377": { "content": "ｉ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92378": { "content": "ｊ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92379": { "content": "ｋ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92380": { "content": "ｌ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92381": { "content": "ｍ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92382": { "content": "ｎ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92383": { "content": "ｏ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92384": { "content": "ｐ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92385": { "content": "ｑ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92386": { "content": "ｒ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92387": { "content": "ｓ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92388": { "content": "ｔ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92389": { "content": "ｕ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92390": { "content": "ｖ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92391": { "content": "ｗ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92392": { "content": "ｘ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92393": { "content": "ｙ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92394": { "content": "ｚ", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92395": { "content": "——", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92396": { "content": "……", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92397": { "content": "[UNUSED_TOKEN_0]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92398": { "content": "[UNUSED_TOKEN_1]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92399": { "content": "[UNUSED_TOKEN_2]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92400": { "content": "[UNUSED_TOKEN_3]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92401": { "content": "[UNUSED_TOKEN_4]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92402": { "content": "[UNUSED_TOKEN_5]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92403": { "content": "[UNUSED_TOKEN_6]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92404": { "content": "[UNUSED_TOKEN_7]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92405": { "content": "[UNUSED_TOKEN_8]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92406": { "content": "[UNUSED_TOKEN_9]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92407": { "content": "[UNUSED_TOKEN_10]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92408": { "content": "[UNUSED_TOKEN_11]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92409": { "content": "[UNUSED_TOKEN_12]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92410": { "content": "[UNUSED_TOKEN_13]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92411": { "content": "[UNUSED_TOKEN_14]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92412": { "content": "[UNUSED_TOKEN_15]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92413": { "content": "[UNUSED_TOKEN_16]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92414": { "content": "[UNUSED_TOKEN_17]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92415": { "content": "[UNUSED_TOKEN_18]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92416": { "content": "[UNUSED_TOKEN_19]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92417": { "content": "[UNUSED_TOKEN_20]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92418": { "content": "[UNUSED_TOKEN_21]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92419": { "content": "[UNUSED_TOKEN_22]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92420": { "content": "[UNUSED_TOKEN_23]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92421": { "content": "[UNUSED_TOKEN_24]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92422": { "content": "[UNUSED_TOKEN_25]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92423": { "content": "[UNUSED_TOKEN_26]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92424": { "content": "[UNUSED_TOKEN_27]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92425": { "content": "[UNUSED_TOKEN_28]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92426": { "content": "[UNUSED_TOKEN_29]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92427": { "content": "[UNUSED_TOKEN_30]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92428": { "content": "[UNUSED_TOKEN_31]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92429": { "content": "[UNUSED_TOKEN_32]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92430": { "content": "[UNUSED_TOKEN_33]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92431": { "content": "[UNUSED_TOKEN_34]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92432": { "content": "[UNUSED_TOKEN_35]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92433": { "content": "[UNUSED_TOKEN_36]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92434": { "content": "[UNUSED_TOKEN_37]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92435": { "content": "[UNUSED_TOKEN_38]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92436": { "content": "[UNUSED_TOKEN_39]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92437": { "content": "[UNUSED_TOKEN_40]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92438": { "content": "[UNUSED_TOKEN_41]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92439": { "content": "[UNUSED_TOKEN_42]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92440": { "content": "[UNUSED_TOKEN_43]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92441": { "content": "[UNUSED_TOKEN_44]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92442": { "content": "[UNUSED_TOKEN_45]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92443": { "content": "[UNUSED_TOKEN_46]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92444": { "content": "[UNUSED_TOKEN_47]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92445": { "content": "[UNUSED_TOKEN_48]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92446": { "content": "[UNUSED_TOKEN_49]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92447": { "content": "[UNUSED_TOKEN_50]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92448": { "content": "[UNUSED_TOKEN_51]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92449": { "content": "[UNUSED_TOKEN_52]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92450": { "content": "[UNUSED_TOKEN_53]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92451": { "content": "[UNUSED_TOKEN_54]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92452": { "content": "[UNUSED_TOKEN_55]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92453": { "content": "[UNUSED_TOKEN_56]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92454": { "content": "[UNUSED_TOKEN_57]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92455": { "content": "[UNUSED_TOKEN_58]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92456": { "content": "[UNUSED_TOKEN_59]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92457": { "content": "[UNUSED_TOKEN_60]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92458": { "content": "[UNUSED_TOKEN_61]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92459": { "content": "[UNUSED_TOKEN_62]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92460": { "content": "[UNUSED_TOKEN_63]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92461": { "content": "[UNUSED_TOKEN_64]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92462": { "content": "[UNUSED_TOKEN_65]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92463": { "content": "[UNUSED_TOKEN_66]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92464": { "content": "[UNUSED_TOKEN_67]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92465": { "content": "[UNUSED_TOKEN_68]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92466": { "content": "[UNUSED_TOKEN_69]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92467": { "content": "[UNUSED_TOKEN_70]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92468": { "content": "[UNUSED_TOKEN_71]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92469": { "content": "[UNUSED_TOKEN_72]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92470": { "content": "[UNUSED_TOKEN_73]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92471": { "content": "[UNUSED_TOKEN_74]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92472": { "content": "[UNUSED_TOKEN_75]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92473": { "content": "[UNUSED_TOKEN_76]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92474": { "content": "[UNUSED_TOKEN_77]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92475": { "content": "[UNUSED_TOKEN_78]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92476": { "content": "[UNUSED_TOKEN_79]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92477": { "content": "[UNUSED_TOKEN_80]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92478": { "content": "[UNUSED_TOKEN_81]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92479": { "content": "[UNUSED_TOKEN_82]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92480": { "content": "[UNUSED_TOKEN_83]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92481": { "content": "[UNUSED_TOKEN_84]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92482": { "content": "[UNUSED_TOKEN_85]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92483": { "content": "[UNUSED_TOKEN_86]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92484": { "content": "[UNUSED_TOKEN_87]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92485": { "content": "[UNUSED_TOKEN_88]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92486": { "content": "[UNUSED_TOKEN_89]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92487": { "content": "[UNUSED_TOKEN_90]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92488": { "content": "[UNUSED_TOKEN_91]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92489": { "content": "[UNUSED_TOKEN_92]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92490": { "content": "[UNUSED_TOKEN_93]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92491": { "content": "[UNUSED_TOKEN_94]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92492": { "content": "[UNUSED_TOKEN_95]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92493": { "content": "[UNUSED_TOKEN_96]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92494": { "content": "[UNUSED_TOKEN_97]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92495": { "content": "[UNUSED_TOKEN_98]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92496": { "content": "[UNUSED_TOKEN_99]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92497": { "content": "[UNUSED_TOKEN_100]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92498": { "content": "[UNUSED_TOKEN_101]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92499": { "content": "[UNUSED_TOKEN_102]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92500": { "content": "[UNUSED_TOKEN_103]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92501": { "content": "[UNUSED_TOKEN_104]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92502": { "content": "[UNUSED_TOKEN_105]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92503": { "content": "[UNUSED_TOKEN_106]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92504": { "content": "[UNUSED_TOKEN_107]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92505": { "content": "[UNUSED_TOKEN_108]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92506": { "content": "[UNUSED_TOKEN_109]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92507": { "content": "[UNUSED_TOKEN_110]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92508": { "content": "[UNUSED_TOKEN_111]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92509": { "content": "[UNUSED_TOKEN_112]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92510": { "content": "[UNUSED_TOKEN_113]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92511": { "content": "[UNUSED_TOKEN_114]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92512": { "content": "[UNUSED_TOKEN_115]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92513": { "content": "[UNUSED_TOKEN_116]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92514": { "content": "[UNUSED_TOKEN_117]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92515": { "content": "[UNUSED_TOKEN_118]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92516": { "content": "[UNUSED_TOKEN_119]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92517": { "content": "[UNUSED_TOKEN_120]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92518": { "content": "[UNUSED_TOKEN_121]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92519": { "content": "[UNUSED_TOKEN_122]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92520": { "content": "[UNUSED_TOKEN_123]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92521": { "content": "[UNUSED_TOKEN_124]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92522": { "content": "[UNUSED_TOKEN_125]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92523": { "content": "[UNUSED_TOKEN_126]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92524": { "content": "[UNUSED_TOKEN_127]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92525": { "content": "[UNUSED_TOKEN_128]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92526": { "content": "[UNUSED_TOKEN_129]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92527": { "content": "[UNUSED_TOKEN_130]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92528": { "content": "[UNUSED_TOKEN_131]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92529": { "content": "[UNUSED_TOKEN_132]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92530": { "content": "[UNUSED_TOKEN_133]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92531": { "content": "[UNUSED_TOKEN_134]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92532": { "content": "[UNUSED_TOKEN_135]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92533": { "content": "[UNUSED_TOKEN_136]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92534": { "content": "[UNUSED_TOKEN_137]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92535": { "content": "[UNUSED_TOKEN_138]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92536": { "content": "[UNUSED_TOKEN_139]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92537": { "content": "[UNUSED_TOKEN_140]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92538": { "content": "<|plugin|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92539": { "content": "<|interpreter|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92540": { "content": "<|action_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92541": { "content": "<|action_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92542": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92543": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "92544": { "content": "[UNUSED_TOKEN_141]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92545": { "content": "[UNUSED_TOKEN_142]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92546": { "content": "[UNUSED_TOKEN_143]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92547": { "content": "[UNUSED_TOKEN_144]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92548": { "content": "[UNUSED_TOKEN_145]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "92549": { "content": "[UNUSED_TOKEN_146]", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false } }, "additional_special_tokens": [ "<|im_start|>", "<|im_end|>", "<|action_start|>", "<|action_end|>", "<|interpreter|>", "<|plugin|>" ], "auto_map": { "AutoTokenizer": [ "tokenization_internlm2.InternLM2Tokenizer", "tokenization_internlm2_fast.InternLM2TokenizerFast" ] }, "bos_token": "", "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", "clean_up_tokenization_spaces": false, "decode_with_prefix_space": false, "eos_token": "", "max_length": 869, "model_max_length": 1000000000000000019884624838656, "pad_token": "", "sp_model_kwargs": null, "stride": 0, "tokenizer_class": "InternLM2Tokenizer", "truncation_side": "right", "truncation_strategy": "longest_first", "unk_token": "" }