add tokenizer

runs/Oct19_06-28-56_0e4cdf942af6/events.out.tfevents.1666161780.0e4cdf942af6.9735.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:99255eb4180c03e175a45086d34bc34860285e10b92d04b1038811f9b28fa306
-size 5217
+oid sha256:53abfe94201314a371ce848b8d9f8493e11f6cb9d7e92d44a150b99824ba2979
+size 9206

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "tokenizer_class": "Wav2Vec2CTCTokenizer"}

vocab.json

@@ -0,0 +1 @@
+{"\n": 0, "0": 2, "1": 3, "2": 4, "3": 5, "4": 6, "5": 7, "6": 8, "7": 9, "8": 10, "9": 11, "a": 12, "b": 13, "c": 14, "h": 15, "l": 16, "m": 17, "n": 18, "o": 19, "p": 20, "s": 21, "t": 22, "z": 23, "ㄱ": 24, "ㄲ": 25, "ㄴ": 26, "ㄵ": 27, "ㄶ": 28, "ㄷ": 29, "ㄸ": 30, "ㄹ": 31, "ㄺ": 32, "ㄻ": 33, "ㄼ": 34, "ㄾ": 35, "ㅀ": 36, "ㅁ": 37, "ㅂ": 38, "ㅃ": 39, "ㅄ": 40, "ㅅ": 41, "ㅆ": 42, "ㅇ": 43, "ㅈ": 44, "ㅉ": 45, "ㅊ": 46, "ㅋ": 47, "ㅌ": 48, "ㅍ": 49, "ㅎ": 50, "ㅏ": 51, "ㅐ": 52, "ㅑ": 53, "ㅒ": 54, "ㅓ": 55, "ㅔ": 56, "ㅕ": 57, "ㅖ": 58, "ㅗ": 59, "ㅘ": 60, "ㅙ": 61, "ㅚ": 62, "ㅛ": 63, "ㅜ": 64, "ㅝ": 65, "ㅞ": 66, "ㅟ": 67, "ㅠ": 68, "ㅡ": 69, "ㅢ": 70, "ㅣ": 71, "，": 72, "|": 1, "[UNK]": 73, "[PAD]": 74}