Reihaneh commited on
Commit
bbcdfc3
·
verified ·
1 Parent(s): 0762dc1

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +41 -41
vocab.json CHANGED
@@ -1,47 +1,47 @@
1
  {
2
- "'": 31,
3
- "A": 20,
4
- "D": 16,
5
- "[": 1,
6
  "[DA]": 44,
7
  "[PAD]": 43,
8
  "[UNK]": 42,
9
- "]": 30,
10
- "a": 14,
11
- "b": 22,
12
- "c": 5,
13
- "d": 0,
14
- "e": 32,
15
- "f": 15,
16
- "g": 21,
17
- "h": 10,
18
- "i": 18,
19
- "j": 6,
20
- "k": 3,
21
- "l": 17,
22
- "m": 25,
23
- "n": 35,
24
- "o": 23,
25
- "p": 11,
26
- "q": 40,
27
- "r": 37,
28
- "s": 9,
29
- "t": 26,
30
- "u": 27,
31
- "v": 4,
32
- "w": 34,
33
- "x": 7,
34
- "y": 28,
35
- "z": 24,
36
- "|": 38,
37
- "«": 19,
38
- "»": 13,
39
- "å": 12,
40
- "æ": 2,
41
- "é": 36,
42
- "í": 41,
43
- "ó": 33,
44
- "ø": 39,
45
- "–": 29,
46
  "—": 8
47
  }
 
1
  {
2
+ "'": 19,
3
+ "A": 39,
4
+ "D": 21,
5
+ "[": 24,
6
  "[DA]": 44,
7
  "[PAD]": 43,
8
  "[UNK]": 42,
9
+ "]": 4,
10
+ "a": 28,
11
+ "b": 20,
12
+ "c": 31,
13
+ "d": 3,
14
+ "e": 9,
15
+ "f": 32,
16
+ "g": 23,
17
+ "h": 33,
18
+ "i": 2,
19
+ "j": 17,
20
+ "k": 7,
21
+ "l": 27,
22
+ "m": 40,
23
+ "n": 25,
24
+ "o": 18,
25
+ "p": 35,
26
+ "q": 16,
27
+ "r": 22,
28
+ "s": 14,
29
+ "t": 41,
30
+ "u": 34,
31
+ "v": 1,
32
+ "w": 12,
33
+ "x": 6,
34
+ "y": 38,
35
+ "z": 30,
36
+ "|": 26,
37
+ "«": 37,
38
+ "»": 5,
39
+ "å": 11,
40
+ "æ": 36,
41
+ "é": 10,
42
+ "í": 0,
43
+ "ó": 13,
44
+ "ø": 29,
45
+ "–": 15,
46
  "—": 8
47
  }