Reihaneh commited on
Commit
3c9c8e3
·
verified ·
1 Parent(s): f4e6e57

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +58 -58
vocab.json CHANGED
@@ -1,65 +1,65 @@
1
  {
2
- "'": 44,
3
- "-": 55,
4
- "E": 3,
5
- "F": 43,
6
- "L": 31,
7
- "N": 6,
8
- "S": 2,
9
- "V": 52,
10
- "Y": 26,
11
- "[": 33,
12
  "[FY-NL]": 61,
13
  "[PAD]": 60,
14
  "[SV-SE]": 62,
15
  "[UNK]": 59,
16
- "]": 34,
17
- "a": 36,
18
- "b": 13,
19
  "c": 30,
20
- "d": 14,
21
- "e": 54,
22
- "f": 8,
23
- "g": 35,
24
- "h": 28,
25
- "i": 4,
26
- "j": 10,
27
- "k": 23,
28
- "l": 38,
29
- "m": 41,
30
- "n": 7,
31
- "o": 5,
32
- "p": 24,
33
- "q": 11,
34
- "r": 27,
35
- "s": 48,
36
- "t": 0,
37
- "u": 20,
38
- "v": 25,
39
- "w": 40,
40
- "x": 19,
41
- "y": 39,
42
- "z": 32,
43
- "|": 1,
44
- "à": 53,
45
- "á": 17,
46
- "â": 22,
47
- "ä": 50,
48
- "å": 9,
49
- "è": 51,
50
- "é": 49,
51
- "ê": 46,
52
- "ë": 12,
53
- "ï": 29,
54
- "ó": 56,
55
- "ô": 45,
56
- "ö": 18,
57
- "ú": 58,
58
- "û": 47,
59
- "ü": 16,
60
- "–": 15,
61
- "—": 21,
62
- "’": 57,
63
- "“": 37,
64
- "”": 42
65
  }
 
1
  {
2
+ "'": 52,
3
+ "-": 19,
4
+ "E": 38,
5
+ "F": 54,
6
+ "L": 45,
7
+ "N": 5,
8
+ "S": 22,
9
+ "V": 56,
10
+ "Y": 34,
11
+ "[": 16,
12
  "[FY-NL]": 61,
13
  "[PAD]": 60,
14
  "[SV-SE]": 62,
15
  "[UNK]": 59,
16
+ "]": 24,
17
+ "a": 37,
18
+ "b": 35,
19
  "c": 30,
20
+ "d": 47,
21
+ "e": 46,
22
+ "f": 26,
23
+ "g": 36,
24
+ "h": 27,
25
+ "i": 18,
26
+ "j": 23,
27
+ "k": 3,
28
+ "l": 41,
29
+ "m": 39,
30
+ "n": 33,
31
+ "o": 53,
32
+ "p": 17,
33
+ "q": 43,
34
+ "r": 28,
35
+ "s": 9,
36
+ "t": 25,
37
+ "u": 11,
38
+ "v": 49,
39
+ "w": 20,
40
+ "x": 2,
41
+ "y": 0,
42
+ "z": 55,
43
+ "|": 10,
44
+ "à": 14,
45
+ "á": 4,
46
+ "â": 48,
47
+ "ä": 57,
48
+ "å": 42,
49
+ "è": 12,
50
+ "é": 44,
51
+ "ê": 32,
52
+ "ë": 8,
53
+ "ï": 15,
54
+ "ó": 40,
55
+ "ô": 21,
56
+ "ö": 6,
57
+ "ú": 31,
58
+ "û": 13,
59
+ "ü": 51,
60
+ "–": 58,
61
+ "—": 1,
62
+ "’": 7,
63
+ "“": 50,
64
+ "”": 29
65
  }