Fix incorrect unk_id assignment
#16
by
codecho
- opened
- tokenization_kimia.py +1 -1
tokenization_kimia.py
CHANGED
@@ -127,7 +127,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
|
|
127 |
self.pad_id: int = self.special_tokens[self.pad_token]
|
128 |
|
129 |
self.unk_token: str = special_tokens[-2]
|
130 |
-
self.unk_id: int = self.special_tokens[self.
|
131 |
|
132 |
self.stop_tokens = {
|
133 |
self.special_tokens["[EOS]"],
|
|
|
127 |
self.pad_id: int = self.special_tokens[self.pad_token]
|
128 |
|
129 |
self.unk_token: str = special_tokens[-2]
|
130 |
+
self.unk_id: int = self.special_tokens[self.unk_token]
|
131 |
|
132 |
self.stop_tokens = {
|
133 |
self.special_tokens["[EOS]"],
|