Fix incorrect unk_id assignment

#16
Files changed (1) hide show
  1. tokenization_kimia.py +1 -1
tokenization_kimia.py CHANGED
@@ -127,7 +127,7 @@ class TikTokenTokenizer(PreTrainedTokenizer):
127
  self.pad_id: int = self.special_tokens[self.pad_token]
128
 
129
  self.unk_token: str = special_tokens[-2]
130
- self.unk_id: int = self.special_tokens[self.pad_token]
131
 
132
  self.stop_tokens = {
133
  self.special_tokens["[EOS]"],
 
127
  self.pad_id: int = self.special_tokens[self.pad_token]
128
 
129
  self.unk_token: str = special_tokens[-2]
130
+ self.unk_id: int = self.special_tokens[self.unk_token]
131
 
132
  self.stop_tokens = {
133
  self.special_tokens["[EOS]"],