|
|
""" |
|
|
NeuralQuantum Ollama Tokenizer for Hugging Face Transformers |
|
|
""" |
|
|
|
|
|
import json |
|
|
from typing import List, Optional, Union |
|
|
from transformers import PreTrainedTokenizer |
|
|
|
|
|
|
|
|
class NeuralQuantumOllamaTokenizer(PreTrainedTokenizer): |
|
|
"""Tokenizer for NeuralQuantum Ollama model""" |
|
|
|
|
|
def __init__( |
|
|
self, |
|
|
vocab_file=None, |
|
|
merges_file=None, |
|
|
tokenizer_file=None, |
|
|
unk_token="<|endoftext|>", |
|
|
bos_token="<|endoftext|>", |
|
|
eos_token="<|endoftext|>", |
|
|
pad_token="<|endoftext|>", |
|
|
quantum_token="<|quantum|>", |
|
|
classical_token="<|classical|>", |
|
|
system_token="<|system|>", |
|
|
user_token="<|user|>", |
|
|
assistant_token="<|assistant|>", |
|
|
add_prefix_space=False, |
|
|
**kwargs |
|
|
): |
|
|
|
|
|
vocab = { |
|
|
"<|endoftext|>": 0, |
|
|
"<|quantum|>": 1, |
|
|
"<|classical|>": 2, |
|
|
"<|system|>": 3, |
|
|
"<|user|>": 4, |
|
|
"<|assistant|>": 5, |
|
|
} |
|
|
|
|
|
|
|
|
for i, char in enumerate("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,!?;:'\"-()[]{}"): |
|
|
vocab[char] = i + 6 |
|
|
|
|
|
|
|
|
self._vocab = vocab |
|
|
self._ids_to_tokens = {v: k for k, v in vocab.items()} |
|
|
|
|
|
super().__init__( |
|
|
unk_token=unk_token, |
|
|
bos_token=bos_token, |
|
|
eos_token=eos_token, |
|
|
pad_token=pad_token, |
|
|
add_prefix_space=add_prefix_space, |
|
|
**kwargs |
|
|
) |
|
|
|
|
|
self.quantum_token = quantum_token |
|
|
self.classical_token = classical_token |
|
|
self.system_token = system_token |
|
|
self.user_token = user_token |
|
|
self.assistant_token = assistant_token |
|
|
|
|
|
@property |
|
|
def vocab_size(self): |
|
|
return len(self._vocab) |
|
|
|
|
|
def get_vocab(self): |
|
|
return dict(self._vocab) |
|
|
|
|
|
def _tokenize(self, text): |
|
|
"""Basic tokenization - split by whitespace and characters""" |
|
|
tokens = [] |
|
|
current_token = "" |
|
|
|
|
|
for char in text: |
|
|
if char.isspace(): |
|
|
if current_token: |
|
|
tokens.append(current_token) |
|
|
current_token = "" |
|
|
else: |
|
|
current_token += char |
|
|
|
|
|
if current_token: |
|
|
tokens.append(current_token) |
|
|
|
|
|
return tokens |
|
|
|
|
|
def _convert_token_to_id(self, token): |
|
|
"""Convert token to ID""" |
|
|
return self._vocab.get(token, self._vocab[self.unk_token]) |
|
|
|
|
|
def _convert_id_to_token(self, index): |
|
|
"""Convert ID to token""" |
|
|
return self._ids_to_tokens.get(index, self.unk_token) |
|
|
|
|
|
def convert_tokens_to_string(self, tokens): |
|
|
"""Convert tokens back to string""" |
|
|
return " ".join(tokens) |
|
|
|
|
|
def save_vocabulary(self, save_directory, filename_prefix=None): |
|
|
"""Save vocabulary to files""" |
|
|
vocab_file = f"{filename_prefix}-vocab.json" if filename_prefix else "vocab.json" |
|
|
vocab_path = f"{save_directory}/{vocab_file}" |
|
|
|
|
|
with open(vocab_path, 'w') as f: |
|
|
json.dump(self._vocab, f, indent=2) |
|
|
|
|
|
return (vocab_path,) |
|
|
|
|
|
def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): |
|
|
"""Build input with special tokens for Ollama format""" |
|
|
if token_ids_1 is None: |
|
|
return token_ids_0 + [self.eos_token_id] |
|
|
return token_ids_0 + token_ids_1 + [self.eos_token_id] |
|
|
|
|
|
def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): |
|
|
"""Get special tokens mask""" |
|
|
if already_has_special_tokens: |
|
|
return super().get_special_tokens_mask( |
|
|
token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True |
|
|
) |
|
|
|
|
|
if token_ids_1 is not None: |
|
|
return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] |
|
|
return [1] + ([0] * len(token_ids_0)) + [1] |
