BuddAi / tokenizergen.py
CaptMetal's picture
Update tokenizergen.py
f92e6f0 verified
raw
history blame
295 Bytes
from transformers import AutoTokenizer
# Download and save the tokenizer files
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files
print("Tokenizer files saved to 'BuddAi/' directory!")