from transformers import AutoTokenizer | |
# Download and save the tokenizer files | |
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2") | |
tokenizer.save_pretrained("BuddAi") # Creates a folder with all tokenizer files | |
print("Tokenizer files saved to 'BuddAi/' directory!") |