Add llama tokenizer

app.py

@@ -2,6 +2,7 @@ import gradio as gr
 from huggingface_hub import InferenceClient
 import os
 from rag import local_retriever, global_retriever
+from transformers import LlamaTokenizer
 
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference

rag.py

@@ -14,7 +14,7 @@ from langchain_huggingface import HuggingFaceEndpoint
 
 from typing import Dict, Any
 from tqdm import tqdm
-import tiktoken
+from transformers import LlamaTokenizer
 
 NEO4J_URI = os.getenv("NEO4J_URI")
 NEO4J_USERNAME = os.getenv("NEO4J_USERNAME")
@@ -274,8 +274,10 @@ def global_retriever(query: str, level: int, response_type: str):
         i += 1
 
     ###Debug####
-    enc = tiktoken.encoding_for_model("gpt-4")
-    tokens = enc.encode(intermediate_results)
+    global_tokenizer = LlamaTokenizer.from_pretrained(
+        "meta-llama/Meta-Llama-3-8B-Instruct"
+    )
+    tokens = global_tokenizer.encode(intermediate_results)
     print(f"Number of input tokens: {len(tokens)}")
     ###Debug###
     final_response = reduce_chain.invoke(