Update app.py

app.py

@@ -34,6 +34,10 @@ dotenv.load_dotenv()
 seamless_client = Client("facebook/seamless_m4t")
 HuggingFace_Token = os.getenv("HuggingFace_Token")
 hf_token = os.getenv("HuggingFace_Token")
+base_model_id = os.getenv('BASE_MODEL_ID', 'default_base_model_id')
+model_directory = os.getenv('MODEL_DIRECTORY', 'default_model_directory')
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
 
 def check_hallucination(assertion,citation):
     API_URL = "https://api-inference.huggingface.co/models/vectara/hallucination_evaluation_model"
@@ -336,13 +340,6 @@ def multimodal_prompt(user_input, system_prompt="You are an expert medical analy
 
     return response_text
 
-# Define the device
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-# Use the base model's ID
-base_model_id = "stabilityai/stablelm-3b-4e1t"
-model_directory = "Tonic/stablemed"
-
 # Instantiate the Tokenizer
 tokenizer = AutoTokenizer.from_pretrained("stabilityai/stablelm-3b-4e1t", token=hf_token, trust_remote_code=True, padding_side="left")
 # tokenizer = AutoTokenizer.from_pretrained("Tonic/stablemed", trust_remote_code=True, padding_side="left")
@@ -360,7 +357,7 @@ class ChatBot:
         self.history = []
 
     def predict(self, user_input, system_prompt="You are an expert medical analyst:"):
-        formatted_input = f"<s>[INST]{system_prompt} {user_input}[/INST]"
+        formatted_input = f"<s>[INST] {user_input}</s>[/INST]{system_prompt}"
         user_input_ids = tokenizer.encode(formatted_input, return_tensors="pt")
         response = peft_model.generate(input_ids=user_input_ids, max_length=512, pad_token_id=tokenizer.eos_token_id)
         response_text = tokenizer.decode(response[0], skip_special_tokens=True)