update backend

src/backend/model_operations.py

@@ -23,11 +23,14 @@ import anthropic
 import replicate
 # pip install -U google-generativeai
 import google.generativeai as genai
+from mistralai.client import MistralClient
+from mistralai.models.chat_completion import ChatMessage
+
 
 import src.backend.util as util
 import src.envs as envs
 
-litellm.set_verbose=False
+litellm.set_verbose=True
 
 # Set up basic configuration for logging
 logging.basicConfig(level=logging.INFO,
@@ -171,9 +174,11 @@ class SummaryGenerator:
     def generate_summary(self, system_prompt: str, user_prompt: str):
         # Using Together AI API
         using_together_api = False
-        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3-', 'qwen'] #, 'mistralai'
+        together_ai_api_models = ['mixtral', 'dbrx', 'wizardlm', 'llama-3-', 'qwen', 'zero-one-ai'] #, 'mistralai'
         using_replicate_api = False
         replicate_api_models = ['snowflake', 'llama-3.1-405b']
+        using_pipeline = False
+        pipeline_models = ['llama-3.1', 'phi-3-mini','falcon-7b']
 
         for replicate_api_model in replicate_api_models:
             if replicate_api_model in self.model_id.lower():
@@ -186,7 +191,12 @@ class SummaryGenerator:
                     using_together_api = True
                     break
 
-        
+        if not using_replicate_api and not using_together_api:
+            for pipeline_model in pipeline_models:
+                if pipeline_model in self.model_id.lower():
+                    using_pipeline = True
+                    break
+
         # if 'mixtral' in self.model_id.lower() or 'dbrx' in self.model_id.lower() or 'wizardlm' in self.model_id.lower(): # For mixtral and dbrx models, use Together AI API
         if using_together_api:
             # print('using together api')
@@ -221,10 +231,11 @@ class SummaryGenerator:
                     result = result["text"]
                     result_candidates = [result_cancdidate for result_cancdidate in result.split('\n\n') if len(result_cancdidate) > 0]
                     result = result_candidates[0]
-                print(result)
+                # print(result)
             except:
-                print(response)
+                # print(response)
                 result = ''
+            print(result)
             return result
 
         # Using OpenAI API
@@ -334,6 +345,24 @@ class SummaryGenerator:
             print(result)
             return result
                 
+        elif 'mistral-large' in self.model_id.lower():
+            api_key = os.environ["MISTRAL_API_KEY"]
+            client = MistralClient(api_key=api_key)
+
+            messages = [
+                ChatMessage(role="system", content=system_prompt),
+                ChatMessage(role="user", content=user_prompt)
+            ]
+
+            # No streaming
+            chat_response = client.chat(
+                model=self.model_id,
+                messages=messages,
+            )
+            result = chat_response.choices[0].message.content
+            print(result)
+            return result
+
         # Using HF API or download checkpoints
         elif self.local_model is None and self.local_pipeline is None:
             try: # try use HuggingFace API
@@ -347,23 +376,24 @@ class SummaryGenerator:
                     api_base=self.api_base,
                 )
                 result = response['choices'][0]['message']['content']
+                result = result.split('<|im_end|>')[0]
                 print(result)
                 return result
             except Exception as e:
-                if 'Rate limit reached' in str(e):
+                if 'Rate limit reached' in str(e) and 'yi-1.5' not in self.model_id.lower():
                     wait_time = 300
                     current_time = datetime.now().strftime('%H:%M:%S')
                     print(f"Rate limit hit at {current_time}. Waiting for 5 minutes before retrying...")
                     time.sleep(wait_time)
                 else:
-                    try:
+                    if using_pipeline:
                         self.local_pipeline = pipeline(
                             "text-generation",
                             model=self.model_id,
                             model_kwargs={"torch_dtype": torch.bfloat16},
                             device_map="auto",
                         )
-                    except:
+                    else:
                         self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf" if 'openelm' in self.model_id.lower() else self.model_id, trust_remote_code=True)
                         print("Tokenizer loaded")
                         self.local_model = AutoModelForCausalLM.from_pretrained(self.model_id, trust_remote_code=True, device_map="auto", torch_dtype="auto")
@@ -372,6 +402,7 @@ class SummaryGenerator:
             
         # Using local model/pipeline
         if self.local_pipeline:
+            print('Using Transformers pipeline')
             messages=[
                 {"role": "system", "content": system_prompt}, 
                 {"role": "user", "content": user_prompt}
@@ -385,6 +416,7 @@ class SummaryGenerator:
             return result
 
         elif self.local_model: # cannot call API. using local model / pipeline
+            print('Using local model')
             if 'gemma' in self.model_id.lower() or 'mistral-7b' in self.model_id.lower():
                 messages=[
                     # gemma-1.1, mistral-7b does not accept system role
@@ -395,20 +427,29 @@ class SummaryGenerator:
             elif 'phi-2' in self.model_id.lower():
                 prompt = system_prompt + '\n' + user_prompt
 
+            elif 'intel' in self.model_id.lower():
+                prompt = f"### System:\n{system_prompt}\n### User:\n{user_prompt}\n### Assistant:\n"
+
             else:
                 messages=[
                     {"role": "system", "content": system_prompt}, 
                     {"role": "user", "content": user_prompt}
                 ]
                 prompt = self.tokenizer.apply_chat_template(messages,add_generation_prompt=True, tokenize=False)
-            print(prompt)
-            print('-'*50)
+            # print(prompt)
+            # print('-'*50)
             input_ids = self.tokenizer(prompt, return_tensors="pt").to('cuda')
             with torch.no_grad():
                 outputs = self.local_model.generate(**input_ids, max_new_tokens=250, do_sample=True, temperature=0.01, pad_token_id=self.tokenizer.eos_token_id)
+                if 'glm' in self.model_id.lower():
+                    outputs = outputs[:, input_ids['input_ids'].shape[1]:]
             result = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
             if 'gemma-2' in self.model_id.lower():
                 result = result.split(user_prompt + '\nmodel')[-1].strip()
+
+            elif 'intel' in self.model_id.lower():
+                result = result.split("### Assistant:\n")[-1]
+
             else:
                 result = result.replace(prompt.strip(), '')
             
@@ -486,7 +527,7 @@ class EvaluationModel:
                 try:
                     # summary_pieces = summary.split('\n')
                     # summary = summary_pieces[0] if len(summary_pieces[0].strip()) > 0 else summary_pieces[1]
-                    summary = summary.replace('<bos>','').replace('<eos>','')
+                    summary = summary.replace('<bos>','').replace('<eos>','').strip()
                     score = self.model.predict([doc, summary])# [0]
                     if not isinstance(score, float):
                         try:

src/backend/run_eval_suite.py

@@ -56,8 +56,10 @@ def run_evaluation(eval_request: EvalRequest, batch_size, device,
                 path_in_repo=envs.LEADERBOARD_DATASET_PATH.split('/')[-1],
                 repo_id=envs.LEADERBOARD_DATASET_REPO,
                 repo_type="dataset",
+                commit_message=f"Update results for {eval_request.model}"
             )
-            
+            logging.info(f"Leaderboard result dataset has been updated to {envs.LEADERBOARD_DATASET_PATH}/{envs.LEADERBOARD_DATASET_PATH.split('/')[-1]}")
+
     except Exception as e:
         logging.error(f"Error during evaluation: {e}")
         raise
@@ -70,10 +72,10 @@ def run_evaluation(eval_request: EvalRequest, batch_size, device,
     os.makedirs(output_folder, exist_ok=True)
     with open(output_path, "w") as f:
         f.write(dumped)
-    print(f"Results have been saved to{output_path}")
+    logging.info(f"Results have been saved to{output_path}")
 
     if not need_check:
-        print("Path in the repo:", f"{eval_request.model}/results_{datetime.now()}.json")
+        logging.info(f"Path in the repo: {eval_request.model}/results_{datetime.now()}.json")
         envs.API.upload_file(
             path_or_fileobj=output_path,
             path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",