Minor fixes for Gemini, model calls. Updated Dockerfile for non-GPU systems

Dockerfile

@@ -23,6 +23,7 @@ ENV CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS"
 COPY requirements_aws.txt .
 
 RUN pip install --no-cache-dir --target=/install torch==2.7.1+cpu --extra-index-url https://download.pytorch.org/whl/cpu \
+&& pip install --no-cache-dir --target=/install --verbose llama-cpp-python==0.3.16 \
 && pip install --no-cache-dir --target=/install -r requirements_aws.txt
 
 RUN rm requirements_aws.txt
@@ -33,6 +34,7 @@ FROM public.ecr.aws/docker/library/python:3.11.13-slim-bookworm
 # Install system dependencies.
 RUN apt-get update \
     && apt-get clean \
+    && apt-get install -y libopenblas0 \
     && rm -rf /var/lib/apt/lists/*
 
 # Set up a new user named "user" with user ID 1000

app.py

@@ -427,7 +427,7 @@ with app:
     # SUMMARISE WHOLE TABLE PAGE
     overall_summarise_previous_data_btn.click(fn= enforce_cost_codes, inputs=[enforce_cost_code_textbox, cost_code_choice_drop, cost_code_dataframe_base]).\
             success(load_in_previous_data_files, inputs=[overall_summarisation_input_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, deduplication_input_files_status, working_data_file_name_textbox, unique_topics_table_file_name_textbox]).\
-            success(overall_summary, inputs=[master_unique_topics_df_state, model_choice, google_api_key_textbox, temperature_slide, unique_topics_table_file_name_textbox, output_folder_state, in_colnames, context_textbox, aws_access_key_textbox, aws_secret_key_textbox, model_name_map_state], outputs=[overall_summary_output_files, overall_summarised_output_markdown, summarised_output_df, conversation_metadata_textbox, input_tokens_num, output_tokens_num, number_of_calls_num, estimated_time_taken_number], scroll_to_output=True, api_name="overall_summary")
+            success(overall_summary, inputs=[master_unique_topics_df_state, model_choice, google_api_key_textbox, temperature_slide, working_data_file_name_textbox, output_folder_state, in_colnames, context_textbox, aws_access_key_textbox, aws_secret_key_textbox, model_name_map_state], outputs=[overall_summary_output_files, overall_summarised_output_markdown, summarised_output_df, conversation_metadata_textbox, input_tokens_num, output_tokens_num, number_of_calls_num, estimated_time_taken_number], scroll_to_output=True, api_name="overall_summary")
 
     ###
     # CONTINUE PREVIOUS TOPIC EXTRACTION PAGE
@@ -502,9 +502,6 @@ with app:
     usage_callback.setup([session_hash_textbox, original_data_file_name_textbox, in_colnames, model_choice, conversation_metadata_textbox, input_tokens_num,
                 output_tokens_num, number_of_calls_num, estimated_time_taken_number, cost_code_choice_drop], USAGE_LOGS_FOLDER)
 
-
-    number_of_calls_num.change(conversation_metadata_textbox_change, inputs=[conversation_metadata_textbox], outputs=[conversation_metadata_textbox])
-
     number_of_calls_num.change(lambda *args: usage_callback.flag(list(args), save_to_csv=SAVE_LOGS_TO_CSV, save_to_dynamodb=SAVE_LOGS_TO_DYNAMODB,  dynamodb_table_name=USAGE_LOG_DYNAMODB_TABLE_NAME, dynamodb_headers=DYNAMODB_USAGE_LOG_HEADERS, replacement_headers=CSV_USAGE_LOG_HEADERS), [session_hash_textbox, original_data_file_name_textbox, in_colnames, model_choice, conversation_metadata_textbox, input_tokens_num, output_tokens_num, number_of_calls_num, estimated_time_taken_number, cost_code_choice_drop], None, preprocess=False, api_name="usage_logs").\
         success(fn = upload_file_to_s3, inputs=[usage_logs_state, usage_s3_logs_loc_state, s3_log_bucket_name, aws_access_key_textbox, aws_secret_key_textbox], outputs=[s3_logs_output_textbox])
 

requirements_aws.txt

@@ -13,5 +13,5 @@ html5lib==1.1
 beautifulsoup4==4.12.3
 rapidfuzz==3.13.0
 python-dotenv==1.1.0
-torch==2.7.1 --extra-index-url https://download.pytorch.org/whl/cpu
-llama-cpp-python==0.3.16
+# torch==2.7.1 --extra-index-url https://download.pytorch.org/whl/cpu # Commented out as Dockerfile should install torch
+# llama-cpp-python==0.3.16 # Commented out as Dockerfile should install llama-cpp-python

tools/config.py

@@ -211,7 +211,7 @@ model_full_names = list()
 model_short_names = list()
 model_source = list()
 
-CHOSEN_LOCAL_MODEL_TYPE = get_or_create_env_var("CHOSEN_LOCAL_MODEL_TYPE", "gpt-oss-20b") # Gemma 3 1B #  "Gemma 2b" # "Gemma 3 4B"
+CHOSEN_LOCAL_MODEL_TYPE = get_or_create_env_var("CHOSEN_LOCAL_MODEL_TYPE", "Gemma 3 4B") # Gemma 3 1B #  "Gemma 2b" # "Gemma 3 4B"
 
 if RUN_LOCAL_MODEL == "1" and CHOSEN_LOCAL_MODEL_TYPE:
     model_full_names.append(CHOSEN_LOCAL_MODEL_TYPE)

tools/dedup_summaries.py

@@ -523,6 +523,8 @@ def summarise_output_topics(sampled_reference_table_df:pd.DataFrame,
 
     tic = time.perf_counter()
 
+    model_choice_clean = clean_column_name(model_name_map[model_choice]["short_name"], max_length=20, front_characters=False)
+
     if log_output_files is None: log_output_files = list()   
 
     # Check for data for summarisations
@@ -568,7 +570,6 @@ def summarise_output_topics(sampled_reference_table_df:pd.DataFrame,
     if do_summaries == "Yes":
         
         bedrock_runtime = connect_to_bedrock_runtime(model_name_map, model_choice, aws_access_key_textbox, aws_secret_key_textbox)
-        model_source = model_name_map[model_choice]["source"]
 
         for summary_no in summary_loop:
             print("Current summary number is:", summary_no)
@@ -609,7 +610,7 @@ def summarise_output_topics(sampled_reference_table_df:pd.DataFrame,
     if latest_summary_completed >= length_all_summaries:
         print("All summaries completed. Creating outputs.")        
 
-        batch_file_path_details = create_batch_file_path_details(reference_data_file_name, model_name_map, model_choice)
+        batch_file_path_details = create_batch_file_path_details(reference_data_file_name)
 
         sampled_reference_table_df["Revised summary"] = summarised_outputs           
 
@@ -770,7 +771,9 @@ def overall_summary(topic_summary_df:pd.DataFrame,
     # else:
     #     batch_file_path_details = f"{file_name_cleaned}_col_{in_column_cleaned}"
 
-    batch_file_path_details = create_batch_file_path_details(reference_data_file_name, model_name_map, model_choice)
+    print("reference_data_file_name:", reference_data_file_name)
+
+    batch_file_path_details = create_batch_file_path_details(reference_data_file_name)
 
     tic = time.perf_counter()
 

tools/helper_functions.py

@@ -744,19 +744,16 @@ def _get_env_list(env_var_name: str) -> List[str]:
     # Split by comma and filter out any empty strings that might result from extra commas
     return [s.strip() for s in value.split(',') if s.strip()]
 
-def create_batch_file_path_details(reference_data_file_name: str, model_name_map: dict, model_choice: str) -> str:
+def create_batch_file_path_details(reference_data_file_name: str) -> str:
             """
             Creates a standardized batch file path detail string from a reference data filename.
             
             Args:
                 reference_data_file_name (str): Name of the reference data file
-                model_name_map (dict): Dictionary mapping model choices to their properties
-                model_choice (str): The chosen model name
             
             Returns:
                 str: Formatted batch file path detail string
             """
-            model_choice_clean = model_name_map[model_choice]["short_name"]
             
             # Extract components from filename using regex
             file_name = re.search(r'(.*?)(?:_all_|_final_|_batch_|_col_)', reference_data_file_name).group(1) if re.search(r'(.*?)(?:_all_|_final_|_batch_|_col_)', reference_data_file_name) else reference_data_file_name

tools/llm_api_call.py

@@ -363,12 +363,17 @@ def write_llm_output_and_logs(response_text: str,
     whole_conversation_metadata_str = '\n'.join(whole_conversation_metadata)
     start_row_reported = start_row + 1
 
-    batch_file_path_details = create_batch_file_path_details(file_name, model_name_map, model_choice_clean)
+    print("model_choice_clean in write_llm_output_and_logs:", model_choice_clean)
+
+    batch_file_path_details = create_batch_file_path_details(file_name)
 
     # Need to reduce output file names as full length files may be too long
     model_choice_clean_short = clean_column_name(model_choice_clean, max_length=20, front_characters=False)
     # in_column_cleaned = clean_column_name(in_column, max_length=20)    
-    # file_name_clean = clean_column_name(file_name, max_length=20, front_characters=True)    
+    # file_name_clean = clean_column_name(file_name, max_length=20, front_characters=True)   
+    # 
+
+    print("model_choice_clean_short in write_llm_output_and_logs:", model_choice_clean_short)
 
     # # Save outputs for each batch. If master file created, label file as master
     # batch_file_path_details = f"{file_name_clean}_batch_{latest_batch_completed + 1}_size_{batch_size_number}_col_{in_column_cleaned}"
@@ -383,15 +388,6 @@ def write_llm_output_and_logs(response_text: str,
     with open(whole_conversation_path_meta, "w", encoding='utf-8-sig', errors='replace') as f: f.write(whole_conversation_metadata_str)
 
     log_files_output_paths.append(whole_conversation_path_meta)
-    
-    # if isinstance(responses[-1], ResponseObject): response_text =  responses[-1].text
-    # elif "choices" in responses[-1]:
-    #     full_response_text = responses[-1]['choices'][0]['message']['content']
-    #     if "gpt-oss" in model_choice_clean:
-    #         response_text = full_response_text.split('<|start|>assistant<|channel|>final<|message|>')[1]
-    #     else:
-    #         response_text = full_response_text
-    # else: response_text =  responses[-1].text
 
     # Convert response text to a markdown table
     try:
@@ -422,7 +418,7 @@ def write_llm_output_and_logs(response_text: str,
     topic_with_response_df["Subtopic"] = topic_with_response_df["Subtopic"].astype(str).str.strip().str.lower().str.capitalize()
     topic_with_response_df["Sentiment"] = topic_with_response_df["Sentiment"].astype(str).str.strip().str.lower().str.capitalize()
     
-    topic_table_out_path = output_folder + batch_file_path_details + "_topic_table_" + model_choice_clean  + ".csv"
+    topic_table_out_path = output_folder + batch_file_path_details + "_topic_table_" + model_choice_clean_short  + ".csv"
 
     # Table to map references to topics
     reference_data = list()
@@ -431,7 +427,6 @@ def write_llm_output_and_logs(response_text: str,
 
     # Iterate through each row in the original DataFrame
     for index, row in topic_with_response_df.iterrows():
-        #references = re.split(r',\s*|\s+', str(row.iloc[4])) if pd.notna(row.iloc[4]) else ""
         references = re.findall(r'\d+', str(row.iloc[3])) if pd.notna(row.iloc[3]) else []
         # If no numbers found in the Response References column, check the Summary column in case reference numbers were put there by mistake
         if not references:
@@ -456,8 +451,7 @@ def write_llm_output_and_logs(response_text: str,
         if not summary and (len(str(row.iloc[3])) > 30):
             summary = row.iloc[3]        
 
-        if produce_structures_summary_radio != "Yes":
-            summary = row_number_string_start + summary
+        if produce_structures_summary_radio != "Yes": summary = row_number_string_start + summary
 
         # Create a new entry for each reference number
         for ref in references:
@@ -515,7 +509,7 @@ def write_llm_output_and_logs(response_text: str,
     out_reference_df["Group"] = group_name
 
     # Save the new DataFrame to CSV
-    reference_table_out_path = output_folder + batch_file_path_details + "_reference_table_" + model_choice_clean + ".csv"    
+    reference_table_out_path = output_folder + batch_file_path_details + "_reference_table_" + model_choice_clean_short + ".csv"    
 
     # Table of all unique topics with descriptions
     new_topic_summary_df = topic_with_response_df[["General topic", "Subtopic", "Sentiment"]]
@@ -545,7 +539,7 @@ def write_llm_output_and_logs(response_text: str,
 
     out_topic_summary_df["Group"] = group_name
 
-    topic_summary_df_out_path = output_folder + batch_file_path_details + "_unique_topics_" + model_choice_clean + ".csv"
+    topic_summary_df_out_path = output_folder + batch_file_path_details + "_unique_topics_" + model_choice_clean_short + ".csv"
 
     return topic_table_out_path, reference_table_out_path, topic_summary_df_out_path, topic_with_response_df, out_reference_df, out_topic_summary_df, batch_file_path_details, is_error
 
@@ -955,7 +949,7 @@ def extract_topics(in_data_file: GradioFileData,
                     responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text = call_llm_with_markdown_table_checks(summary_prompt_list, formatted_system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, google_client, google_config, model_choice, temperature, reported_batch_no, local_model, bedrock_runtime, model_source, MAX_OUTPUT_VALIDATION_ATTEMPTS, assistant_prefill=add_existing_topics_assistant_prefill, master = True)
 
                     # Return output tables
-                    topic_table_out_path, reference_table_out_path, topic_summary_df_out_path, new_topic_df, new_reference_df, new_topic_summary_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(response_text, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_topic_summary_df, batch_size, chosen_cols, batch_basic_response_df, model_name_map, group_name, produce_structures_summary_radio, first_run=False, output_folder=output_folder)                   
+                    topic_table_out_path, reference_table_out_path, topic_summary_df_out_path, new_topic_df, new_reference_df, new_topic_summary_df, master_batch_out_file_part, is_error = write_llm_output_and_logs(response_text, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_topic_summary_df, batch_size, chosen_cols, batch_basic_response_df, model_name_map, group_name, produce_structures_summary_radio, first_run=False, output_folder=output_folder)                   
                     
                     # Write final output to text file for logging purposes
                     try:
@@ -1150,7 +1144,7 @@ def extract_topics(in_data_file: GradioFileData,
         # # Save outputs for each batch. If master file created, label file as master
         # file_path_details = f"{file_name_cleaned}_col_{in_column_cleaned}"
 
-        file_path_details = create_batch_file_path_details(file_name, model_name_map, model_choice_clean)
+        file_path_details = create_batch_file_path_details(file_name)
 
         # Create a pivoted reference table
         existing_reference_df_pivot = convert_reference_table_to_pivot_table(existing_reference_df)        

tools/llm_funcs.py

@@ -7,7 +7,6 @@ import pandas as pd
 import json
 from tqdm import tqdm
 from huggingface_hub import hf_hub_download
-from llama_cpp.llama_speculative import LlamaPromptLookupDecoding
 from typing import List, Tuple, TypeVar
 from google import genai as ai
 from google.genai import types
@@ -52,6 +51,7 @@ print("GPU layers assigned to cuda:", gpu_layers)
 if RUN_LOCAL_MODEL == "1":
     print("Running local model - importing llama-cpp-python")
     from llama_cpp import Llama
+    from llama_cpp.llama_speculative import LlamaPromptLookupDecoding
 
 max_tokens = MAX_TOKENS
 timeout_wait = TIMEOUT_WAIT
@@ -370,10 +370,11 @@ def construct_gemini_generative_model(in_api_key: str, temperature: float, model
             api_key = os.environ["GOOGLE_API_KEY"]
             client = ai.Client(api_key=api_key)
         else:
-            print("No API key foound")
-            raise gr.Error("No API key found.")
+            print("No Gemini API key found")
+            raise Warning("No Gemini API key found.")
     except Exception as e:
-        print(e)
+        print("Error constructing Gemini generative model:", e)
+        raise Warning("Error constructing Gemini generative model:", e)
         
     config = types.GenerateContentConfig(temperature=temperature, max_output_tokens=max_tokens, seed=random_seed)
 
@@ -553,11 +554,6 @@ def send_request(prompt: str, conversation_history: List[dict], google_client: a
         response_text = response_text.strip()
         conversation_history.append({'role': 'assistant', 'parts': [response_text]})
     
-    # Print the updated conversation history
-    #print("conversation_history:", conversation_history)
-
-    print("response_text:", response_text)
-    
     return response, conversation_history, response_text
 
 def process_requests(prompts: List[str], system_prompt: str, conversation_history: List[dict], whole_conversation: List[str], whole_conversation_metadata: List[str], google_client: ai.Client, config: types.GenerateContentConfig, model_choice: str, temperature: float, bedrock_runtime:boto3.Session.client, model_source:str, batch_no:int = 1, local_model = list(), master:bool = False, assistant_prefill="") -> Tuple[List[ResponseObject], List[dict], List[str], List[str]]:
@@ -619,11 +615,14 @@ def process_requests(prompts: List[str], system_prompt: str, conversation_histor
                     # Append the clean, standardised data
                     whole_conversation_metadata.append('outputTokens: ' + str(output_tokens) + ' inputTokens: ' + str(input_tokens))
 
-                elif "Gemini" in model_source:
+                elif "Gemini" in model_source:                    
+
+                    output_tokens = response.usage_metadata.candidates_token_count
+                    input_tokens = response.usage_metadata.prompt_token_count
+
                     whole_conversation_metadata.append(str(response.usage_metadata))
 
                 elif "Local" in model_source:
-                    #print("Adding usage metadata to whole conversation metadata:", response['usage'])
                     output_tokens = response['usage'].get('completion_tokens', 0)
                     input_tokens = response['usage'].get('prompt_tokens', 0)
                     whole_conversation_metadata.append(str(response['usage']))

tools/verify_titles.py

@@ -80,14 +80,11 @@ def write_llm_output_and_logs_verify(response_text: str,
 
     model_choice_clean_short = clean_column_name(model_choice_clean, max_length=20, front_characters=False)
 
-    # Example usage
-    in_column_cleaned = clean_column_name(in_column, max_length=20)
-
     # Need to reduce output file names as full length files may be too long
-    file_name = clean_column_name(file_name, max_length=30)    
+    file_name = clean_column_name(file_name, max_length=20)    
 
     # Save outputs for each batch. If master file created, label file as master
-    batch_file_path_details = create_batch_file_path_details(file_name, model_name_map, model_choice_clean)
+    batch_file_path_details = create_batch_file_path_details(file_name)
     row_number_string_start = f"Rows {start_row_reported} to {end_row}: "
 
     whole_conversation_path = output_folder + batch_file_path_details + "_full_conversation_" + model_choice_clean_short + "_temp_" + str(temperature) + ".txt"