Update app.py

app.py

@@ -4,122 +4,42 @@ import gradio as gr
 import requests
 import pandas as pd
 import json
-import re
-import string
-import warnings
-import numpy as np
+import time
 from agent.agent import chat_with_agent
 
 # --- Constants ---
 DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
 
-# --- Scoring Functions ---
-def normalize_number_str(number_str: str) -> float:
-    # we replace these common units and commas to allow
-    # conversion to float
-    for char in ["$", "%", ","]:
-        number_str = number_str.replace(char, "")
-    try:
-        return float(number_str)
-    except ValueError:
-        print(f"String {number_str} cannot be normalized to number str.")
-        return float("inf")
-
-def split_string(
-    s: str,
-    char_list: list[str] = [",", ";"],
-) -> list[str]:
-    pattern = f"[{''.join(char_list)}]"
-    return re.split(pattern, s)
-
-def normalize_str(input_str, remove_punct=True) -> str:
-    """
-    Normalize a string by:
-    - Removing all white spaces
-    - Optionally removing punctuation (if remove_punct is True)
-    - Converting to lowercase
-    Parameters:
-    - input_str: str, the string to normalize
-    - remove_punct: bool, whether to remove punctuation (default: True)
-    Returns:
-    - str, the normalized string
-    """
-    # Remove all white spaces. Required e.g for seagull vs. sea gull
-    no_spaces = re.sub(r"\s", "", input_str)
-
-    # Remove punctuation, if specified.
-    if remove_punct:
-        translator = str.maketrans("", "", string.punctuation)
-        return no_spaces.lower().translate(translator)
-    else:
-        return no_spaces.lower()
-
-def question_scorer(
-    model_answer: str,
-    ground_truth: str,
-) -> bool:
-    def is_float(element: any) -> bool:
-        try:
-            float(element)
-            return True
-        except ValueError:
-            return False
-        
-    if model_answer is None:
-        model_answer = "None"
-
-    # if gt is a number
-    if is_float(ground_truth):
-        print(f"Evaluating {model_answer} as a number.")
-        normalized_answer = normalize_number_str(model_answer)
-        return normalized_answer == float(ground_truth)
-
-    # if gt is a list
-    elif any(char in ground_truth for char in [",", ";"]):
-        print(f"Evaluating {model_answer} as a comma separated list.")
-        # question with the fish: normalization removes punct
-
-        gt_elems = split_string(ground_truth)
-        ma_elems = split_string(model_answer)
-
-        # check length is the same
-        if len(gt_elems) != len(ma_elems):
-            warnings.warn(
-                "Answer lists have different lengths, returning False.", UserWarning
-            )
-            return False
-
-        # compare each element as float or str
-        comparisons = []
-        for ma_elem, gt_elem in zip(ma_elems, gt_elems):
-            if is_float(gt_elem):
-                normalized_ma_elem = normalize_number_str(ma_elem)
-                comparisons.append(normalized_ma_elem == float(gt_elem))
-            else:
-                # we do not remove punct since comparisons can include punct
-                comparisons.append(
-                    normalize_str(ma_elem, remove_punct=False)
-                    == normalize_str(gt_elem, remove_punct=False)
-                )
-        return all(comparisons)
-
-    # if gt is a str
-    else:
-        print(f"Evaluating {model_answer} as a string.")
-        return normalize_str(model_answer) == normalize_str(ground_truth)
-
 # --- Agent Definition ---
 class BasicAgent:
-    def __call__(self, question: str) -> dict:
-        print(f"Agent received question (first 50 chars): {question[:50]}...")
-        # Get response from the agent
+    def __call__(self, question: str) -> str:
+        print(f"Agent received question: {question}")
+        # Get response from the agent using your LLM
         answer = chat_with_agent(question)
+        return answer.strip()  # Return just the clean answer
+
+def download_task_file(task_id, api_url):
+    """Download file associated with a task ID"""
+    url = f"{api_url}/files/{task_id}"
+    
+    try:
+        response = requests.get(url)
         
-        # Return in the format expected by the API
-        return {
-            "model_answer": answer,
-            "reasoning_trace": answer  # Using the full response as reasoning trace
-        }
+        if response.status_code == 200:
+            try:
+                content = response.text
+                if len(content) > 50000:  # Limit to 50KB
+                    content = content[:50000]
+                return content
+            except UnicodeDecodeError:
+                return f"[Binary file content - {len(response.content)} bytes]"
+        elif response.status_code == 404:
+            return None
+        else:
+            return None
+            
+    except Exception as e:
+        return None
 
 def run_and_submit_all(username_input=""):
     """
@@ -148,7 +68,7 @@ def run_and_submit_all(username_input=""):
         print(f"Error instantiating agent: {e}")
         return f"Error initializing agent: {e}", None
     
-    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "https://huggingface.co/spaces/your-space-id/tree/main"
+    agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "https://huggingface.co/spaces/kamil1300/agent_course/tree/main"
     print(agent_code)
 
     # 2. Fetch Questions
@@ -160,69 +80,96 @@ def run_and_submit_all(username_input=""):
         if not questions_data:
              print("Fetched questions list is empty.")
              return "Fetched questions list is empty or invalid format.", None
-        print(f"Fetched {len(questions_data)} questions.")
-    except requests.exceptions.RequestException as e:
+        
+        # Limit to only 20 questions
+        questions_data = questions_data[:20]
+        print(f"Fetched {len(questions_data)} questions (limited to 20).")
+    except Exception as e:
         print(f"Error fetching questions: {e}")
         return f"Error fetching questions: {e}", None
-    except requests.exceptions.JSONDecodeError as e:
-         print(f"Error decoding JSON response from questions endpoint: {e}")
-         print(f"Response text: {response.text[:500]}")
-         return f"Error decoding server response for questions: {e}", None
-    except Exception as e:
-        print(f"An unexpected error occurred fetching questions: {e}")
-        return f"An unexpected error occurred fetching questions: {e}", None
 
     # 3. Run your Agent
     results_log = []
     answers_payload = []
     print(f"Running agent on {len(questions_data)} questions...")
+    
     for item in questions_data:
         task_id = item.get("task_id")
         question_text = item.get("question")
-        ground_truth = item.get("ground_truth", "")  # Get ground truth if available
         
         if not task_id or question_text is None:
             print(f"Skipping item with missing task_id or question: {item}")
             continue
+            
         try:
-            # Get structured response from agent
-            agent_response = agent(question_text)
+            # Download task file if available
+            task_file_content = download_task_file(task_id, api_url)
+            
+            # Prepare the full context for the agent
+            if task_file_content:
+                full_context = f"Context/File Content:\n{task_file_content}\n\nQuestion: {question_text}"
+                print(f"\n--- Question {task_id} ---")
+                print(f"Question: {question_text}")
+                print(f"File content length: {len(task_file_content)} characters")
+                print(f"File content preview: {task_file_content[:200]}...")
+            else:
+                full_context = question_text
+                print(f"\n--- Question {task_id} ---")
+                print(f"Question: {question_text}")
+                print("No file content available")
+            
+            # Get answer from your LLM agent with full context
+            submitted_answer = agent(full_context)
+            
+            # Clean up the answer - extract only the final answer after "FINAL ANSWER:"
+            if "FINAL ANSWER:" in submitted_answer:
+                submitted_answer = submitted_answer.split("FINAL ANSWER:")[-1].strip()
+            
+            # Remove any extra explanations or context
+            if "\n\n" in submitted_answer:
+                submitted_answer = submitted_answer.split("\n\n")[0].strip()
+            
+            # Take only the first sentence if it's still too long
+            if len(submitted_answer.split()) > 5:
+                submitted_answer = submitted_answer.split('.')[0].strip()
+            
+            # Better answer cleaning
+            submitted_answer = submitted_answer.strip()
+            submitted_answer = submitted_answer.replace('"', '')  # Remove quotes
+            submitted_answer = submitted_answer.lower()  # Standardize case
             
-            # Extract model_answer and reasoning_trace
-            model_answer = agent_response.get("model_answer", "")
-            reasoning_trace = agent_response.get("reasoning_trace", "")
+            # Print the answer for debugging
+            print(f"Answer: {submitted_answer}")
             
-            # Score the answer if ground truth is available
-            score = None
-            if ground_truth:
-                score = question_scorer(model_answer, ground_truth)
+            # Small delay to avoid overwhelming the API
+            time.sleep(1)
             
-            # Create JSON-line format entry
-            json_line_entry = {
+            # Create answer entry in the required format
+            answer_entry = {
                 "task_id": task_id,
-                "model_answer": model_answer,
-                "reasoning_trace": reasoning_trace
+                "submitted_answer": submitted_answer
             }
             
-            answers_payload.append(json_line_entry)
+            answers_payload.append(answer_entry)
+            print(f"Answer Entry: {answer_entry}")
+            print("-" * 50)
             
             # For display in the table, show truncated versions
             display_question = question_text[:200] + "..." if len(question_text) > 200 else question_text
-            display_answer = model_answer[:200] + "..." if len(model_answer) > 200 else model_answer
+            display_answer = submitted_answer[:200] + "..." if len(submitted_answer) > 200 else submitted_answer
             
             results_log.append({
                 "Task ID": task_id, 
                 "Question": display_question, 
                 "Model Answer": display_answer,
-                "Score": "✓" if score else "✗" if score is False else "N/A"
+                "Score": "N/A"  # No scoring since ground truth not available
             })
             
         except Exception as e:
              print(f"Error running agent on task {task_id}: {e}")
              error_response = {
                  "task_id": task_id,
-                 "model_answer": f"AGENT ERROR: {e}",
-                 "reasoning_trace": f"Agent encountered an error while processing the question: {str(e)}"
+                 "submitted_answer": f"AGENT ERROR: {e}"
              }
              answers_payload.append(error_response)
              results_log.append({
@@ -236,8 +183,20 @@ def run_and_submit_all(username_input=""):
         print("Agent did not produce any answers to submit.")
         return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
 
-    # 4. Prepare Submission 
-    submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload}
+    # 4. Prepare Submission in the required format
+    submission_data = {
+        "username": username.strip(), 
+        "agent_code": agent_code, 
+        "answers": answers_payload
+    }
+    
+    # Print the final submission format
+    print("\n" + "="*60)
+    print("FINAL SUBMISSION FORMAT:")
+    print("="*60)
+    print(json.dumps(submission_data, indent=2))
+    print("="*60)
+    
     status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
     print(status_update)
 
@@ -257,29 +216,8 @@ def run_and_submit_all(username_input=""):
         print("Submission successful.")
         results_df = pd.DataFrame(results_log)
         return final_status, results_df
-    except requests.exceptions.HTTPError as e:
-        error_detail = f"Server responded with status {e.response.status_code}."
-        try:
-            error_json = e.response.json()
-            error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
-        except requests.exceptions.JSONDecodeError:
-            error_detail += f" Response: {e.response.text[:500]}"
-        status_message = f"Submission Failed: {error_detail}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.Timeout:
-        status_message = "Submission Failed: The request timed out."
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
-    except requests.exceptions.RequestException as e:
-        status_message = f"Submission Failed: Network error - {e}"
-        print(status_message)
-        results_df = pd.DataFrame(results_log)
-        return status_message, results_df
     except Exception as e:
-        status_message = f"An unexpected error occurred during submission: {e}"
+        status_message = f"Submission Failed: {e}"
         print(status_message)
         results_df = pd.DataFrame(results_log)
         return status_message, results_df