Spaces:
Runtime error
Runtime error
| """ Agent Evaluation Runner""" | |
| import os | |
| import gradio as gr | |
| import requests | |
| import pandas as pd | |
| import json | |
| import time | |
| from agent.agent import chat_with_agent | |
| # --- Constants --- | |
| DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" | |
| # --- Agent Definition --- | |
| class BasicAgent: | |
| def __call__(self, question: str) -> str: | |
| print(f"Agent received question: {question}") | |
| # Get response from the agent using your LLM | |
| answer = chat_with_agent(question) | |
| return answer.strip() # Return just the clean answer | |
| def download_task_file(task_id, api_url): | |
| """Download file associated with a task ID""" | |
| url = f"{api_url}/files/{task_id}" | |
| try: | |
| response = requests.get(url) | |
| if response.status_code == 200: | |
| try: | |
| content = response.text | |
| if len(content) > 50000: # Limit to 50KB | |
| content = content[:50000] | |
| return content | |
| except UnicodeDecodeError: | |
| return f"[Binary file content - {len(response.content)} bytes]" | |
| elif response.status_code == 404: | |
| return None | |
| else: | |
| return None | |
| except Exception as e: | |
| return None | |
| def run_and_submit_all(username_input=""): | |
| """ | |
| Fetches all questions, runs the BasicAgent on them, submits all answers, | |
| and displays the results. | |
| """ | |
| # --- Determine HF Space Runtime URL and Repo URL --- | |
| space_id = os.getenv("SPACE_ID") | |
| # Get username from input | |
| if username_input: | |
| username = username_input.strip() | |
| print(f"Using provided username: {username}") | |
| else: | |
| print("No username provided.") | |
| return "Please provide a username.", None | |
| api_url = DEFAULT_API_URL | |
| questions_url = f"{api_url}/questions" | |
| submit_url = f"{api_url}/submit" | |
| # 1. Instantiate Agent | |
| try: | |
| agent = BasicAgent() | |
| except Exception as e: | |
| print(f"Error instantiating agent: {e}") | |
| return f"Error initializing agent: {e}", None | |
| agent_code = f"https://huggingface.co/spaces/{space_id}/tree/main" if space_id else "https://huggingface.co/spaces/kamil1300/agent_course/tree/main" | |
| print(agent_code) | |
| # 2. Fetch Questions | |
| print(f"Fetching questions from: {questions_url}") | |
| try: | |
| response = requests.get(questions_url, timeout=15) | |
| response.raise_for_status() | |
| questions_data = response.json() | |
| if not questions_data: | |
| print("Fetched questions list is empty.") | |
| return "Fetched questions list is empty or invalid format.", None | |
| # Limit to only 20 questions | |
| questions_data = questions_data[:20] | |
| print(f"Fetched {len(questions_data)} questions (limited to 20).") | |
| except Exception as e: | |
| print(f"Error fetching questions: {e}") | |
| return f"Error fetching questions: {e}", None | |
| # 3. Run your Agent | |
| results_log = [] | |
| answers_payload = [] | |
| print(f"Running agent on {len(questions_data)} questions...") | |
| for item in questions_data: | |
| task_id = item.get("task_id") | |
| question_text = item.get("question") | |
| if not task_id or question_text is None: | |
| print(f"Skipping item with missing task_id or question: {item}") | |
| continue | |
| try: | |
| # Download task file if available | |
| task_file_content = download_task_file(task_id, api_url) | |
| # Prepare the full context for the agent | |
| if task_file_content: | |
| full_context = f"Context/File Content:\n{task_file_content}\n\nQuestion: {question_text}" | |
| print(f"\n--- Question {task_id} ---") | |
| print(f"Question: {question_text}") | |
| print(f"File content length: {len(task_file_content)} characters") | |
| print(f"File content preview: {task_file_content[:200]}...") | |
| else: | |
| full_context = question_text | |
| print(f"\n--- Question {task_id} ---") | |
| print(f"Question: {question_text}") | |
| print("No file content available") | |
| # Get answer from your LLM agent with full context | |
| submitted_answer = agent(full_context) | |
| # Clean up the answer - extract only the final answer after "FINAL ANSWER:" | |
| if "FINAL ANSWER:" in submitted_answer: | |
| submitted_answer = submitted_answer.split("FINAL ANSWER:")[-1].strip() | |
| # Remove any extra explanations or context | |
| if "\n\n" in submitted_answer: | |
| submitted_answer = submitted_answer.split("\n\n")[0].strip() | |
| # Take only the first sentence if it's still too long | |
| if len(submitted_answer.split()) > 5: | |
| submitted_answer = submitted_answer.split('.')[0].strip() | |
| # Better answer cleaning | |
| submitted_answer = submitted_answer.strip() | |
| submitted_answer = submitted_answer.replace('"', '') # Remove quotes | |
| submitted_answer = submitted_answer.lower() # Standardize case | |
| # Print the answer for debugging | |
| print(f"Answer: {submitted_answer}") | |
| # Small delay to avoid overwhelming the API | |
| time.sleep(1) | |
| # Create answer entry in the required format | |
| answer_entry = { | |
| "task_id": task_id, | |
| "submitted_answer": submitted_answer | |
| } | |
| answers_payload.append(answer_entry) | |
| print(f"Answer Entry: {answer_entry}") | |
| print("-" * 50) | |
| # For display in the table, show truncated versions | |
| display_question = question_text[:200] + "..." if len(question_text) > 200 else question_text | |
| display_answer = submitted_answer[:200] + "..." if len(submitted_answer) > 200 else submitted_answer | |
| results_log.append({ | |
| "Task ID": task_id, | |
| "Question": display_question, | |
| "Model Answer": display_answer, | |
| "Score": "N/A" # No scoring since ground truth not available | |
| }) | |
| except Exception as e: | |
| print(f"Error running agent on task {task_id}: {e}") | |
| error_response = { | |
| "task_id": task_id, | |
| "submitted_answer": f"AGENT ERROR: {e}" | |
| } | |
| answers_payload.append(error_response) | |
| results_log.append({ | |
| "Task ID": task_id, | |
| "Question": question_text[:200] + "..." if question_text and len(question_text) > 200 else question_text, | |
| "Model Answer": f"AGENT ERROR: {e}", | |
| "Score": "ERROR" | |
| }) | |
| if not answers_payload: | |
| print("Agent did not produce any answers to submit.") | |
| return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) | |
| # 4. Prepare Submission in the required format | |
| submission_data = { | |
| "username": username.strip(), | |
| "agent_code": agent_code, | |
| "answers": answers_payload | |
| } | |
| # Print the final submission format | |
| print("\n" + "="*60) | |
| print("FINAL SUBMISSION FORMAT:") | |
| print("="*60) | |
| print(json.dumps(submission_data, indent=2)) | |
| print("="*60) | |
| status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..." | |
| print(status_update) | |
| # 5. Submit | |
| print(f"Submitting {len(answers_payload)} answers to: {submit_url}") | |
| try: | |
| response = requests.post(submit_url, json=submission_data, timeout=60) | |
| response.raise_for_status() | |
| result_data = response.json() | |
| final_status = ( | |
| f"Submission Successful!\n" | |
| f"User: {result_data.get('username')}\n" | |
| f"Overall Score: {result_data.get('score', 'N/A')}% " | |
| f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" | |
| f"Message: {result_data.get('message', 'No message received.')}" | |
| ) | |
| print("Submission successful.") | |
| results_df = pd.DataFrame(results_log) | |
| return final_status, results_df | |
| except Exception as e: | |
| status_message = f"Submission Failed: {e}" | |
| print(status_message) | |
| results_df = pd.DataFrame(results_log) | |
| return status_message, results_df | |
| # --- Build Gradio Interface --- | |
| with gr.Blocks() as demo: | |
| gr.Markdown("# Agent Evaluation Runner") | |
| gr.Markdown( | |
| """ | |
| **Instructions:** | |
| 1. Enter your Hugging Face username in the text box below. | |
| 2. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score. | |
| **Note:** This will take some time as the agent processes all questions. | |
| """ | |
| ) | |
| username_input = gr.Textbox(label="Enter your Hugging Face username", placeholder="your_username") | |
| run_button = gr.Button("Run Evaluation & Submit All Answers") | |
| status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) | |
| results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) | |
| run_button.click( | |
| fn=run_and_submit_all, | |
| inputs=[username_input], | |
| outputs=[status_output, results_table] | |
| ) | |
| if __name__ == "__main__": | |
| print("\n" + "-"*30 + " App Starting " + "-"*30) | |
| space_host_startup = os.getenv("SPACE_HOST") | |
| space_id_startup = os.getenv("SPACE_ID") | |
| if space_host_startup: | |
| print(f"✅ SPACE_HOST found: {space_host_startup}") | |
| print(f" Runtime URL should be: https://{space_host_startup}.hf.space") | |
| else: | |
| print("ℹ️ SPACE_HOST environment variable not found (running locally?).") | |
| if space_id_startup: | |
| print(f"✅ SPACE_ID found: {space_id_startup}") | |
| print(f" Repo URL: https://huggingface.co/spaces/{space_id_startup}") | |
| print(f" Repo Tree URL: https://huggingface.co/spaces/{space_id_startup}/tree/main") | |
| else: | |
| print("ℹ️ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") | |
| print("-"*(60 + len(" App Starting ")) + "\n") | |
| print("Launching Gradio Interface for Agent Evaluation...") | |
| demo.launch(debug=True, share=True) |