init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

README.md

@@ -7,42 +7,11 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
-short_description: Duplicate this leaderboard to initialize your own!
+short_description: Ranking of LLMs for agentic tasks
 sdk_version: 5.43.1
 tags:
 - leaderboard
 ---
 
-# Start the configuration
 
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).

app.py

@@ -1,204 +1,20 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# Add this at the top of your script
+import warnings
 
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
+warnings.filterwarnings("ignore")
 
+import gradio as gr
+from tabs.leaderboard_v1 import create_leaderboard_v2_interface
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
 
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+def create_app():
+    with gr.Blocks(
+        theme=gr.themes.Default(primary_hue=gr.themes.colors.red)
+    ) as app:
+        create_leaderboard_v2_interface()
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+    return app
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo = create_app()
+demo.launch()

combined_evaluation_summary.csv

@@ -0,0 +1,7 @@
+Model,Vendor,Model Type,L1_Total_Tasks,L2_Total_Tasks,L3_Total_Tasks,L4_Total_Tasks,L5_Total_Tasks,L6_Total_Tasks,L7_Total_Tasks,L1_Evaluated_Tasks,L2_Evaluated_Tasks,L3_Evaluated_Tasks,L4_Evaluated_Tasks,L5_Evaluated_Tasks,L6_Evaluated_Tasks,L7_Evaluated_Tasks,L1_Avg_Exec_Time,L2_Avg_Exec_Time,L3_Avg_Exec_Time,L4_Avg_Exec_Time,L5_Avg_Exec_Time,L6_Avg_Exec_Time,L7_Avg_Exec_Time,L1_Avg_Tokens,L2_Avg_Tokens,L3_Avg_Tokens,L4_Avg_Tokens,L5_Avg_Tokens,L6_Avg_Tokens,L7_Avg_Tokens,L1_Avg_TPS,L2_Avg_TPS,L3_Avg_TPS,L4_Avg_TPS,L5_Avg_TPS,L6_Avg_TPS,L7_Avg_TPS,L1_Avg_TTFT,L2_Avg_TTFT,L3_Avg_TTFT,L4_Avg_TTFT,L5_Avg_TTFT,L6_Avg_TTFT,L7_Avg_TTFT,L1_RRR,L2_RRR,L3_RRR,L4_RRR,L5_RRR,L6_RRR,L7_RRR,L1_SR,L2_SR,L3_SR,L4_SR,L5_SR,L6_SR,L7_SR,L1_EPR_CVR,L2_EPR_CVR,L3_EPR_CVR,L4_EPR_CVR,L5_EPR_CVR,L6_EPR_CVR,L7_EPR_CVR,L1_pass@k,L2_pass@k,L3_pass@k,L4_pass@k,L5_pass@k,L6_pass@k,L7_pass@k,L1_TooAcc,L1_ArgAcc,L1_CallEM,L1_RespOK,L2_SelectAcc,L3_FSM,L3_PSM,L3_ΔSteps_norm,L3_ProvAcc,L4_Coverage,L4_SourceEPR,L5_AdaptiveRoutingScore,L5_FallbackSR,L6_ReuseRage,L6_RedundantCallRate,L6_EffScore,L7_ContextRetention,L7_RefRecall
+kanana-1.5-8b-instruct-2505,Kakao,OSS,11,30,10,10,20,15,10,11,30,10,10,20,15,10,5.53,17.22,14.51,23.78,9.44,52.98,47.39,4556.36,6107.6,5723.4,7188.3,5665.9,28502.33,28738.1,823.46,354.62,394.38,302.24,599.94,538.01,606.41,1.5236,6.7827,5.9015,7.4927,1.4163,7.764,5.1605,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.8409,0.925,0.55,0.55,0.45,0.7167,0.4,1.0,1.0,1.0,0.9,0.225,1.0,0.9,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.6364,0.2727,1.0,1.0,0.0,0.5333,0.0,0.0,0.2667,0.2667,0.225,0.45,0.4,1.0,0.6,0.825,0.75
+skt_A.X-4.0-Light,SKT,OSS,11,30,10,10,20,15,10,11,30,10,10,20,15,10,5.15,17.37,21.51,9.06,9.23,38.97,33.94,4286.73,7456.1,13579.8,2284.9,6500.85,27744.0,25032.0,833.07,429.13,631.27,252.27,704.42,711.88,737.55,1.3615,5.8379,6.0725,6.2881,1.3627,5.3648,3.902,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.5455,0.7417,0.525,0.35,0.2875,0.55,0.45,1.0,1.0,1.0,0.3,0.2583,0.8667,0.9,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.8182,0.4545,1.0,1.0,0.2,0.7833,0.65,0.1,0.05,0.05,0.25,0.55,0.4,1.0,0.4667,0.8,0.775
+qwen3-8B,알리바바,OSS,11,30,10,10,20,15,10,11,30,10,10,20,15,10,24.54,33.11,38.89,61.09,46.28,102.03,92.19,5798.0,7600.07,8380.0,14758.8,9789.4,45946.13,55163.2,236.28,229.53,215.5,241.58,211.54,450.34,598.37,11.0876,13.3456,23.3045,16.4015,8.5784,16.7883,11.2336,1.0,1.0,0.9,0.9,1.0,1.0,1.0,0.5909,0.8083,0.175,0.35,0.45,0.7833,0.525,1.0,1.0,0.4,0.9,0.2258,1.0,0.95,1.0,1.0,0.9,0.8,0.9667,1.0,1.0,1.0,0.7955,0.4545,1.0,1.0,0.2,0.3,0.2,0.1,0.4667,0.4667,0.2333,0.55,0.2,1.0,0.5667,0.85,0.775
+gemini-2.5-pro,Google,API,11,30,10,10,20,15,10,11,30,10,10,20,15,10,9.01,10.45,11.43,29.65,15.91,43.0,33.16,5257.45,5761.23,6384.2,22304.6,7592.2,54436.6,50150.6,583.2,551.49,558.73,752.35,477.25,1266.0,1512.44,4.6263,5.4812,7.9657,8.8433,4.9659,7.1894,5.2974,0.9091,0.8,0.8,1.0,0.8,0.8667,0.9,0.8409,0.6583,0.2,0.425,0.4,0.4,0.35,0.9091,0.7667,0.2,0.7,0.1583,0.8667,0.9,0.9091,0.8,0.8,1.0,0.8,0.8667,0.9,0.9091,0.6364,0.2727,0.9091,0.7667,0.1,0.1667,0.1,0.0,0.4833,0.4833,0.1583,0.35,0.5333,1.0,0.1222,0.825,0.7
+Qwen3-4B-Instruct-2507,알리바바,OSS,11,30,10,10,20,15,10,11,30,10,10,20,15,10,6.66,22.89,14.8,51.19,11.71,86.63,60.09,5273.09,6447.9,9087.8,17502.5,5363.85,36058.4,37068.1,791.39,281.66,613.83,341.91,458.02,416.23,616.84,2.093,9.1244,4.4172,13.7638,1.8319,14.8681,8.245,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.6364,0.6583,0.15,0.375,0.3,0.6167,0.425,1.0,1.0,1.0,0.9,0.15,1.0,1.0,1.0,1.0,1.0,0.9333,1.0,1.0,1.0,1.0,0.75,0.3636,1.0,1.0,0.2,0.6333,0.7,0.0,0.5167,0.5167,0.15,0.3,0.1333,1.0,0.4,0.875,0.8
+Midm-2.0-Base-Instruct,KT,OSS,11,30,10,10,20,15,10,11,30,10,10,20,15,10,5.39,3.9,3.06,3.75,8.13,28.66,16.08,4185.82,2514.93,3418.3,2388.8,3084.5,22909.13,14079.1,775.89,644.46,1117.59,636.3,379.51,799.33,875.38,1.4775,1.8563,1.8855,1.6781,1.0824,1.6794,1.1356,1.0,1.0,1.0,1.0,0.95,1.0,1.0,0.5909,0.5167,0.25,0.325,0.275,0.4833,0.35,0.9091,0.5667,0.2,0.3,0.0667,0.9333,0.6,1.0,1.0,1.0,0.8667,0.9833,1.0,1.0,0.9091,0.6364,0.2727,1.0,0.5667,0.0,0.1,0.0,0.0,0.0,0.0,0.0667,0.15,0.0,0.9333,0.3,0.55,0.5

components/leaderboard_components.py

@@ -0,0 +1,467 @@
+"""
+Reusable components for the Agent Leaderboard v2
+These are stable components that don't change frequently
+"""
+
+def get_chart_colors():
+    return {
+        "Private": "#1098F7",  # Airglow Blue for Proprietary
+        "Open source": "#58BC82",  # Green for Open source
+        "performance_bands": ["#DCFCE7", "#FEF9C3", "#FEE2E2"],
+        "text": "#F5F6F7",
+        "background": "#01091A",
+        "grid": (0, 0, 0, 0.1),  # RGBA tuple for grid
+    }
+
+
+def get_rank_badge(rank):
+    """Generate HTML for rank badge with appropriate styling"""
+    badge_styles = {
+        1: ("1st", "linear-gradient(145deg, #ffd700, #ffc400)", "#000"),
+        2: ("2nd", "linear-gradient(145deg, #9ca3af, #787C7E)", "#fff"),
+        3: ("3rd", "linear-gradient(145deg, #CD7F32, #b36a1d)", "#fff"),
+    }
+
+    if rank in badge_styles:
+        label, gradient, text_color = badge_styles[rank]
+        return f"""
+            <div style="
+                display: inline-flex;
+                align-items: center;
+                justify-content: center;
+                min-width: 48px;
+                padding: 4px 12px;
+                background: {gradient};
+                color: {text_color};
+                border-radius: 6px;
+                font-weight: 600;
+                font-size: 0.9em;
+                box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+            ">
+                {label}
+            </div>
+        """
+    return f"""
+        <div style="
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            min-width: 28px;
+            color: #a1a1aa;
+            font-weight: 500;
+        ">
+            {rank}
+        </div>
+    """
+
+
+def get_type_badge(model_type):
+    """Generate HTML for model type badge"""
+    colors = get_chart_colors()
+    color_map = {
+        "Open source": colors.get("Open source", "#58BC82"),
+        "Proprietary": colors.get("Private", "#1098F7"),
+        "Private": colors.get("Private", "#1098F7"),
+    }
+    label_map = {
+        "Open source": "OSS",
+        "Proprietary": "API",
+        "Private": "API",
+    }
+    bg_color = color_map.get(model_type, "#4F46E5")
+    display_label = label_map.get(model_type, model_type)
+    return f"""
+        <div style="
+            display: inline-flex;
+            align-items: center;
+            padding: 4px 8px;
+            background: {bg_color};
+            color: white;
+            border-radius: 4px;
+            font-size: 0.85em;
+            font-weight: 500;
+        ">
+            {display_label}
+        </div>
+    """
+
+
+def get_output_type_badge(output_type):
+    """Generate HTML for output type badge"""
+    if output_type == "Reasoning":
+        bg_color = "#9333ea"  # Purple for reasoning
+    else:
+        bg_color = "#6b7280"  # Gray for normal
+    
+    return f"""
+        <div style="
+            display: inline-flex;
+            align-items: center;
+            gap: 4px;
+            padding: 4px 8px;
+            background: {bg_color};
+            color: white;
+            border-radius: 4px;
+            font-size: 0.85em;
+            font-weight: 500;
+        ">
+            {output_type}
+        </div>
+    """
+
+
+def get_score_bar(score):
+    """Generate HTML for score bar with gradient styling and tooltip"""
+    width = score * 100
+    return f"""
+        <div style="display: flex; align-items: center; gap: 12px; width: 100%;">
+            <div style="
+                flex-grow: 1;
+                height: 8px;
+                background: rgba(245, 246, 247, 0.1);
+                border-radius: 4px;
+                overflow: hidden;
+                max-width: 200px;
+            ">
+                <div style="
+                    width: {width}%;
+                    height: 100%;
+                    background: #ffd21e;
+                    border-radius: 4px;
+                    transition: width 0.3s ease;
+                "></div>
+            </div>
+            <span style="
+                font-family: 'SF Mono', monospace;
+                font-weight: 600;
+                color: #F5F6F7;
+                min-width: 60px;
+            ">{score:.3f}</span>
+        </div>
+    """
+
+
+def get_metric_tooltip(metric):
+    """Return tooltip text for different metrics"""
+    tooltips = {
+        "Avg AC": "Action Completion (AC): Measures how well the agent accomplishes user goals and completes tasks successfully. Higher is better (0-1 scale).",
+        "Avg TSQ": "Tool Selection Quality (TSQ): Evaluates the accuracy of selecting the right tools and using them with correct parameters. Higher is better (0-1 scale).",
+        "Avg Total Cost": "Average cost per conversation session in USD, including all API calls and processing. Lower is better.",
+        "Avg Session Duration": "Average time taken to complete a full conversation session from start to finish, measured in seconds. Lower is generally better.",
+        "Avg Turns": "Average number of back-and-forth exchanges needed to complete a task. Lower typically indicates more efficient task completion.",
+        "Banking AC": "Action Completion score specific to banking domain tasks.",
+        "Banking TSQ": "Tool Selection Quality score specific to banking domain tasks.",
+        "Healthcare AC": "Action Completion score specific to healthcare domain tasks.",
+        "Healthcare TSQ": "Tool Selection Quality score specific to healthcare domain tasks.",
+        "Insurance AC": "Action Completion score specific to insurance domain tasks.",
+        "Insurance TSQ": "Tool Selection Quality score specific to insurance domain tasks.",
+        "Investment AC": "Action Completion score specific to investment domain tasks.",
+        "Investment TSQ": "Tool Selection Quality score specific to investment domain tasks.",
+        "Telecom AC": "Action Completion score specific to telecom domain tasks.",
+        "Telecom TSQ": "Tool Selection Quality score specific to telecom domain tasks.",
+    }
+    return tooltips.get(metric, "")
+
+
+def get_responsive_styles():
+    """Return responsive CSS styles for mobile devices"""
+    return """
+    <style>
+    /* Enhanced mobile responsiveness */
+    @media (max-width: 768px) {
+        /* Stack grid layouts vertically on mobile */
+        .insight-card-grid,
+        .metric-card-grid {
+            grid-template-columns: 1fr !important;
+            gap: 12px !important;
+        }
+        
+        /* Adjust table for mobile */
+        .v2-styled-table {
+            font-size: 12px !important;
+        }
+        
+        .v2-styled-table th,
+        .v2-styled-table td {
+            padding: 8px 6px !important;
+        }
+        
+        /* Hide less important columns on mobile */
+        .v2-styled-table th:nth-child(8),
+        .v2-styled-table td:nth-child(8),
+        .v2-styled-table th:nth-child(9),
+        .v2-styled-table td:nth-child(9) {
+            display: none !important;
+        }
+        
+        /* Make score bars more compact */
+        .score-cell {
+            min-width: 120px !important;
+        }
+        
+        /* Adjust domain selector for mobile */
+        .domain-radio .wrap {
+            flex-direction: column !important;
+            gap: 8px !important;
+        }
+        
+        .domain-radio label {
+            min-width: 100% !important;
+            max-width: 100% !important;
+        }
+        
+        /* Compact filter controls on mobile */
+        .compact-filter-row {
+            flex-direction: column !important;
+        }
+        
+        .compact-radio .wrap > label {
+            font-size: 0.7rem !important;
+            padding: 4px 8px !important;
+        }
+        
+        /* Adjust navigation buttons */
+        .nav-buttons-container {
+            flex-direction: column !important;
+            gap: 8px !important;
+        }
+        
+        .nav-link-button {
+            width: 100% !important;
+            justify-content: center !important;
+        }
+        
+        /* Header adjustments */
+        h1 {
+            font-size: 2rem !important;
+        }
+        
+        h2 {
+            font-size: 1.5rem !important;
+        }
+        
+        h3 {
+            font-size: 1.2rem !important;
+        }
+        
+        /* Card padding adjustments */
+        .dark-container {
+            padding: 16px !important;
+        }
+        
+        .info-box {
+            padding: 12px !important;
+        }
+        
+        /* Chart container adjustments */
+        .chart-container {
+            overflow-x: auto !important;
+            -webkit-overflow-scrolling: touch !important;
+        }
+        
+        /* Badge adjustments */
+        .badge-row {
+            flex-wrap: wrap !important;
+        }
+        
+        .badge {
+            font-size: 0.65rem !important;
+            padding: 3px 8px !important;
+        }
+    }
+    
+    @media (max-width: 480px) {
+        /* Ultra-compact layout for very small screens */
+        .v2-styled-table {
+            font-size: 10px !important;
+        }
+        
+        /* Show only essential columns */
+        .v2-styled-table th:nth-child(n+6),
+        .v2-styled-table td:nth-child(n+6) {
+            display: none !important;
+        }
+        
+        /* Keep only Rank, Model, Type, and main scores visible */
+        .v2-styled-table th:nth-child(5),
+        .v2-styled-table td:nth-child(5) {
+            display: table-cell !important;
+        }
+        
+        /* Reduce all padding */
+        * {
+            padding-left: 8px !important;
+            padding-right: 8px !important;
+        }
+        
+        /* Stack all buttons vertically */
+        .header-action-button {
+            width: 90% !important;
+            margin: 0 auto !important;
+        }
+    }
+    
+    /* Tooltip improvements for mobile */
+    @media (hover: none) and (pointer: coarse) {
+        /* Show tooltips on tap for mobile */
+        .tooltip-trigger {
+            position: relative;
+            cursor: help;
+        }
+        
+        .tooltip-trigger:active .tooltip-content,
+        .tooltip-trigger:focus .tooltip-content {
+            display: block !important;
+        }
+    }
+    </style>
+    """
+
+
+def get_faq_section():
+    """Return the FAQ section HTML"""
+    return """
+    <div class="dark-container" style="margin-top: 40px; margin-bottom: 40px;">
+        <div class="section-header">
+            <span class="section-icon" style="color: var(--accent-primary);">❓</span>
+            <h3 style="margin: 0; color: var(--text-primary); font-size: 1.5rem; font-family: 'Geist', sans-serif; font-weight: 700;">
+                Frequently Asked Questions
+            </h3>
+        </div>
+        
+        <div style="margin-top: 24px;">
+            <!-- FAQ Item 1 -->
+            <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);">
+                <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;">
+                    <span style="color: var(--accent-primary);"></span> Does the methodology favor GPT-4.1 since it uses GPT-4.1 to simulate users and tools, so GPT-4.1 ranks itself highest.
+                </summary>
+                <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;">
+                    <strong style="color: var(--accent-secondary);"></strong> GPT's top ranking isn't due to simulator bias. Scenarios are pre-generated with Claude and fixed for all models. The user simulator drives goal-based conversations, and the tool simulator provides synthetic responses without influencing outcomes. Evaluation uses Claude as a judge, which should theoretically favor Claude (per sycophancy theory), but GPTs still lead.
+                </div>
+            </details>
+            
+            <!-- FAQ Item 2 -->
+            <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);">
+                <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;">
+                    <span style="color: var(--accent-primary);"></span> Why does a specific model rank lower when our internal results show otherwise?
+                </summary>
+                <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;">
+                    <strong style="color: var(--accent-secondary);"></strong> Performance varies by prompt, task, complexity, and domain. Our evaluations kept prompts identical across models for consistency. Different evaluation methodologies and task sets can lead to different rankings.
+                </div>
+            </details>
+            
+            <!-- FAQ Item 3 -->
+            <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);">
+                <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;">
+                    <span style="color: var(--accent-primary);"></span> Why is my favorite model missing?
+                </summary>
+                <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;">
+                    <strong style="color: var(--accent-secondary);"></strong> We were not able to add certain models either because they were not in our initial list or had issues while running the experiments, such as improper tool call output format. We skipped some of the models which performed poorly in our leaderboard v1.
+                </div>
+            </details>
+            
+            <!-- FAQ Item 4 -->
+            <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);">
+                <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;">
+                    <span style="color: var(--accent-primary);"></span> We were surprised Gemini 2.5 Pro ranked lower. Our internal benchmarks show it's excellent for code research and AI code review tasks.
+                </summary>
+                <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;">
+                    <strong style="color: var(--accent-secondary);"></strong> Results differ because this leaderboard evaluates support agent scenarios only, not coding ones. Different models excel at different types of tasks, and this benchmark focuses specifically on business support agent use cases across banking, healthcare, insurance, investment, and telecom domains.
+                </div>
+            </details>
+            
+            <!-- About Metrics -->
+            <div style="margin-top: 32px; padding: 20px; background: #ffd21e0d; border-radius: 12px; border: 1px solid var(--border-default);">
+                <h4 style="color: var(--text-primary); margin-top: 0; margin-bottom: 16px; font-size: 1.2rem; font-family: 'Geist', sans-serif; font-weight: 600; display: flex; align-items: center; gap: 8px;">
+                    <span style="font-size: 1.3rem;">📊</span>
+                    Understanding the Metrics
+                </h4>
+                
+                <div style="display: grid; gap: 16px;">
+                    <div>
+                        <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Action Completion (AC)</h5>
+                        <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;">
+                            A score from 0 to 1 measuring how successfully the agent completes the user's requested tasks. This evaluates whether the agent achieves the intended goals, follows instructions accurately, and provides complete solutions. Higher scores indicate better task completion.
+                        </p>
+                    </div>
+                    
+                    <div>
+                        <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Tool Selection Quality (TSQ)</h5>
+                        <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;">
+                            A score from 0 to 1 evaluating how well the agent selects and uses the appropriate tools for each task. This includes choosing the right tool, using correct parameters, and proper sequencing of tool calls. Higher scores indicate better tool utilization.
+                        </p>
+                    </div>
+                    
+                    <div>
+                        <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Domain-Specific Performance</h5>
+                        <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;">
+                            Models are tested across five business domains: Banking, Healthcare, Insurance, Investment, and Telecom. Each domain has specific scenarios and requirements that test the agent's ability to handle industry-specific tasks and terminology.
+                        </p>
+                    </div>
+                    
+                    <div>
+                        <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Efficiency Metrics</h5>
+                        <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;">
+                            • <strong>Cost:</strong> Total API cost per session in USD<br>
+                            • <strong>Duration:</strong> Time to complete tasks in seconds<br>
+                            • <strong>Turns:</strong> Number of exchanges to reach resolution<br>
+                            These metrics help identify the most cost-effective and efficient models for production use.
+                        </p>
+                    </div>
+                </div>
+                
+                <div style="margin-top: 20px; padding-top: 16px; border-top: 1px solid var(--border-subtle);">
+                    <p style="color: var(--text-secondary); margin: 0; font-size: 0.9rem; line-height: 1.5;">
+                        <strong>Learn More:</strong> For detailed methodology and evaluation criteria, visit the 
+                        <a href="https://galileo.ai/blog/agent-leaderboard-v2" target="_blank" style="color: var(--accent-primary); text-decoration: none;">
+                            official blog post ↗
+                        </a>
+                        or explore the 
+                        <a href="https://github.com/rungalileo/agent-leaderboard" target="_blank" style="color: var(--accent-primary); text-decoration: none;">
+                            GitHub repository ↗
+                        </a>
+                    </p>
+                </div>
+            </div>
+        </div>
+        
+        <style>
+            .faq-item {
+                transition: all 0.3s ease;
+            }
+            
+            .faq-item:hover {
+                border-color: var(--accent-primary) !important;
+                box-shadow: 0 4px 12px rgba(255, 210, 30, 0.1);
+            }
+            
+            .faq-item summary::-webkit-details-marker {
+                display: none;
+            }
+            
+            .faq-item summary::before {
+                content: '▶';
+                display: inline-block;
+                margin-right: 8px;
+                transition: transform 0.3s ease;
+                color: var(--accent-secondary);
+            }
+            
+            .faq-item[open] summary::before {
+                transform: rotate(90deg);
+            }
+            
+            .faq-item summary:hover {
+                color: var(--accent-primary) !important;
+            }
+        </style>
+    </div>
+    """
+
+
+# Column mapping for sorting
+SORT_COLUMN_MAP = {
+    "Avg Action Completion": "Avg AC",
+    "Avg Tool Selection Quality": "Avg TSQ", 
+    "Avg Session Cost": "Avg Total Cost",
+}

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,16 +1,5 @@
-APScheduler
-black
-datasets
-gradio
-gradio[oauth]
-gradio_leaderboard==0.0.13
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
+gradio==5.35.0
 pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+matplotlib
+plotly==5.24.1
+pydantic==2.10.6

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
-    font-size: 16px !important;
-}
-
-#models-to-add-text {
-    font-size: 18px !important;
-}
-
-#citation-button span {
-    font-size: 16px !important;
-}
-
-#citation-button textarea {
-    font-size: 16px !important;
-}
-
-#citation-button > label > button {
-    margin: 6px;
-    transform: scale(1.3);
-}
-
-#leaderboard-table {
-    margin-top: 15px
-}
-
-#leaderboard-table-lite {
-    margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
-    background: none;
-    border: none;
-}
- 
-#search-bar {
-    padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-#leaderboard-table td:nth-child(2),
-#leaderboard-table th:nth-child(2) {
-    max-width: 400px;
-    overflow: auto;
-    white-space: nowrap;
-}
-
-.tab-buttons button {
-    font-size: 20px;
-}
-
-#scale-logo {
-    border-style: none !important;
-    box-shadow: none;
-    display: block;
-    margin-left: auto;
-    margin-right: auto;
-    max-width: 600px;
-}
-
-#scale-logo .download {
-    display: none;
-}
-#filter_type{
-    border: 0;
-    padding-left: 0;
-    padding-top: 0;
-}
-#filter_type label {
-    display: flex;
-}
-#filter_type label > span{
-    margin-top: var(--spacing-lg);
-    margin-right: 0.5em;
-}
-#filter_type label > .wrap{
-    width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{  
-    padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
-    width: 1px
-}
-#filter-columns-type{
-    border:0;
-    padding:0.5;
-}
-#filter-columns-size{
-    border:0;
-    padding:0.5;
-}
-#box-filter > .form{
-    border: 0
-}
-"""
-
-get_window_url_params = """
-    function(url_params) {
-        const params = new URLSearchParams(window.location.search);
-        url_params = Object.fromEntries(params);
-        return url_params;
-    }
-    """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

styles/leaderboard_styles.py

@@ -0,0 +1,397 @@
+"""
+CSS styles for the Agent Leaderboard v2
+This file contains all the styling that doesn't change frequently
+"""
+
+def get_leaderboard_css():
+    """Return the complete CSS for the leaderboard"""
+    return """
+    <style>
+    /* Import Geist fonts */
+    @import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap');
+    
+    @font-face {
+        font-family: 'Geist';
+        src: url('https://raw.githubusercontent.com/vercel/geist-font/main/packages/next/dist/fonts/geist-sans/Geist-Variable.woff2') format('woff2');
+        font-weight: 100 900;
+        font-style: normal;
+    }
+    
+    @font-face {
+        font-family: 'Geist Mono';
+        src: url('https://raw.githubusercontent.com/vercel/geist-font/main/packages/next/dist/fonts/geist-mono/GeistMono-Variable.woff2') format('woff2');
+        font-weight: 100 900;
+        font-style: normal;
+    }
+    
+    /* Root variables for enhanced color scheme */
+    :root {
+        --bg-primary: #01091A;
+        --bg-secondary: rgba(245, 246, 247, 0.03);
+        --bg-card: rgba(245, 246, 247, 0.02);
+        --border-subtle: rgba(245, 246, 247, 0.08);
+        --border-default: rgba(245, 246, 247, 0.12);
+        --border-strong: rgba(245, 246, 247, 0.2);
+        --text-primary: #F5F6F7;
+        --text-secondary: #94A3B8;
+        --text-muted: #64748B;
+        --accent-primary: #ffd21e;
+        --accent-secondary: #1098F7;
+        --accent-tertiary: #F5F6F7;
+        --glow-primary: rgba(255, 210, 30, 0.4);
+        --glow-secondary: rgba(16, 152, 247, 0.4);
+        --glow-tertiary: rgba(245, 246, 247, 0.3);
+    }
+    
+    /* Global font and background */
+    .gradio-container {
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, 'Inter', sans-serif !important;
+        background: var(--bg-primary) !important;
+        color: var(--text-primary) !important;
+    }
+    
+    /* Headers and text */
+    h1, h2, h3, h4 {
+        color: var(--text-primary) !important;
+        font-weight: 700 !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+    }
+    
+    p, span, div {
+        color: var(--text-primary) !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+    }
+    
+    /* Labels and info text */
+    label {
+        color: var(--text-primary) !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+    }
+    
+    .gr-box label {
+        color: var(--text-primary) !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+    }
+    
+    .gr-info {
+        color: var(--text-secondary) !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+    }
+    
+    /* Simple metric cards */
+    .metric-card {
+        background: var(--bg-card);
+        border-radius: 16px;
+        padding: 24px;
+        position: relative;
+        border: 1px solid var(--border-subtle);
+        transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
+    }
+    
+    .metric-card:hover {
+        transform: translateY(-4px);
+        border-color: var(--accent-primary);
+        box-shadow: 0 8px 24px rgba(255, 210, 30, 0.2);
+    }
+    
+    /* Metric icon with glow effect */
+    .metric-icon {
+        width: 48px;
+        height: 48px;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        font-size: 2rem;
+        margin-bottom: 16px;
+        filter: drop-shadow(0 0 20px currentColor);
+        transition: all 0.3s ease;
+    }
+    
+    .metric-card:hover .metric-icon {
+        transform: scale(1.1);
+        filter: drop-shadow(0 0 30px currentColor);
+    }
+    
+    /* Table with tooltips */
+    .v2-styled-table th {
+        position: relative;
+    }
+    
+    .tooltip-trigger {
+        cursor: help;
+        text-decoration: underline;
+        text-decoration-style: dotted;
+        text-underline-offset: 2px;
+        text-decoration-color: var(--accent-secondary);
+    }
+    
+    .tooltip-content {
+        display: none;
+        position: absolute;
+        bottom: 100%;
+        left: 50%;
+        transform: translateX(-50%);
+        background: var(--bg-primary);
+        border: 1px solid var(--border-default);
+        border-radius: 8px;
+        padding: 12px;
+        max-width: 300px;
+        font-size: 0.85rem;
+        color: var(--text-secondary);
+        z-index: 1000;
+        white-space: normal;
+        box-shadow: 0 8px 24px rgba(0, 0, 0, 0.4);
+        margin-bottom: 8px;
+    }
+    
+    .tooltip-trigger:hover .tooltip-content {
+        display: block;
+    }
+    
+    /* Enhanced radio buttons with primary accent */
+    input[type="radio"] {
+        background-color: var(--bg-secondary) !important;
+        border-color: var(--border-default) !important;
+    }
+    
+    input[type="radio"]:checked {
+        background-color: var(--accent-primary) !important;
+        border-color: var(--accent-primary) !important;
+        box-shadow: 0 0 10px var(--glow-primary) !important;
+    }
+    
+    /* Enhanced dropdown styling */
+    .dropdown {
+        border-color: var(--border-default) !important;
+        background: var(--bg-card) !important;
+        color: var(--text-primary) !important;
+        transition: all 0.2s ease !important;
+    }
+    
+    .dropdown:hover {
+        border-color: var(--accent-primary) !important;
+        box-shadow: 0 0 15px var(--glow-primary) !important;
+    }
+    
+    /* Enhanced table styling */
+    .dataframe {
+        background: var(--bg-card) !important;
+        border-radius: 16px !important;
+        overflow: hidden !important;
+        border: 1px solid var(--border-subtle) !important;
+        font-size: 15px !important;
+        max-height: 600px !important;
+        overflow-y: auto !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+        box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3) !important;
+    }
+    
+    /* Button styling */
+    button {
+        background: var(--bg-card) !important;
+        color: var(--text-primary) !important;
+        border: 1px solid var(--border-default) !important;
+        transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
+    }
+    
+    button:hover {
+        transform: translateY(-2px) !important;
+        border-color: var(--accent-primary) !important;
+        box-shadow: 0 4px 16px rgba(255, 210, 30, 0.2) !important;
+    }
+    
+    /* Info boxes */
+    .info-box {
+        background: var(--bg-card);
+        border: 1px solid var(--border-subtle);
+        border-radius: 12px;
+        padding: 20px;
+        margin: 8px 0;
+        backdrop-filter: blur(10px);
+        position: relative;
+        overflow: hidden;
+        transition: all 0.3s ease;
+    }
+    
+    .info-box:hover {
+        border-color: var(--accent-primary);
+        box-shadow: 0 4px 20px var(--glow-primary);
+    }
+    
+    /* Dark containers */
+    .dark-container {
+        background: var(--bg-card);
+        border: 1px solid var(--border-subtle);
+        border-radius: 20px;
+        padding: 28px;
+        box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4);
+        backdrop-filter: blur(10px);
+        position: relative;
+        overflow: hidden;
+    }
+    
+    /* Section headers */
+    .section-header {
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        margin-bottom: 24px;
+    }
+    
+    .section-icon {
+        filter: drop-shadow(0 0 12px currentColor);
+        transition: all 0.3s ease;
+    }
+    
+    /* Scrollbar styling */
+    ::-webkit-scrollbar {
+        width: 8px;
+        height: 8px;
+    }
+    
+    ::-webkit-scrollbar-track {
+        background: var(--bg-secondary);
+        border-radius: 4px;
+    }
+    
+    ::-webkit-scrollbar-thumb {
+        background: var(--accent-secondary);
+        border-radius: 4px;
+    }
+    
+    ::-webkit-scrollbar-thumb:hover {
+        background: var(--accent-primary);
+    }
+    
+    /* Pulse animation */
+    @keyframes pulse-glow {
+        0% { box-shadow: 0 0 0 0 var(--glow-primary); }
+        70% { box-shadow: 0 0 0 10px transparent; }
+        100% { box-shadow: 0 0 0 0 transparent; }
+    }
+    
+    .pulse {
+        animation: pulse-glow 2s infinite;
+    }
+    
+    /* Chart containers */
+    .chart-container {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        width: 100%;
+        margin: 0 auto;
+    }
+    
+    .chart-container > div {
+        width: 100%;
+        max-width: 1400px;
+        margin: 0 auto;
+    }
+    
+    /* Radar chart - remove all boundaries */
+    .radar-chart-container {
+        width: fit-content !important;
+        margin: 0 auto !important;
+        padding: 0 !important;
+    }
+    
+    .radar-chart-container > div,
+    .radar-chart,
+    .radar-chart .gradio-plot {
+        width: fit-content !important;
+        max-width: none !important;
+        margin: 0 auto !important;
+        padding: 0 !important;
+    }
+    
+    
+    /* Grid layouts for cards */
+    .insight-card-grid {
+        display: grid;
+        grid-template-columns: repeat(5, 1fr);
+        gap: 16px;
+    }
+    
+    .metric-card-grid {
+        display: grid;
+        grid-template-columns: repeat(3, 1fr);
+        gap: 16px;
+    }
+    
+    /* Custom button container */
+    .custom-button-container {
+        text-align: center;
+        padding: 20px 0 10px 0;
+        margin-bottom: 10px;
+    }
+    
+    .header-action-button {
+        display: inline-block !important;
+        padding: 14px 28px !important;
+        background: #ffd21e !important;
+        color: #FFFFFF !important;
+        text-decoration: none !important;
+        border-radius: 16px !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+        font-weight: 700 !important;
+        font-size: 1.1rem !important;
+        transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important;
+        border: none !important;
+        cursor: pointer !important;
+        box-shadow: 0 8px 24px rgba(255, 210, 30, 0.4), 0 4px 12px rgba(0, 0, 0, 0.3) !important;
+        position: relative !important;
+        overflow: hidden !important;
+        text-shadow: 0 1px 2px rgba(0, 0, 0, 0.35) !important;
+    }
+    
+    .header-action-button:hover {
+        transform: translateY(-3px) !important;
+        box-shadow: 0 12px 32px rgba(255, 210, 30, 0.5), 0 8px 16px rgba(0, 0, 0, 0.4) !important;
+        background: #ffd21e !important;
+        color: #FFFFFF !important;
+        text-decoration: none !important;
+        text-shadow: 0 2px 6px rgba(0, 0, 0, 0.45) !important;
+    }
+    
+    /* Navigation buttons */
+    .nav-buttons-container {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        gap: 16px;
+        flex-wrap: wrap;
+        margin: 24px 0;
+        padding: 0 20px;
+    }
+    
+    .nav-link-button {
+        display: inline-flex !important;
+        align-items: center !important;
+        gap: 8px !important;
+        padding: 12px 20px !important;
+        background: rgba(1, 9, 26, 0.8) !important;
+        color: #F5F6F7 !important;
+        text-decoration: none !important;
+        border-radius: 12px !important;
+        font-family: 'Geist', -apple-system, BlinkMacSystemFont, sans-serif !important;
+        font-weight: 600 !important;
+        font-size: 0.95rem !important;
+        transition: all 0.3s ease !important;
+        border: 2px solid rgba(245, 246, 247, 0.15) !important;
+        backdrop-filter: blur(10px) !important;
+        -webkit-backdrop-filter: blur(10px) !important;
+        position: relative !important;
+        overflow: hidden !important;
+        box-shadow: 0 4px 16px rgba(0, 0, 0, 0.3) !important;
+    }
+    
+    .nav-link-button:hover {
+        transform: translateY(-3px) scale(1.02) !important;
+        border-color: #ffd21e !important;
+        box-shadow: 0 8px 24px rgba(255, 210, 30, 0.3), 0 4px 12px rgba(0, 0, 0, 0.4) !important;
+        text-decoration: none !important;
+        color: #FFFFFF !important;
+    }
+    </style>
+    """

utils.py

@@ -0,0 +1,149 @@
+def get_chart_colors():
+    # if is_dark_theme():
+    #     return {
+    #         "Private": "#60A5FA",  # accent-blue
+    #         "Open source": "#A78BFA",  # accent-purple
+    #         "performance_bands": ["#DCFCE7", "#FEF9C3", "#FEE2E2"],
+    #         "text": "#FFFFFF",
+    #         "background": "#1a1b1e",
+    #         "grid": (1, 1, 1, 0.1),  # RGBA tuple for grid
+    #     }
+    return {
+        "Private": "#3F78FA",  # accent-blue light
+        "Open source": "#A13AE2",  # accent-purple light
+        "performance_bands": ["#DCFCE7", "#FEF9C3", "#FEE2E2"],
+        "text": "#111827",
+        "background": "#FFFFFF",
+        "grid": (0, 0, 0, 0.1),  # RGBA tuple for grid
+    }
+
+
+def get_rank_badge(rank):
+    """Generate HTML for rank badge with appropriate styling"""
+    badge_styles = {
+        1: ("1st", "linear-gradient(145deg, #ffd700, #ffc400)", "#000"),
+        2: ("2nd", "linear-gradient(145deg, #9ca3af, #787C7E)", "#fff"),
+        3: ("3rd", "linear-gradient(145deg, #CD7F32, #b36a1d)", "#fff"),
+    }
+
+    if rank in badge_styles:
+        label, gradient, text_color = badge_styles[rank]
+        return f"""
+            <div style="
+                display: inline-flex;
+                align-items: center;
+                justify-content: center;
+                min-width: 48px;
+                padding: 4px 12px;
+                background: {gradient};
+                color: {text_color};
+                border-radius: 6px;
+                font-weight: 600;
+                font-size: 0.9em;
+                box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
+            ">
+                {label}
+            </div>
+        """
+    return f"""
+        <div style="
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            min-width: 28px;
+            color: #a1a1aa;
+            font-weight: 500;
+        ">
+            {rank}
+        </div>
+    """
+
+
+def get_type_badge(model_type):
+    """Generate HTML for model type badge"""
+    colors = get_chart_colors()
+    color_map = {
+        "Open source": colors.get("Open source", "#A13AE2"),
+        "Proprietary": colors.get("Private", "#3F78FA"),
+        "Private": colors.get("Private", "#3F78FA"),
+    }
+    label_map = {
+        "Open source": "OSS",
+        "Proprietary": "API",
+        "Private": "API",
+    }
+    bg_color = color_map.get(model_type, "#4F46E5")
+    display_label = label_map.get(model_type, model_type)
+    return f"""
+        <div style="
+            display: inline-flex;
+            align-items: center;
+            padding: 4px 8px;
+            background: {bg_color};
+            color: white;
+            border-radius: 4px;
+            font-size: 0.85em;
+            font-weight: 500;
+        ">
+            {display_label}
+        </div>
+    """
+
+
+def get_score_bar(score):
+    """Generate HTML for score bar with gradient styling"""
+    width = score * 100
+    return f"""
+        <div style="display: flex; align-items: center; gap: 12px; width: 100%;">
+            <div style="
+                flex-grow: 1;
+                height: 8px;
+                background: var(--score-bg, rgba(255, 255, 255, 0.1));
+                border-radius: 4px;
+                overflow: hidden;
+                max-width: 200px;
+            ">
+                <div style="
+                    width: {width}%;
+                    height: 100%;
+                    background: linear-gradient(90deg, var(--accent-blue, #60A5FA), var(--accent-purple, #A78BFA));
+                    border-radius: 4px;
+                    transition: width 0.3s ease;
+                "></div>
+            </div>
+            <span style="
+                font-family: 'SF Mono', monospace;
+                font-weight: 600;
+                color: var(--text-primary, #ffffff);
+                min-width: 60px;
+            ">{score:.3f}</span>
+        </div>
+    """
+
+
+def get_output_type_badge(output_type):
+    """Generate HTML for output type badges with different colors, supporting both light and dark themes"""
+    type_styles = {
+        "Normal": {
+            "light": {"bg": "#F3F4F6", "color": "#374151"},
+            "dark": {"bg": "#374151", "color": "#F3F4F6"},
+        },
+        "Reasoning": {
+            "light": {"bg": "#DBEAFE", "color": "#1E40AF"},
+            "dark": {"bg": "#1E40AF", "color": "#DBEAFE"},
+        },
+    }
+
+    style = type_styles.get(output_type, type_styles["Normal"])
+    return f"""
+        <span style="
+            background: var(--bg-color, {style['light']['bg']});
+            color: var(--text-color, {style['light']['color']});
+            padding: 4px 8px;
+            border-radius: 4px;
+            font-size: 0.875rem;
+            font-weight: 500;
+        ">
+            {output_type}
+        </span>
+    """

visualization.py

@@ -0,0 +1,258 @@
+from utils import get_chart_colors
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import plotly.graph_objects as go
+
+
+def setup_matplotlib():
+    matplotlib.use("Agg")
+    plt.close("all")
+
+
+def get_performance_chart(df, category_name="Overall"):
+    plt.close("all")
+    colors = get_chart_colors()
+    score_column = "Category Score"
+    df_sorted = df.sort_values(score_column, ascending=True)
+
+    height = max(8, len(df_sorted) * 0.8)
+    fig, ax = plt.subplots(figsize=(16, height))
+    plt.rcParams.update({"font.size": 12})
+
+    fig.patch.set_facecolor(colors["background"])
+    ax.set_facecolor(colors["background"])
+
+    try:
+        bars = ax.barh(
+            np.arange(len(df_sorted)),
+            df_sorted[score_column],
+            height=0.4,
+            capstyle="round",
+            color=[colors[t] for t in df_sorted["Model Type"]],
+        )
+
+        ax.set_title(
+            f"Model Performance - {category_name}",
+            pad=20,
+            fontsize=20,
+            fontweight="bold",
+            color=colors["text"],
+        )
+        ax.set_xlabel(
+            "Average Score (Tool Selection Quality)",
+            fontsize=14,
+            fontweight="bold",
+            labelpad=10,
+            color=colors["text"],
+        )
+        ax.set_xlim(0.0, 1.0)
+
+        ax.set_yticks(np.arange(len(df_sorted)))
+        ax.set_yticklabels(
+            df_sorted["Model"], fontsize=12, fontweight="bold", color=colors["text"]
+        )
+
+        plt.subplots_adjust(left=0.35)
+
+        for i, v in enumerate(df_sorted[score_column]):
+            ax.text(
+                v + 0.01,
+                i,
+                f"{v:.3f}",
+                va="center",
+                fontsize=12,
+                fontweight="bold",
+                color=colors["text"],
+            )
+
+        ax.grid(True, axis="x", linestyle="--", alpha=0.2, color=colors["grid"])
+        ax.spines[["top", "right"]].set_visible(False)
+        ax.spines[["bottom", "left"]].set_color(colors["grid"])
+        ax.tick_params(colors=colors["text"])
+
+        legend_elements = [
+            plt.Rectangle((0, 0), 1, 1, facecolor=color, label=label)
+            for label, color in {
+                k: colors[k] for k in ["Private", "Open source"]
+            }.items()
+        ]
+        ax.legend(
+            handles=legend_elements,
+            title="Model Type",
+            loc="lower right",
+            fontsize=12,
+            title_fontsize=14,
+            facecolor=colors["background"],
+            labelcolor=colors["text"],
+        )
+
+        plt.tight_layout()
+        return fig
+    finally:
+        plt.close(fig)
+
+def create_radar_plot(df, model_names):
+    datasets = [col for col in df.columns[7:] if col != "IO Cost"]
+    fig = go.Figure()
+
+    colors = ["rgba(99, 102, 241, 0.3)", "rgba(34, 197, 94, 0.3)"]
+    line_colors = ["#4F46E5", "#16A34A"]
+
+    for idx, model_name in enumerate(model_names):
+        model_data = df[df["Model"] == model_name].iloc[0]
+        values = [model_data[m] for m in datasets]
+        values.append(values[0])
+        datasets_plot = datasets + [datasets[0]]
+
+        fig.add_trace(
+            go.Scatterpolar(
+                r=values,
+                theta=datasets_plot,
+                fill="toself",
+                fillcolor=colors[idx % len(colors)],
+                line=dict(color=line_colors[idx % len(line_colors)], width=2),
+                name=model_name,
+                text=[f"{val:.3f}" for val in values],
+                textposition="middle right",
+                mode="lines+markers+text",
+            )
+        )
+
+    fig.update_layout(
+        polar=dict(
+            radialaxis=dict(
+                visible=True, range=[0, 1], showline=False, tickfont=dict(size=12)
+            ),
+            angularaxis=dict(
+                tickfont=dict(size=13, family="Arial"),
+                rotation=90,
+                direction="clockwise",
+            ),
+            domain=dict(x=[0.1, 0.9], y=[0.1, 0.9])
+        ),
+        showlegend=True,
+        legend=dict(
+            orientation="h",
+            yanchor="bottom",
+            y=-0.15,
+            xanchor="center",
+            x=0.5,
+            font=dict(size=14),
+        ),
+        title=dict(
+            text="Model Comparison",
+            x=0.5,
+            y=0.95,
+            font=dict(size=24, family="Arial", color="#1F2937"),
+        ),
+        paper_bgcolor="white",
+        plot_bgcolor="white",
+        height=800,
+        width=900,
+        margin=dict(t=30, b=50, l=10, r=10),
+        autosize=True,
+    )
+
+    return fig
+
+
+def get_performance_cost_chart(df, category_name="Overall"):
+    colors = get_chart_colors()
+    fig, ax = plt.subplots(figsize=(12, 8), dpi=300)
+
+    fig.patch.set_facecolor(colors["background"])
+    ax.set_facecolor(colors["background"])
+    ax.grid(True, linestyle="--", alpha=0.15, which="both", color=colors["grid"])
+
+    score_column = "Category Score"
+
+    for _, row in df.iterrows():
+        color = colors[row["Model Type"]]
+        size = 100 if row[score_column] > 0.85 else 80
+        edge_color = (
+            colors["Private"]
+            if row["Model Type"] == "Private"
+            else colors["Open source"]
+        )
+
+        ax.scatter(
+            row["IO Cost"],
+            row[score_column] * 100,
+            c=color,
+            s=size,
+            alpha=0.9,
+            edgecolor=edge_color,
+            linewidth=1,
+            zorder=5,
+        )
+
+        bbox_props = dict(
+            boxstyle="round,pad=0.3", fc=colors["background"], ec="none", alpha=0.8
+        )
+
+        ax.annotate(
+            f"{row['Model']}\n(${row['IO Cost']:.2f})",
+            (row["IO Cost"], row[score_column] * 100),
+            xytext=(5, 5),
+            textcoords="offset points",
+            fontsize=8,
+            fontweight="bold",
+            color=colors["text"],
+            bbox=bbox_props,
+            zorder=6,
+        )
+
+    ax.set_xscale("log")
+    ax.set_xlim(0.08, 1000)
+    ax.set_ylim(60, 100)
+
+    ax.set_xlabel(
+        "I/O Cost per Million Tokens ($)",
+        fontsize=10,
+        fontweight="bold",
+        labelpad=10,
+        color=colors["text"],
+    )
+    ax.set_ylabel(
+        "Model Performance Score",
+        fontsize=10,
+        fontweight="bold",
+        labelpad=10,
+        color=colors["text"],
+    )
+
+    legend_elements = [
+        plt.scatter([], [], c=colors[label], label=label, s=80)
+        for label in ["Private", "Open source"]
+    ]
+    ax.legend(
+        handles=legend_elements,
+        loc="upper right",
+        frameon=True,
+        facecolor=colors["background"],
+        edgecolor="none",
+        fontsize=9,
+        labelcolor=colors["text"],
+    )
+
+    ax.set_title(
+        f"Performance vs. Cost - {category_name}",
+        fontsize=14,
+        pad=15,
+        fontweight="bold",
+        color=colors["text"],
+    )
+
+    for y1, y2, color in zip([85, 75, 60], [100, 85, 75], colors["performance_bands"]):
+        ax.axhspan(y1, y2, alpha=0.2, color=color, zorder=1)
+
+    ax.tick_params(axis="both", which="major", labelsize=9, colors=colors["text"])
+    ax.tick_params(axis="both", which="minor", labelsize=8, colors=colors["text"])
+    ax.xaxis.set_minor_locator(plt.LogLocator(base=10.0, subs=np.arange(2, 10) * 0.1))
+
+    for spine in ax.spines.values():
+        spine.set_color(colors["grid"])
+
+    plt.tight_layout()
+    return fig