Add agg

app.py

@@ -13,7 +13,7 @@ Evaluation of H4 and community models across a diverse range of benchmarks from
 BENCHMARKS_TO_SKIP = ["math", "mini_math"]
 
 
-def get_leaderboard_df(merge_values: bool = True):
+def get_leaderboard_df(agg: str = "max"):
     filepaths = list(Path("eval_results").rglob("*.json"))
 
     # Parse filepaths to get unique models
@@ -29,9 +29,9 @@ def get_leaderboard_df(merge_values: bool = True):
     # Extract data from each file and populate the DataFrame
     for filepath in filepaths:
         path_parts = Path(filepath).parts
-        date = filepath.stem.split("_")[-1][:-3].split("T")[0]
+        date = filepath.stem.split("_")[-1][:-3]
         model_revision = "_".join(path_parts[1:4]) + "_" + date
-        task = path_parts[4]  # .capitalize()
+        task = path_parts[4]
         df.loc[model_revision, "Date"] = date
 
         with open(filepath, "r") as file:
@@ -115,13 +115,14 @@ def get_leaderboard_df(merge_values: bool = True):
 
     # Drop rows where every entry is NaN
     df = df.dropna(how="all", axis=0, subset=[c for c in df.columns if c != "Date"])
+
     # Trim minimath column names
     df.columns = [c.replace("_level_", "_l") for c in df.columns]
 
     # Trim AIMO column names
     df.columns = [c.replace("aimo_", "") for c in df.columns]
 
-    df.insert(loc=1, column="Average", value=df.mean(axis=1, numeric_only=True))
+    df.insert(loc=0, column="Average", value=df.mean(axis=1, numeric_only=True))
 
     # Convert all values to percentage
     df[df.select_dtypes(include=["number"]).columns] *= 100.0
@@ -130,22 +131,19 @@ def get_leaderboard_df(merge_values: bool = True):
     # Strip off date from model name
     df["Model"] = df["Model"].apply(lambda x: x.rsplit("_", 1)[0])
 
-    if merge_values:
-        merged_df = df.drop(["Date", "Average"], axis=1).groupby("Model").max().reset_index()
-        merged_df.insert(loc=0, column="Average", value=merged_df.mean(axis=1, numeric_only=True))
-        df = df[["Model", "Date"]].merge(merged_df, on="Model", how="left")
-        df.drop_duplicates(subset=["Model"], inplace=True)
-        df = df.sort_values(by=["Average"], ascending=False).round(2)
+    # Drop date and aggregate results by model name
+    df = df.drop("Date", axis=1).groupby("Model").agg(agg).reset_index()
+
     return df
 
 
-def refresh(merge_values: bool = True):
-    return get_leaderboard_df(merge_values)
+def refresh(agg: str = "max"):
+    return get_leaderboard_df(agg=agg)
 
 
 # Function to update the table based on search query
-def update_table(search_query):
-    df = get_leaderboard_df()
+def update_table(search_query, agg):
+    df = get_leaderboard_df(agg)
     if search_query:
         search_terms = search_query.split(";")
         search_terms = [term.strip().lower() for term in search_terms]
@@ -157,14 +155,14 @@ def update_table(search_query):
 
 
 def filter_columns(cols):
-    index_cols = list(leaderboard_df.columns[:2])
+    index_cols = list(leaderboard_df.columns[:1])
     new_cols = index_cols + cols
     df = get_leaderboard_df()
     df = df.copy()[new_cols]
     # Drop rows with NaN values
     df = df.copy().dropna(how="all", axis=0, subset=[c for c in df.columns if c in cols])
     # Recompute average
-    df.insert(loc=2, column="Average", value=df.mean(axis=1, numeric_only=True))
+    df.insert(loc=1, column="Average", value=df.mean(axis=1, numeric_only=True))
     return df
 
 
@@ -178,14 +176,15 @@ with demo:
         gr.Markdown(DESCRIPTION, elem_classes="markdown-text")
         with gr.Row():
             search_bar = gr.Textbox(placeholder="Search for your model...", show_label=False)
-            merge_values = gr.Checkbox(
-                value=True,
-                label="Merge evals",
-                info="Merge evals for the same model. If there are duplicates, we display the largest one.",
+            agg = gr.Radio(
+                ["min", "max", "mean"],
+                value="max",
+                label="Aggregation",
+                info="How to aggregate results for each model",
             )
         with gr.Row():
             cols_bar = gr.CheckboxGroup(
-                choices=[c for c in leaderboard_df.columns[2:] if c != "Average"],
+                choices=[c for c in leaderboard_df.columns[1:] if c != "Average"],
                 show_label=False,
                 info="Select columns to display",
             )
@@ -195,14 +194,14 @@ with demo:
                 value=leaderboard_df,
                 wrap=True,
                 height=1000,
-                column_widths=[400, 110] + [(220 + len(c)) for c in leaderboard_df.columns[2:]],
+                column_widths=[400, 110] + [(220 + len(c)) for c in leaderboard_df.columns[1:]],
             )
         with gr.Row():
             refresh_button = gr.Button("Refresh")
 
     cols_bar.change(filter_columns, inputs=[cols_bar], outputs=[leaderboard_table])
-    merge_values.change(refresh, inputs=[merge_values], outputs=[leaderboard_table])
-    search_bar.submit(update_table, inputs=[search_bar], outputs=[leaderboard_table])
+    agg.change(refresh, inputs=[agg], outputs=[leaderboard_table])
+    search_bar.submit(update_table, inputs=[search_bar, agg], outputs=[leaderboard_table])
     refresh_button.click(refresh, inputs=[], outputs=[leaderboard_table])
 
 demo.launch()

debug.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -10,16 +10,18 @@
     "from pathlib import Path\n",
     "\n",
     "import gradio as gr\n",
-    "import pandas as pd"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def get_leaderboard_df():\n",
+    "import pandas as pd\n",
+    "\n",
+    "TITLE = \"\"\"<h1 align=\"center\" id=\"space-title\">LLM Leaderboard for H4 Models</h1>\"\"\"\n",
+    "\n",
+    "DESCRIPTION = f\"\"\"\n",
+    "Evaluation of H4 and community models across a diverse range of benchmarks from [LightEval](https://github.com/huggingface/lighteval). All scores are reported as accuracy.\n",
+    "\"\"\"\n",
+    "\n",
+    "BENCHMARKS_TO_SKIP = [\"math\", \"mini_math\"]\n",
+    "\n",
+    "\n",
+    "def get_leaderboard_df(agg : str = \"max\"):\n",
     "    filepaths = list(Path(\"eval_results\").rglob(\"*.json\"))\n",
     "\n",
     "    # Parse filepaths to get unique models\n",
@@ -35,14 +37,17 @@
     "    # Extract data from each file and populate the DataFrame\n",
     "    for filepath in filepaths:\n",
     "        path_parts = Path(filepath).parts\n",
-    "        date = filepath.stem.split(\"_\")[-1][:-3].split(\"T\")[0]\n",
+    "        date = filepath.stem.split(\"_\")[-1][:-3]\n",
     "        model_revision = \"_\".join(path_parts[1:4]) + \"_\" + date\n",
-    "        task = path_parts[4].capitalize()\n",
+    "        task = path_parts[4]\n",
     "        df.loc[model_revision, \"Date\"] = date\n",
     "\n",
     "        with open(filepath, \"r\") as file:\n",
     "            data = json.load(file)\n",
     "            first_result_key = next(iter(data[\"results\"]))  # gets the first key in 'results'\n",
+    "            # Skip benchmarks that we don't want to include in the leaderboard\n",
+    "            if task.lower() in BENCHMARKS_TO_SKIP:\n",
+    "                continue\n",
     "            # TruthfulQA has two metrics, so we need to pick the `mc2` one that's reported on the leaderboard\n",
     "            if task.lower() == \"truthfulqa\":\n",
     "                value = data[\"results\"][first_result_key][\"truthfulqa_mc2\"]\n",
@@ -51,44 +56,116 @@
     "                value = data[\"results\"][first_result_key][\"prompt_level_loose_acc\"]\n",
     "            # MMLU has several metrics but we report just the average one\n",
     "            elif task.lower() == \"mmlu\":\n",
-    "                value = data[\"results\"][\"lighteval|mmlu:_average|5\"][\"acc\"]\n",
+    "                value = [v[\"acc\"] for k, v in data[\"results\"].items() if \"_average\" in k.lower()][0]\n",
     "            # HellaSwag and ARC reports acc_norm\n",
     "            elif task.lower() in [\"hellaswag\", \"arc\"]:\n",
     "                value = data[\"results\"][first_result_key][\"acc_norm\"]\n",
+    "            # BBH has several metrics but we report just the average one\n",
+    "            elif task.lower() == \"bbh\":\n",
+    "                if \"all\" in data[\"results\"]:\n",
+    "                    value = data[\"results\"][\"all\"][\"acc\"]\n",
+    "                else:\n",
+    "                    value = -100\n",
+    "            # AGIEval reports acc_norm\n",
+    "            elif task.lower() == \"agieval\":\n",
+    "                value = data[\"results\"][\"all\"][\"acc_norm\"]\n",
+    "            # MATH reports qem\n",
+    "            elif task.lower() in [\"math\", \"math_v2\", \"aimo_kaggle\"]:\n",
+    "                value = data[\"results\"][\"all\"][\"qem\"]\n",
     "            else:\n",
     "                first_metric_key = next(\n",
     "                    iter(data[\"results\"][first_result_key])\n",
     "                )  # gets the first key in the first result\n",
     "                value = data[\"results\"][first_result_key][first_metric_key]  # gets the value of the first metric\n",
-    "            df.loc[model_revision, task] = value\n",
     "\n",
-    "    # Put IFEval in first column\n",
-    "    ifeval_col = df.pop(\"Ifeval\")\n",
-    "    df.insert(1, \"Ifeval\", ifeval_col)\n",
+    "            # For mini_math we report 5 metrics, one for each level and store each one as a separate row in the dataframe\n",
+    "            if task.lower() in [\"mini_math_v2\"]:\n",
+    "                for k, v in data[\"results\"].items():\n",
+    "                    if k != \"all\":\n",
+    "                        level = k.split(\"|\")[1].split(\":\")[-1]\n",
+    "                        value = v[\"qem\"]\n",
+    "                        df.loc[model_revision, f\"{task}_{level}\"] = value\n",
+    "            # For kaggle_pot we report N metrics, one for each prompt and store each one as a separate row in the dataframe\n",
+    "            elif task.lower() in [\"aimo_kaggle_medium_pot\"]:\n",
+    "                for k, v in data[\"results\"].items():\n",
+    "                    if k != \"all\" and \"_average\" not in k:\n",
+    "                        version = k.split(\"|\")[1].split(\":\")[-1]\n",
+    "                        value = v[\"qem\"] if \"qem\" in v else v[\"score\"]\n",
+    "                        df.loc[model_revision, f\"{task}_{version}\"] = value\n",
+    "            # For kaggle_pot we report N metrics, one for each prompt and store each one as a separate row in the dataframe\n",
+    "            elif task.lower() in [\"aimo_kaggle_hard_pot\"]:\n",
+    "                for k, v in data[\"results\"].items():\n",
+    "                    if k != \"all\" and \"_average\" not in k:\n",
+    "                        version = k.split(\"|\")[1].split(\":\")[-1]\n",
+    "                        value = v[\"qem\"] if \"qem\" in v else v[\"score\"]\n",
+    "                        df.loc[model_revision, f\"{task}_{version}\"] = value\n",
+    "            # For kaggle_tora we report accuracy, so need  to divide by 100\n",
+    "            elif task.lower() in [\n",
+    "                \"aimo_tora_eval_kaggle_medium\",\n",
+    "                \"aimo_tora_eval_kaggle_hard\",\n",
+    "                \"aimo_kaggle_fast_eval_hard\",\n",
+    "                \"aimo_kaggle_tora_medium\",\n",
+    "                \"aimo_kaggle_tora_hard\",\n",
+    "                \"aimo_kaggle_tora_medium_extended\",\n",
+    "                \"aimo_kaggle_tora_hard_extended\",\n",
+    "            ]:\n",
+    "                for k, v in data[\"results\"].items():\n",
+    "                    value = float(v[\"qem\"]) / 100.0\n",
+    "                    df.loc[model_revision, f\"{task}\"] = value\n",
+    "            # For AlpacaEval we report base winrate and lenght corrected one\n",
+    "            elif task.lower() == \"alpaca_eval\":\n",
+    "                value = data[\"results\"][first_result_key][\"win_rate\"]\n",
+    "                df.loc[model_revision, \"Alpaca_eval\"] = value / 100.0\n",
+    "                value = data[\"results\"][first_result_key][\"length_controlled_winrate\"]\n",
+    "                df.loc[model_revision, \"Alpaca_eval_lc\"] = value / 100.0\n",
+    "            else:\n",
+    "                df.loc[model_revision, task] = float(value)\n",
+    "\n",
     "    # Drop rows where every entry is NaN\n",
     "    df = df.dropna(how=\"all\", axis=0, subset=[c for c in df.columns if c != \"Date\"])\n",
-    "    df.insert(loc=1, column=\"Average\", value=df.mean(axis=1, numeric_only=True))\n",
+    "\n",
+    "    # Trim minimath column names\n",
+    "    df.columns = [c.replace(\"_level_\", \"_l\") for c in df.columns]\n",
+    "\n",
+    "    # Trim AIMO column names\n",
+    "    df.columns = [c.replace(\"aimo_\", \"\") for c in df.columns]\n",
+    "\n",
+    "    df.insert(loc=0, column=\"Average\", value=df.mean(axis=1, numeric_only=True))\n",
+    "\n",
     "    # Convert all values to percentage\n",
     "    df[df.select_dtypes(include=[\"number\"]).columns] *= 100.0\n",
     "    df = df.sort_values(by=[\"Average\"], ascending=False)\n",
     "    df = df.reset_index().rename(columns={\"index\": \"Model\"}).round(2)\n",
     "    # Strip off date from model name\n",
     "    df[\"Model\"] = df[\"Model\"].apply(lambda x: x.rsplit(\"_\", 1)[0])\n",
+    "\n",
+    "    # Drop date and aggregate results by model name\n",
+    "    df = df.drop(\"Date\", axis=1).groupby(\"Model\").agg(agg).reset_index()\n",
+    "\n",
     "    return df"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = get_leaderboard_df(agg='mean')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = get_leaderboard_df()"
+    "# df"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 40,
    "metadata": {},
    "outputs": [
     {
@@ -113,208 +190,43 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Model</th>\n",
-       "      <th>Date</th>\n",
        "      <th>Average</th>\n",
-       "      <th>Ifeval</th>\n",
-       "      <th>Truthfulqa</th>\n",
-       "      <th>Winogrande</th>\n",
-       "      <th>Gsm8k</th>\n",
-       "      <th>Mmlu</th>\n",
-       "      <th>Hellaswag</th>\n",
-       "      <th>Arc</th>\n",
+       "      <th>kaggle_tora_medium_extended</th>\n",
+       "      <th>kaggle_tora_hard_extended</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>NousResearch_Nous-Hermes-2-Yi-34B_main</td>\n",
-       "      <td>2024-03-04</td>\n",
-       "      <td>74.01</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>61.44</td>\n",
-       "      <td>80.58</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>76.24</td>\n",
-       "      <td>83.79</td>\n",
-       "      <td>68.00</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>deepseek-ai_deepseek-llm-67b-chat_main</td>\n",
-       "      <td>2024-03-05</td>\n",
-       "      <td>71.62</td>\n",
-       "      <td>55.27</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>76.12</td>\n",
-       "      <td>71.18</td>\n",
-       "      <td>83.94</td>\n",
-       "      <td>NaN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>NousResearch_Nous-Hermes-2-Mixtral-8x7B-DPO_main</td>\n",
-       "      <td>2024-03-02</td>\n",
-       "      <td>70.43</td>\n",
-       "      <td>59.33</td>\n",
-       "      <td>64.76</td>\n",
-       "      <td>78.53</td>\n",
-       "      <td>62.17</td>\n",
-       "      <td>71.96</td>\n",
-       "      <td>85.42</td>\n",
-       "      <td>70.82</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>mistralai_Mixtral-8x7B-Instruct-v0.1_main</td>\n",
-       "      <td>2024-03-02</td>\n",
-       "      <td>69.80</td>\n",
-       "      <td>55.08</td>\n",
-       "      <td>70.79</td>\n",
-       "      <td>73.56</td>\n",
-       "      <td>59.89</td>\n",
-       "      <td>70.60</td>\n",
-       "      <td>86.68</td>\n",
-       "      <td>72.01</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>deepseek-ai_deepseek-llm-67b-chat_main</td>\n",
-       "      <td>2024-03-04</td>\n",
-       "      <td>67.03</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>57.78</td>\n",
-       "      <td>79.16</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>64.16</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>269</th>\n",
-       "      <td>HuggingFaceH4_starcoder2-15b-ift_v18.0</td>\n",
-       "      <td>2024-03-10</td>\n",
-       "      <td>11.23</td>\n",
-       "      <td>21.63</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>0.83</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>270</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v49.0</td>\n",
-       "      <td>2024-03-07</td>\n",
-       "      <td>10.07</td>\n",
-       "      <td>20.15</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>0.00</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>271</th>\n",
-       "      <td>HuggingFaceH4_starchat-beta_main</td>\n",
-       "      <td>2024-03-12</td>\n",
-       "      <td>8.13</td>\n",
-       "      <td>8.13</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>272</th>\n",
-       "      <td>HuggingFaceH4_starcoder2-15b-ift_v7.0</td>\n",
-       "      <td>2024-03-10</td>\n",
-       "      <td>7.88</td>\n",
-       "      <td>12.57</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>3.18</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>273</th>\n",
-       "      <td>HuggingFaceH4_zephyr-7b-beta-ift_v1.1</td>\n",
-       "      <td>2024-03-13</td>\n",
-       "      <td>4.71</td>\n",
-       "      <td>9.43</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>0.00</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
-       "      <td>NaN</td>\n",
+       "      <th>1741</th>\n",
+       "      <td>AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits</td>\n",
+       "      <td>28.89</td>\n",
+       "      <td>61.45</td>\n",
+       "      <td>28.89</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>274 rows × 10 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "                                                Model        Date  Average  \\\n",
-       "0              NousResearch_Nous-Hermes-2-Yi-34B_main  2024-03-04    74.01   \n",
-       "1              deepseek-ai_deepseek-llm-67b-chat_main  2024-03-05    71.62   \n",
-       "2    NousResearch_Nous-Hermes-2-Mixtral-8x7B-DPO_main  2024-03-02    70.43   \n",
-       "3           mistralai_Mixtral-8x7B-Instruct-v0.1_main  2024-03-02    69.80   \n",
-       "4              deepseek-ai_deepseek-llm-67b-chat_main  2024-03-04    67.03   \n",
-       "..                                                ...         ...      ...   \n",
-       "269            HuggingFaceH4_starcoder2-15b-ift_v18.0  2024-03-10    11.23   \n",
-       "270                HuggingFaceH4_mistral-7b-ift_v49.0  2024-03-07    10.07   \n",
-       "271                  HuggingFaceH4_starchat-beta_main  2024-03-12     8.13   \n",
-       "272             HuggingFaceH4_starcoder2-15b-ift_v7.0  2024-03-10     7.88   \n",
-       "273             HuggingFaceH4_zephyr-7b-beta-ift_v1.1  2024-03-13     4.71   \n",
+       "                                                  Model  Average  \\\n",
+       "1741  AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits    28.89   \n",
        "\n",
-       "     Ifeval  Truthfulqa  Winogrande  Gsm8k   Mmlu  Hellaswag    Arc  \n",
-       "0       NaN       61.44       80.58    NaN  76.24      83.79  68.00  \n",
-       "1     55.27         NaN         NaN  76.12  71.18      83.94    NaN  \n",
-       "2     59.33       64.76       78.53  62.17  71.96      85.42  70.82  \n",
-       "3     55.08       70.79       73.56  59.89  70.60      86.68  72.01  \n",
-       "4       NaN       57.78       79.16    NaN    NaN        NaN  64.16  \n",
-       "..      ...         ...         ...    ...    ...        ...    ...  \n",
-       "269   21.63         NaN         NaN   0.83    NaN        NaN    NaN  \n",
-       "270   20.15         NaN         NaN   0.00    NaN        NaN    NaN  \n",
-       "271    8.13         NaN         NaN    NaN    NaN        NaN    NaN  \n",
-       "272   12.57         NaN         NaN   3.18    NaN        NaN    NaN  \n",
-       "273    9.43         NaN         NaN   0.00    NaN        NaN    NaN  \n",
-       "\n",
-       "[274 rows x 10 columns]"
+       "      kaggle_tora_medium_extended  kaggle_tora_hard_extended  \n",
+       "1741                        61.45                      28.89  "
       ]
      },
-     "execution_count": 5,
+     "execution_count": 40,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "df"
+    "df.query(\"Model == 'AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits'\").dropna(axis=1, how=\"all\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 38,
    "metadata": {},
    "outputs": [
     {
@@ -339,180 +251,38 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Model</th>\n",
-       "      <th>Ifeval</th>\n",
-       "      <th>Truthfulqa</th>\n",
-       "      <th>Winogrande</th>\n",
-       "      <th>Gsm8k</th>\n",
-       "      <th>Mmlu</th>\n",
-       "      <th>Hellaswag</th>\n",
-       "      <th>Arc</th>\n",
+       "      <th>Average</th>\n",
+       "      <th>kaggle_tora_medium_extended</th>\n",
+       "      <th>kaggle_tora_hard_extended</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v41.0</td>\n",
-       "      <td>44.36</td>\n",
-       "      <td>49.35</td>\n",
-       "      <td>72.93</td>\n",
-       "      <td>37.30</td>\n",
-       "      <td>60.82</td>\n",
-       "      <td>79.70</td>\n",
-       "      <td>58.36</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v41.1</td>\n",
-       "      <td>47.32</td>\n",
-       "      <td>47.89</td>\n",
-       "      <td>72.69</td>\n",
-       "      <td>36.32</td>\n",
-       "      <td>60.34</td>\n",
-       "      <td>79.57</td>\n",
-       "      <td>57.51</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v41.10</td>\n",
-       "      <td>32.72</td>\n",
-       "      <td>51.05</td>\n",
-       "      <td>72.45</td>\n",
-       "      <td>25.93</td>\n",
-       "      <td>59.75</td>\n",
-       "      <td>81.92</td>\n",
-       "      <td>59.22</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>3</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v41.11</td>\n",
-       "      <td>37.89</td>\n",
-       "      <td>51.05</td>\n",
-       "      <td>64.56</td>\n",
-       "      <td>17.59</td>\n",
-       "      <td>57.60</td>\n",
-       "      <td>77.65</td>\n",
-       "      <td>55.89</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>4</th>\n",
-       "      <td>HuggingFaceH4_mistral-7b-ift_v41.12</td>\n",
-       "      <td>37.89</td>\n",
-       "      <td>45.94</td>\n",
-       "      <td>63.30</td>\n",
-       "      <td>21.15</td>\n",
-       "      <td>58.50</td>\n",
-       "      <td>74.94</td>\n",
-       "      <td>52.73</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>258</th>\n",
-       "      <td>mistralai_Mistral-7B-Instruct-v0.2_main</td>\n",
-       "      <td>53.97</td>\n",
-       "      <td>70.68</td>\n",
-       "      <td>68.82</td>\n",
-       "      <td>38.13</td>\n",
-       "      <td>59.43</td>\n",
-       "      <td>83.45</td>\n",
-       "      <td>65.70</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>259</th>\n",
-       "      <td>mistralai_Mixtral-8x7B-Instruct-v0.1_main</td>\n",
-       "      <td>55.08</td>\n",
-       "      <td>70.79</td>\n",
-       "      <td>73.56</td>\n",
-       "      <td>59.89</td>\n",
-       "      <td>70.60</td>\n",
-       "      <td>86.68</td>\n",
-       "      <td>72.01</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>260</th>\n",
-       "      <td>openchat_openchat-3.5-0106_main</td>\n",
-       "      <td>54.71</td>\n",
-       "      <td>57.55</td>\n",
-       "      <td>72.53</td>\n",
-       "      <td>66.19</td>\n",
-       "      <td>63.72</td>\n",
-       "      <td>80.10</td>\n",
-       "      <td>61.01</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>261</th>\n",
-       "      <td>stabilityai_stablelm-zephyr-3b_main</td>\n",
-       "      <td>34.75</td>\n",
-       "      <td>46.19</td>\n",
-       "      <td>58.41</td>\n",
-       "      <td>40.18</td>\n",
-       "      <td>45.18</td>\n",
-       "      <td>71.57</td>\n",
-       "      <td>45.82</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>262</th>\n",
-       "      <td>teknium_OpenHermes-2.5-Mistral-7B_main</td>\n",
-       "      <td>52.68</td>\n",
-       "      <td>58.62</td>\n",
-       "      <td>72.14</td>\n",
-       "      <td>54.06</td>\n",
-       "      <td>63.01</td>\n",
-       "      <td>82.34</td>\n",
-       "      <td>62.97</td>\n",
+       "      <th>1741</th>\n",
+       "      <td>AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits</td>\n",
+       "      <td>65.06</td>\n",
+       "      <td>65.06</td>\n",
+       "      <td>32.22</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
-       "<p>263 rows × 8 columns</p>\n",
        "</div>"
       ],
       "text/plain": [
-       "                                         Model  Ifeval  Truthfulqa  \\\n",
-       "0           HuggingFaceH4_mistral-7b-ift_v41.0   44.36       49.35   \n",
-       "1           HuggingFaceH4_mistral-7b-ift_v41.1   47.32       47.89   \n",
-       "2          HuggingFaceH4_mistral-7b-ift_v41.10   32.72       51.05   \n",
-       "3          HuggingFaceH4_mistral-7b-ift_v41.11   37.89       51.05   \n",
-       "4          HuggingFaceH4_mistral-7b-ift_v41.12   37.89       45.94   \n",
-       "..                                         ...     ...         ...   \n",
-       "258    mistralai_Mistral-7B-Instruct-v0.2_main   53.97       70.68   \n",
-       "259  mistralai_Mixtral-8x7B-Instruct-v0.1_main   55.08       70.79   \n",
-       "260            openchat_openchat-3.5-0106_main   54.71       57.55   \n",
-       "261        stabilityai_stablelm-zephyr-3b_main   34.75       46.19   \n",
-       "262     teknium_OpenHermes-2.5-Mistral-7B_main   52.68       58.62   \n",
-       "\n",
-       "     Winogrande  Gsm8k   Mmlu  Hellaswag    Arc  \n",
-       "0         72.93  37.30  60.82      79.70  58.36  \n",
-       "1         72.69  36.32  60.34      79.57  57.51  \n",
-       "2         72.45  25.93  59.75      81.92  59.22  \n",
-       "3         64.56  17.59  57.60      77.65  55.89  \n",
-       "4         63.30  21.15  58.50      74.94  52.73  \n",
-       "..          ...    ...    ...        ...    ...  \n",
-       "258       68.82  38.13  59.43      83.45  65.70  \n",
-       "259       73.56  59.89  70.60      86.68  72.01  \n",
-       "260       72.53  66.19  63.72      80.10  61.01  \n",
-       "261       58.41  40.18  45.18      71.57  45.82  \n",
-       "262       72.14  54.06  63.01      82.34  62.97  \n",
+       "                                                  Model  Average  \\\n",
+       "1741  AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits    65.06   \n",
        "\n",
-       "[263 rows x 8 columns]"
+       "      kaggle_tora_medium_extended  kaggle_tora_hard_extended  \n",
+       "1741                        65.06                      32.22  "
       ]
      },
-     "execution_count": 14,
+     "execution_count": 38,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "new_df = df.drop([\"Date\", \"Average\"], axis=1).groupby(\"Model\").max().reset_index()\n",
-    "new_df"
+    "df.query(\"Model == 'AI-MO_deepseek-math-7b-sft_aimo_v38.15.gptq-8bits'\").dropna(axis=1, how=\"all\")"
    ]
   },
   {