moved delta to new column, updated ranking

app.py

@@ -26,6 +26,8 @@ def make_default_md(arena_df, elo_results):
 
 LMSYS [Chatbot Arena](https://lmsys.org/blog/2023-05-03-arena/) is a crowdsourced open platform for LLM evals.
 We've collected over **500,000** human preference votes to rank LLMs with the Elo ranking system. Contribute your vote 🗳️ at [chat.lmsys.org](https://chat.lmsys.org)!
+
+Code to recreate leaderboard tables and plots in this [notebook]({notebook_url}) and more discussions in this blog [post](https://lmsys.org/blog/2023-12-07-leaderboard/).
 """
     return leaderboard_md
 
@@ -213,32 +215,57 @@ def get_full_table(arena_df, model_table_df):
 
 def create_ranking_str(ranking, ranking_difference):
     if ranking_difference > 0:
-        return f"{int(ranking)} (\u2191{int(ranking_difference)})"
+        # return f"{int(ranking)} (\u2191{int(ranking_difference)})"
+        return f"{int(ranking)} \u2191"
     elif ranking_difference < 0:
-        return f"{int(ranking)} (\u2193{int(-ranking_difference)})"
+        # return f"{int(ranking)} (\u2193{int(-ranking_difference)})"
+        return f"{int(ranking)} \u2193"
     else:
         return f"{int(ranking)}"
     
+def recompute_final_ranking(arena_df):
+    # compute ranking based on CI
+    ranking = {}
+    for i, model_a in enumerate(arena_df.index):
+        ranking[model_a] = 1
+        for j, model_b in enumerate(arena_df.index):
+            if i == j:
+                continue
+            if arena_df.loc[model_b]["rating_q025"] > arena_df.loc[model_a]["rating_q975"]:
+                ranking[model_a] += 1
+    return list(ranking.values())
+    
 def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
-    # arena_df = arena_df.sort_values(by=["rating"], ascending=False)
-    arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True)
+    arena_df = arena_df.sort_values(by=["rating"], ascending=False)
     arena_df = arena_df[arena_df["num_battles"] > 2000]
+    arena_df["final_ranking"] = recompute_final_ranking(arena_df)
+    arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True)
 
     # arena_df["final_ranking"] = range(1, len(arena_df) + 1)
     # sort by rating
     if arena_subset_df is not None:
         # filter out models not in the arena_df
         arena_subset_df = arena_subset_df[arena_subset_df.index.isin(arena_df.index)]
-        # arena_subset_df = arena_subset_df.sort_values(by=["rating"], ascending=False)
-        arena_subset_df = arena_subset_df.sort_values(by=["final_ranking"], ascending=True)
+        arena_subset_df = arena_subset_df.sort_values(by=["rating"], ascending=False)
+        # arena_subset_df = arena_subset_df.sort_values(by=["final_ranking"], ascending=True)
+        # arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 500]
+        arena_subset_df["final_ranking"] = recompute_final_ranking(arena_subset_df)
+        # keep only the models in the subset in arena_df and recompute final_ranking
+        arena_df = arena_df[arena_df.index.isin(arena_subset_df.index)]
+        # recompute final ranking
+        arena_df["final_ranking"] = recompute_final_ranking(arena_df)
 
         # assign ranking by the order
-        # arena_subset_df["final_ranking"] = range(1, len(arena_subset_df) + 1)
+        arena_subset_df["final_ranking_no_tie"] = range(1, len(arena_subset_df) + 1)
+        arena_df["final_ranking_no_tie"] = range(1, len(arena_df) + 1)
         # join arena_df and arena_subset_df on index
-        arena_df = arena_subset_df.join(arena_df["final_ranking"], rsuffix="_global", how="inner")
-        arena_df['ranking_difference'] =  arena_df['final_ranking_global'] - arena_df['final_ranking']
+        arena_df = arena_subset_df.join(arena_df["final_ranking_no_tie"], rsuffix="_global", how="inner")
+        # arena_df = arena_subset_df.join(arena_df["rating"], rsuffix="_global", how="inner")
+        arena_df["ranking_difference"] =  arena_df["final_ranking_no_tie_global"] - arena_df["final_ranking_no_tie"]
+        # arena_df['ranking_difference'] =  arena_df['rating_global'] - arena_df['rating']
+        arena_df = arena_df.sort_values(by=["final_ranking"], ascending=True)
         arena_df["final_ranking"] = arena_df.apply(lambda x: create_ranking_str(x["final_ranking"], x["ranking_difference"]), axis=1)
-    
+
     values = []
     for i in range(len(arena_df)):
         row = []
@@ -247,10 +274,11 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
             model_name = model_table_df[model_table_df["key"] == model_key]["Model"].values[
                 0
             ]
-
             # rank
             ranking = arena_df.iloc[i].get("final_ranking") or i+1
             row.append(ranking)
+            if arena_subset_df is not None:
+                row.append(arena_df.iloc[i].get("ranking_difference") or 0)
             # model display name
             row.append(model_name)
             # elo rating
@@ -272,7 +300,6 @@ def get_arena_table(arena_df, model_table_df, arena_subset_df=None):
             row.append(
                 model_table_df[model_table_df["key"] == model_key]["License"].values[0]
             )
-
             cutoff_date = model_table_df[model_table_df["key"] == model_key]["Knowledge cutoff date"].values[0]
             if cutoff_date == "-":
                 row.append("Unknown")
@@ -421,13 +448,85 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
     else:
         pass
 
+    def update_leaderboard_df(arena_table_vals):
+        elo_datarame = pd.DataFrame(arena_table_vals, columns=[ "Rank", "Δ", "🤖 Model", "⭐ Arena Elo", "📊 95% CI", "🗳️ Votes", "Organization", "License", "Knowledge Cutoff"])
+
+        # goal: color the rows based on the rank with styler
+        def highlight_max(s):
+            # all items in S which contain up arrow should be green, down arrow should be red, otherwise black
+            return ["color: green; font-weight: bold" if "\u2191" in v else "color: red; font-weight: bold" if "\u2193" in v else "" for v in s]
+            
+        def highlight_rank_max(s):
+            return ["color: green; font-weight: bold" if v > 0 else "color: red; font-weight: bold" if v < 0 else "" for v in s]
+        
+        return elo_datarame.style.apply(highlight_max, subset=["Rank"]).apply(highlight_rank_max, subset=["Δ"])
+
     def update_leaderboard_and_plots(category):
         arena_subset_df = arena_dfs[category]
+        arena_subset_df = arena_subset_df[arena_subset_df["num_battles"] > 500]
         elo_subset_results = category_elo_results[category]
         arena_df = arena_dfs["Total"]
-        arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df)
+        arena_values = get_arena_table(arena_df, model_table_df, arena_subset_df = arena_subset_df if category != "Total" else None)
         if category != "Total":
             arena_values = update_leaderboard_df(arena_values)
+            arena_values = gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "Δ",
+                        "🤖 Model",
+                        "⭐ Arena Elo",
+                        "📊 95% CI",
+                        "🗳️ Votes",
+                        "Organization",
+                        "License",
+                        "Knowledge Cutoff",
+                    ],
+                    datatype=[
+                        "str",
+                        "number",
+                        "markdown",
+                        "number",
+                        "str",
+                        "number",
+                        "str",
+                        "str",
+                        "str",
+                    ],
+                    value=arena_values,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[50, 50, 190, 110, 100, 90, 160, 150, 140],
+                    wrap=True,
+                )
+        else:
+            arena_values = gr.Dataframe(
+                    headers=[
+                        "Rank",
+                        "🤖 Model",
+                        "⭐ Arena Elo",
+                        "📊 95% CI",
+                        "🗳️ Votes",
+                        "Organization",
+                        "License",
+                        "Knowledge Cutoff",
+                    ],
+                    datatype=[
+                        "str",
+                        "markdown",
+                        "number",
+                        "str",
+                        "number",
+                        "str",
+                        "str",
+                        "str",
+                    ],
+                    value=arena_values,
+                    elem_id="arena_leaderboard_dataframe",
+                    height=700,
+                    column_widths=[70, 190, 110, 100, 90, 160, 150, 140],
+                    wrap=True,
+                )
+
         p1 = elo_subset_results["win_fraction_heatmap"]
         p2 = elo_subset_results["battle_count_heatmap"]
         p3 = elo_subset_results["bootstrap_elo_rating"]
@@ -436,18 +535,6 @@ def build_leaderboard_tab(elo_results_file, leaderboard_table_file, show_plot=Fa
         """
         leaderboard_md = make_category_arena_leaderboard_md(arena_df, arena_subset_df, name=category)
         return arena_values, p1, p2, p3, p4, more_stats_md, leaderboard_md
-    
-    def update_leaderboard_df(arena_table_vals):
-        elo_datarame = pd.DataFrame(arena_table_vals, columns=["Rank", "Model", "Arena Elo", "95% CI", "Votes", "Organization", "License", "Knowledge Cutoff"])
-
-        # goal: color the rows based on the rank with styler
-        def highlight_max(s):
-            # all items in S which contain up arrow should be green, down arrow should be red, otherwise black
-            return ["color: green" if "\u2191" in v else "color: red" if "\u2193" in v else "" for v in s]
-        
-        styled_df = elo_datarame.style.apply(highlight_max, subset=["Rank"])
-
-        return styled_df
                 
     category_dropdown.change(update_leaderboard_and_plots, inputs=[category_dropdown], outputs=[elo_display_df, plot_1, plot_2, plot_3, plot_4, more_stats_md, category_deets])