Add app debug mode and dynamic refresh tables

app.py

@@ -3,10 +3,11 @@
 import os
 import datetime
 import socket
+from threading import Thread
 
 import gradio as gr
 import pandas as pd
-
+import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
@@ -38,11 +39,24 @@ from src.display.utils import (
     Precision,
 )
 
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, \
+    QUEUE_REPO, REPO_ID, RESULTS_REPO, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 from src.utils import get_dataset_summary_table
 
+def get_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run the LLM Leaderboard")
+    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
+    return parser.parse_args()
+
+args = get_args()
+if args.debug:
+    print("Running in debug mode")
+    QUEUE_REPO = DEBUG_QUEUE_REPO
+    RESULTS_REPO = DEBUG_RESULTS_REPO
 
 def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
     try:
@@ -76,11 +90,6 @@ def init_space():
     )
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-
-dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-leaderboard_df = original_df.copy()
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str
@@ -143,123 +152,158 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
 
     return filtered_df
 
+shown_columns = None
+dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+leaderboard_df = original_df.copy()
 
-# triggered only once at startup => read query parameter if it exists
-def load_query(request: gr.Request):
-    query = request.query_params.get("query") or ""
-    return query
+def update_leaderboard_table():
+    global leaderboard_df, shown_columns
+    print("Updating leaderboard table")
+    return leaderboard_df[
+                [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                + shown_columns.value
+                + [AutoEvalColumn.dummy.name]
+            ] if not leaderboard_df.empty else leaderboard_df
+        
 
+def update_hidden_leaderboard_table():
+    global original_df
+    return original_df[COLS] if original_df.empty is False else original_df
 
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+def update_dataset_table():
+    global dataset_df
+    return dataset_df
 
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("MOE-LLM-GPU-Poor-Leaderboard Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            with gr.Row():
-                with gr.Column():
-                    with gr.Row():
-                        search_bar = gr.Textbox(
-                            placeholder=" 🔍 Model search (separate multiple queries with `;`)",
-                            show_label=False,
-                            elem_id="search-bar",
-                        )
-                    with gr.Row():
-                        shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if not c.hidden and not c.never_hidden and not c.dummy
-                            ],
-                            value=[
-                                c.name
-                                for c in fields(AutoEvalColumn)
-                                if c.displayed_by_default and not c.hidden and not c.never_hidden
-                            ],
-                            label="Select columns to show",
-                            elem_id="column-select",
-                            interactive=True,
-                        )
+def update_finish_table():
+    global finished_eval_queue_df
+    return finished_eval_queue_df
 
-                with gr.Column(min_width=320):
-                    filter_columns_size = gr.CheckboxGroup(
-                        label="Inference frameworks",
-                        choices=[t.to_str() for t in InferenceFramework],
-                        value=[t.to_str() for t in InferenceFramework],
-                        interactive=True,
-                        elem_id="filter-columns-size",
-                    )
+def update_running_table():
+    global running_eval_queue_df
+    return running_eval_queue_df
 
-                    filter_columns_type = gr.CheckboxGroup(
-                        label="Model types",
-                        choices=[t.to_str() for t in ModelType],
-                        value=[t.to_str() for t in ModelType],
-                        interactive=True,
-                        elem_id="filter-columns-type",
-                    )
+def update_pending_table():
+    global pending_eval_queue_df
+    return pending_eval_queue_df
 
-                    filter_columns_precision = gr.CheckboxGroup(
-                        label="Precision",
-                        choices=[i.value.name for i in Precision],
-                        value=[i.value.name for i in Precision],
-                        interactive=True,
-                        elem_id="filter-columns-precision",
-                    )
+def update_finish_num():
+    global finished_eval_queue_df
+    return len(finished_eval_queue_df)
+
+def update_running_num():
+    global running_eval_queue_df
+    return len(running_eval_queue_df)
+
+def update_pending_num():
+    global pending_eval_queue_df
+    return len(pending_eval_queue_df)
 
-                    # filter_columns_size = gr.CheckboxGroup(
-                    #     label="Model sizes (in billions of parameters)",
-                    #     choices=list(NUMERIC_INTERVALS.keys()),
-                    #     value=list(NUMERIC_INTERVALS.keys()),
-                    #     interactive=True,
-                    #     elem_id="filter-columns-size",
-                    # )
-
-            # breakpoint()
-
-            leaderboard_table = gr.components.Dataframe(
-                value=(
-                    leaderboard_df[
-                        [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
-                        + shown_columns.value
-                        + [AutoEvalColumn.dummy.name]
-                    ]
-                    if leaderboard_df.empty is False
-                    else leaderboard_df
-                ),
-                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
-                datatype=TYPES,
-                elem_id="leaderboard-table",
-                interactive=False,
-                visible=True,
-            )  # column_widths=["2%", "20%"]
-
-            # Dummy leaderboard for handling the case when the user uses backspace key
-            hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df[COLS] if original_df.empty is False else original_df,
-                headers=COLS,
-                datatype=TYPES,
-                visible=False,
-            )
-
-            search_bar.submit(
-                update_table,
-                [
-                    hidden_leaderboard_table_for_search,
-                    shown_columns,
-                    filter_columns_type,
-                    filter_columns_precision,
-                    filter_columns_size,
-                    search_bar,
-                ],
-                leaderboard_table,
-            )
-
-            # Check query parameter once at startup and update search bar
-            demo.load(load_query, inputs=[], outputs=[search_bar])
-
-            for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.change(
+# triggered only once at startup => read query parameter if it exists
+def load_query(request: gr.Request):
+    query = request.query_params.get("query") or ""
+    return query
+    
+def refresh_leaderboard():
+    return gr.update(value=update_leaderboard_table()), gr.update(value=update_hidden_leaderboard_table()), \
+    gr.update(value=update_dataset_table()), gr.update(value=update_finish_table()), \
+    gr.update(value=update_running_table()), gr.update(value=update_pending_table()), \
+    gr.update(value=update_finish_num()), gr.update(value=update_running_num()), gr.update(value=update_pending_num())
+    
+def periodic_init():
+    global dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, leaderboard_df
+    while True:
+        time.sleep(60)
+        dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+        leaderboard_df = original_df.copy()
+
+def block_launch():
+    global dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, leaderboard_df, shown_columns
+    demo = gr.Blocks(css=custom_css)
+    with demo:
+        gr.HTML(TITLE)
+        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+        with gr.Tabs(elem_classes="tab-buttons") as tabs:
+            with gr.TabItem("MOE-LLM-GPU-Poor-Leaderboard Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+                with gr.Row():
+                    with gr.Column():
+                        with gr.Row():
+                            search_bar = gr.Textbox(
+                                placeholder=" 🔍 Model search (separate multiple queries with `;`)",
+                                show_label=False,
+                                elem_id="search-bar",
+                            )
+                        with gr.Row():
+                            shown_columns = gr.CheckboxGroup(
+                                choices=[
+                                    c.name
+                                    for c in fields(AutoEvalColumn)
+                                    if not c.hidden and not c.never_hidden and not c.dummy
+                                ],
+                                value=[
+                                    c.name
+                                    for c in fields(AutoEvalColumn)
+                                    if c.displayed_by_default and not c.hidden and not c.never_hidden
+                                ],
+                                label="Select columns to show",
+                                elem_id="column-select",
+                                interactive=True,
+                            )
+                    with gr.Column(min_width=320):
+                        filter_columns_size = gr.CheckboxGroup(
+                            label="Inference frameworks",
+                            choices=[t.to_str() for t in InferenceFramework],
+                            value=[t.to_str() for t in InferenceFramework],
+                            interactive=True,
+                            elem_id="filter-columns-size",
+                        )
+                        filter_columns_type = gr.CheckboxGroup(
+                            label="Model types",
+                            choices=[t.to_str() for t in ModelType],
+                            value=[t.to_str() for t in ModelType],
+                            interactive=True,
+                            elem_id="filter-columns-type",
+                        )
+                        filter_columns_precision = gr.CheckboxGroup(
+                            label="Precision",
+                            choices=[i.value.name for i in Precision],
+                            value=[i.value.name for i in Precision],
+                            interactive=True,
+                            elem_id="filter-columns-precision",
+                        )
+                        # filter_columns_size = gr.CheckboxGroup(
+                        #     label="Model sizes (in billions of parameters)",
+                        #     choices=list(NUMERIC_INTERVALS.keys()),
+                        #     value=list(NUMERIC_INTERVALS.keys()),
+                        #     interactive=True,
+                        #     elem_id="filter-columns-size",
+                        # )
+                # breakpoint()
+                refresh_button = gr.Button("Refresh", visible=True)
+                leaderboard_table = gr.components.Dataframe(
+                    value=(
+                        leaderboard_df[
+                            [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+                            + shown_columns.value
+                            + [AutoEvalColumn.dummy.name]
+                        ]
+                        if leaderboard_df.empty is False
+                        else leaderboard_df
+                    ),
+                    headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
+                    datatype=TYPES,
+                    elem_id="leaderboard-table",
+                    interactive=False,
+                    visible=True,
+                )  # column_widths=["2%", "20%"]
+                # Dummy leaderboard for handling the case when the user uses backspace key
+                hidden_leaderboard_table_for_search = gr.components.Dataframe(
+                    value=original_df[COLS] if original_df.empty is False else original_df,
+                    headers=COLS,
+                    datatype=TYPES,
+                    visible=False,
+                )
+                # refresh_button.click(fn=update_leaderboard_tables, outputs=[leaderboard_table, hidden_leaderboard_table_for_search])
+                search_bar.submit(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
@@ -270,124 +314,139 @@ with demo:
                         search_bar,
                     ],
                     leaderboard_table,
-                    queue=True,
                 )
-
-        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-            dataset_table = gr.components.Dataframe(
-                value=dataset_df,
-                headers=list(dataset_df.columns),
-                datatype=["str", "markdown", "str", "str", "str"],
-                elem_id="dataset-table",
-                interactive=False,
-                visible=True,
-                column_widths=["15%", "20%"],
-            )
-
-            gr.Markdown(LLM_BENCHMARKS_DETAILS, elem_classes="markdown-text")
-            gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("Submit a model ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-                    with gr.Accordion(f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-                    with gr.Accordion(f"⏳ Scheduled Evaluation Queue ({len(pending_eval_queue_df)})", open=False):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
-                            )
-
-            with gr.Row():
-                gr.Markdown("# Submit your model here", elem_classes="markdown-text")
-
-            with gr.Row():
-                inference_framework = gr.Dropdown(
-                    choices=[t.to_str() for t in InferenceFramework],
-                    label="Inference framework",
-                    multiselect=False,
-                    value=None,
-                    interactive=True,
+                # Check query parameter once at startup and update search bar
+                demo.load(load_query, inputs=[], outputs=[search_bar])
+                for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
+                    selector.change(
+                        update_table,
+                        [
+                            hidden_leaderboard_table_for_search,
+                            shown_columns,
+                            filter_columns_type,
+                            filter_columns_precision,
+                            filter_columns_size,
+                            search_bar,
+                        ],
+                        leaderboard_table,
+                        queue=True,
+                    )
+            with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):
+                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+                dataset_table = gr.components.Dataframe(
+                    value=dataset_df,
+                    headers=list(dataset_df.columns),
+                    datatype=["str", "markdown", "str", "str", "str"],
+                    elem_id="dataset-table",
+                    interactive=False,
+                    visible=True,
+                    column_widths=["15%", "20%"],
                 )
-                
-            with gr.Row():
+                gr.Markdown(LLM_BENCHMARKS_DETAILS, elem_classes="markdown-text")
+                gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")
+                # refresh_button.click(fn=update_dataset_table, outputs=[dataset_table])
+            with gr.TabItem("Submit a model ", elem_id="llm-benchmark-tab-table", id=3):
                 with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
+                    with gr.Row():
+                        gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+                    with gr.Column():
+                        with gr.Accordion(f"✅ Finished Evaluations", open=False):
+                            with gr.Column():
+                                num_fin = gr.Number(len(finished_eval_queue_df), label="Number of finished evaluations", visible=True, interactive=False)
+                                with gr.Row():
+                                    finished_eval_table = gr.components.Dataframe(
+                                        value=finished_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
+                                    )
+                        with gr.Accordion(f"🔄 Running Evaluation Queue", open=False):
+                            with gr.Column():
+                                num_run = gr.Number(len(running_eval_queue_df), label="Number of running evaluations", visible=True, interactive=False)
+                                with gr.Row():
+                                    running_eval_table = gr.components.Dataframe(
+                                        value=running_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
+                                    )
+                        with gr.Accordion(f"⏳ Scheduled Evaluation Queue", open=False):
+                            with gr.Column():
+                                num_sche = gr.Number(len(pending_eval_queue_df), label="Number of scheduled evaluations", visible=True, interactive=False)
+                                with gr.Row():
+                                    pending_eval_table = gr.components.Dataframe(
+                                        value=pending_eval_queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5
+                                    )
+                        # refresh_button.click(fn=update_submit_tables, 
+                        #                      outputs=[finished_eval_table, running_eval_table, pending_eval_table])
+                with gr.Row():
+                    gr.Markdown("# Submit your model here", elem_classes="markdown-text")
+                with gr.Row():
+                    inference_framework = gr.Dropdown(
+                        choices=[t.to_str() for t in InferenceFramework],
+                        label="Inference framework",
                         multiselect=False,
                         value=None,
                         interactive=True,
                     )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float32",
-                        interactive=True,
-                    )
-
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    private,
-                    weight_type,
-                    model_type,
-                    inference_framework,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("Citing this leaderboard", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
+                with gr.Row():
+                    with gr.Column():
+                        model_name_textbox = gr.Textbox(label="Model name")
+                        revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                        private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
+                        model_type = gr.Dropdown(
+                            choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                            label="Model type",
+                            multiselect=False,
+                            value=None,
+                            interactive=True,
+                        )
+                    with gr.Column():
+                        precision = gr.Dropdown(
+                            choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                            label="Precision",
+                            multiselect=False,
+                            value="float32",
+                            interactive=True,
+                        )
+                        weight_type = gr.Dropdown(
+                            choices=[i.value.name for i in WeightType],
+                            label="Weights type",
+                            multiselect=False,
+                            value="Original",
+                            interactive=True,
+                        )
+                        base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                submit_button = gr.Button("Submit Eval")
+                submission_result = gr.Markdown()
+                debug = gr.Checkbox(args.debug, label="Debug", visible=False)
+                submit_button.click(
+                    add_new_eval,
+                    [
+                        model_name_textbox,
+                        base_model_name_textbox,
+                        revision_name_textbox,
+                        precision,
+                        private,
+                        weight_type,
+                        model_type,
+                        inference_framework,
+                        debug
+                    ],
+                    submission_result,
+                )
+        refresh_button.click(refresh_leaderboard,
+                             outputs=[leaderboard_table, hidden_leaderboard_table_for_search, dataset_table, 
+                                      finished_eval_table, running_eval_table, pending_eval_table, num_fin, num_run, num_sche])    
+        
+        with gr.Row():
+            with gr.Accordion("Citing this leaderboard", open=False):
+                citation_button = gr.Textbox(
+                    value=CITATION_BUTTON_TEXT,
+                    label=CITATION_BUTTON_LABEL,
+                    lines=20,
+                    elem_id="citation-button",
+                    show_copy_button=True,
+                )
+    demo.queue(default_concurrency_limit=40).launch()
+    
 scheduler = BackgroundScheduler()
 
-scheduler.add_job(restart_space, "interval", seconds=6 * 60 * 60)
-
+scheduler.add_job(restart_space, "interval", hours=6)
 
 def launch_backend():
     import subprocess
@@ -396,8 +455,9 @@ def launch_backend():
     if DEVICE not in {"cpu"}:
         _ = subprocess.run(["python", "backend-cli.py"])
 
-
+Thread(target=periodic_init, daemon=True).start()
 # scheduler.add_job(launch_backend, "interval", seconds=120)
-
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    scheduler.start()
+    block_launch()
+    

src/envs.py

@@ -12,8 +12,8 @@ QUEUE_REPO = "sparse-generative-ai/requests"
 QUEUE_REPO_OPEN_LLM = "open-llm-leaderboard/requests"
 RESULTS_REPO = "sparse-generative-ai/results"
 
-PRIVATE_QUEUE_REPO = "sparse-generative-ai/private-requests"
-PRIVATE_RESULTS_REPO = "sparse-generative-ai/private-results"
+DEBUG_QUEUE_REPO = "sparse-generative-ai/debug_requests"
+DEBUG_RESULTS_REPO = "sparse-generative-ai/debug_results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA, DEBUG_QUEUE_REPO
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
     already_submitted_models,
@@ -26,12 +26,16 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
     inference_framework: str,
+    debug: bool = False
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    if debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+    
     user_name = ""
     model_path = model
     if "/" in model: