add choice for device, and verify in backend. Add debug mode (#18)

- Add app debug mode and dynamic refresh tables (2a18e0ad941b517867200352ba49273da53f5907)
- Merge branch 'main' into pr/15 (a4829c27aeca63dc5327ec3a4287eb66bb2cbde8)
- Add inference_framework to the queue column (86b14ca786017b5479b54e4402226f7597295729)
- Add requirements (f5ff85d527bfa186b8e105d5637ac4e3793a9721)
- Merge branch 'main' into pr/15 (08b56fc73f03f150ef1baa35f78e762dcbf83fd5)
- Merge branch 'pr/15' into pr/18 (b2a2a5bae92f4b80223988e2059a69dfac7caaa8)
- Add GPU types (60d9c33965a34f63d2026b722afa33c03fe48306)
- Delete requests (22ce8a7836b70c1849ec4aeb77be3fce2642bcab)
- add choices for GPU and Solve leaderboard issue (bc48941fdfee36d8d1510a96b2969daa5d1ebf3a)
- fix a bug (6e99f9d4535fd801ae6b675ef2d833cc109e9d74)
- Apply GPU type verification on backend debug mode (dbe8db4df45ec9d75a8ce5abd46b77ff2e7627b7)
- Fix a bug (0fb715c8b89cef41ec9497c09b6ad8db47f65d78)

app.py

@@ -2,10 +2,11 @@
 import os
 import datetime
 import socket
+from threading import Thread
 
 import gradio as gr
 import pandas as pd
-
+import time
 from apscheduler.schedulers.background import BackgroundScheduler
 
 from huggingface_hub import snapshot_download
@@ -35,13 +36,27 @@ from src.display.utils import (
     fields,
     WeightType,
     Precision,
+    GPUType
 )
 
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, \
+    QUEUE_REPO, REPO_ID, RESULTS_REPO, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 from src.utils import get_dataset_summary_table
 
+def get_args():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run the LLM Leaderboard")
+    parser.add_argument("--debug", action="store_true", help="Run in debug mode")
+    return parser.parse_args()
+
+args = get_args()
+if args.debug:
+    print("Running in debug mode")
+    QUEUE_REPO = DEBUG_QUEUE_REPO
+    RESULTS_REPO = DEBUG_RESULTS_REPO
 
 def ui_snapshot_download(repo_id, local_dir, repo_type, tqdm_class, etag_timeout):
     try:
@@ -75,11 +90,6 @@ def init_space():
     )
     return dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df
 
-
-dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
-leaderboard_df = original_df.copy()
-
-
 # Searching and filtering
 def update_table(
     hidden_df: pd.DataFrame, columns: list, type_query: list, precision_query: list, size_query: list, query: str
@@ -142,6 +152,51 @@ def filter_models(df: pd.DataFrame, type_query: list, size_query: list, precisio
 
     return filtered_df
 
+shown_columns = None
+dataset_df, original_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = init_space()
+leaderboard_df = original_df.copy()
+
+# def update_leaderboard_table():
+#     global leaderboard_df, shown_columns
+#     print("Updating leaderboard table")
+#     return leaderboard_df[
+#                 [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
+#                 + shown_columns.value
+#                 + [AutoEvalColumn.dummy.name]
+#             ] if not leaderboard_df.empty else leaderboard_df
+        
+
+# def update_hidden_leaderboard_table():
+#     global original_df
+#     return original_df[COLS] if original_df.empty is False else original_df
+
+# def update_dataset_table():
+#     global dataset_df
+#     return dataset_df
+
+# def update_finish_table():
+#     global finished_eval_queue_df
+#     return finished_eval_queue_df
+
+# def update_running_table():
+#     global running_eval_queue_df
+#     return running_eval_queue_df
+
+# def update_pending_table():
+#     global pending_eval_queue_df
+#     return pending_eval_queue_df
+
+# def update_finish_num():
+#     global finished_eval_queue_df
+#     return len(finished_eval_queue_df)
+
+# def update_running_num():
+#     global running_eval_queue_df
+#     return len(running_eval_queue_df)
+
+# def update_pending_num():
+#     global pending_eval_queue_df
+#     return len(pending_eval_queue_df)
 
 # triggered only once at startup => read query parameter if it exists
 def load_query(request: gr.Request):
@@ -162,7 +217,7 @@ with demo:
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Model search (separate multiple queries with `;`)",
                             show_label=False,
-                            elem_id="search-bar",
+                            elem_id="search-bar"
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
@@ -251,14 +306,14 @@ with demo:
                     filter_columns_size,
                     search_bar,
                 ],
-                leaderboard_table,
+                leaderboard_table
             )
 
             # Check query parameter once at startup and update search bar
             demo.load(load_query, inputs=[], outputs=[search_bar])
 
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.change(
+                selector.select(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,
@@ -323,6 +378,15 @@ with demo:
                     value=None,
                     interactive=True,
                 )
+                
+                gpu_type = gr.Dropdown(
+                    choices=[t.to_str() for t in GPUType],
+                    label="GPU type",
+                    multiselect=False,
+                    value="NVIDIA-A100-PCIe-80GB",
+                    interactive=True,
+                )
+                
 
             with gr.Row():
                 with gr.Column():
@@ -358,6 +422,7 @@ with demo:
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
+            debug = gr.Checkbox(value=args.debug, label="Debug", visible=False)
             submit_button.click(
                 add_new_eval,
                 [
@@ -369,6 +434,8 @@ with demo:
                     weight_type,
                     model_type,
                     inference_framework,
+                    debug,
+                    gpu_type
                 ],
                 submission_result,
             )
@@ -385,8 +452,7 @@ with demo:
 
 scheduler = BackgroundScheduler()
 
-scheduler.add_job(restart_space, "interval", seconds=6 * 60 * 60)
-
+scheduler.add_job(restart_space, "interval", hours=6)
 
 def launch_backend():
     import subprocess
@@ -395,8 +461,9 @@ def launch_backend():
     if DEVICE not in {"cpu"}:
         _ = subprocess.run(["python", "backend-cli.py"])
 
-
+# Thread(target=periodic_init, daemon=True).start()
 # scheduler.add_job(launch_backend, "interval", seconds=120)
-
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == "__main__":
+    scheduler.start()
+    demo.queue(default_concurrency_limit=40).launch()
+    

backend-cli.py

@@ -16,13 +16,13 @@ from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PAT
 from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
-from src.envs import QUEUE_REPO, RESULTS_REPO, API
+from src.envs import QUEUE_REPO, RESULTS_REPO, API, DEBUG_QUEUE_REPO, DEBUG_RESULTS_REPO
 from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
 from typing import Optional
-
+import GPUtil
 import time
 
 import pprint
@@ -126,6 +126,9 @@ def request_to_result_name(request: EvalRequest) -> str:
 def process_evaluation(task: Task, eval_request: EvalRequest, limit: Optional[int] = None) -> dict:
     batch_size = 1
     batch_size = eval_request.batch_size
+    
+    if args.debug:
+        RESULTS_REPO = DEBUG_RESULTS_REPO
 
     init_gpu_info = analyze_gpu_stats(parse_nvidia_smi())
     # if init_gpu_info['Mem(M)'] > 500:
@@ -364,9 +367,22 @@ def maybe_refresh_results(thr: int, hard_task_lst: Optional[list[str]] = None) -
     return False
 
 
+def get_gpu_details():
+    gpus = GPUtil.getGPUs()
+    gpu = gpus[0]
+    name = gpu.name.replace(" ", "-")
+    # Convert memory from MB to GB and round to nearest whole number
+    memory_gb = round(gpu.memoryTotal / 1024)
+    memory = f"{memory_gb}GB"
+    formatted_name = f"{name}-{memory}"
+    return formatted_name
+
 def process_pending_requests() -> bool:
+    if args.debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+        
     sanity_checks()
-
+    print("Processing pending requests")
     current_pending_status = [PENDING_STATUS]
 
     # Get all eval request that are PENDING, if you want to run other evals, change this parameter
@@ -385,6 +401,12 @@ def process_pending_requests() -> bool:
 
     eval_request = eval_requests[0]
     pp.pprint(eval_request)
+    
+    gpu_type = eval_request.gpu_type
+    curr_gpu_type = get_gpu_details()
+    if gpu_type != curr_gpu_type:
+        print(f"GPU type mismatch: {gpu_type} vs {curr_gpu_type}")
+        return False
 
     my_snapshot_download(
         repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
@@ -426,6 +448,8 @@ def get_args():
     parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
     parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
     parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
+    parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", 
+                        help="GPU type. NVIDIA-A100-PCIe-80GB; NVIDIA-RTX-A5000-24GB; NVIDIA-H100-PCIe-80GB")
     return parser.parse_args()
 
 
@@ -454,8 +478,13 @@ if __name__ == "__main__":
                             status="",
                             json_filepath="",
                             precision=precision,  # Use precision from arguments
-                            inference_framework=args.inference_framework  # Use inference framework from arguments
+                            inference_framework=args.inference_framework,  # Use inference framework from arguments
+                            gpu_type=args.gpu_type
                         )
+                        curr_gpu_type = get_gpu_details()
+                        if eval_request.gpu_type != curr_gpu_type:
+                            print(f"GPU type mismatch: {eval_request.gpu_type} vs {curr_gpu_type}")
+                            raise Exception("GPU type mismatch")
                         results = process_evaluation(task, eval_request, limit=args.limit)
                     except Exception as e:
                         print(f"debug running error: {e}")

requirements.txt

@@ -30,3 +30,4 @@ evaluate
 spacy
 selfcheckgpt
 immutabledict
+gputil

src/backend/manage_requests.py

@@ -28,6 +28,7 @@ class EvalRequest:
     params: Optional[int] = None
     license: Optional[str] = ""
     batch_size: Optional[int] = 1
+    gpu_type: Optional[str] = "NVIDIA-A100-PCIe-80GB"
 
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"

src/display/utils.py

@@ -140,6 +140,7 @@ class EvalQueueColumn:  # Queue column
     private = ColumnContent("private", "bool", True)
     precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
+    model_framework = ColumnContent("inference_framework", "str", True)
     status = ColumnContent("status", "str", True)
 
 
@@ -189,7 +190,25 @@ class InferenceFramework(Enum):
             return InferenceFramework.HF_Chat
         return InferenceFramework.Unknown
 
+class GPUType(Enum):
+    H100_pcie = ModelDetails("NVIDIA-H100-PCIe-80GB")
+    A100_pcie = ModelDetails("NVIDIA-A100-PCIe-80GB")
+    A5000 = ModelDetails("NVIDIA-RTX-A5000-24GB")
+    Unknown = ModelDetails("?")
 
+    def to_str(self):
+        return self.value.name
+
+    @staticmethod
+    def from_str(gpu_type: str):
+        if gpu_type in ["NVIDIA-H100-PCIe-80GB"]:
+            return GPUType.A100_pcie
+        if gpu_type in ["NVIDIA-A100-PCIe-80GB"]:
+            return GPUType.H100_pcie
+        if gpu_type in ["NVIDIA-A5000-24GB"]:
+            return GPUType.A5000
+        return GPUType.Unknown
+    
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")

src/envs.py

@@ -12,8 +12,8 @@ QUEUE_REPO = "sparse-generative-ai/requests"
 QUEUE_REPO_OPEN_LLM = "open-llm-leaderboard/requests"
 RESULTS_REPO = "sparse-generative-ai/results"
 
-PRIVATE_QUEUE_REPO = "sparse-generative-ai/private-requests"
-PRIVATE_RESULTS_REPO = "sparse-generative-ai/private-results"
+DEBUG_QUEUE_REPO = "sparse-generative-ai/debug_requests"
+DEBUG_RESULTS_REPO = "sparse-generative-ai/debug_results"
 
 IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
 

src/populate.py

@@ -95,6 +95,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
             data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
             data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+            data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
 
             all_evals.append(data)
         elif ".md" not in entry:
@@ -107,6 +108,7 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> tuple[pd.DataFrame, p
 
                 data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
                 data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                data[EvalQueueColumn.model_framework.name] = data.get("inference_framework", "-")
                 all_evals.append(data)
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]

src/submission/check_validity.py

@@ -130,7 +130,7 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}")
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}_{info['inference_framework']}_{info['gpu_type']}")
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+from src.envs import API, EVAL_REQUESTS_PATH, H4_TOKEN, QUEUE_REPO, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA, DEBUG_QUEUE_REPO
 from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
 from src.submission.check_validity import (
     already_submitted_models,
@@ -26,12 +26,17 @@ def add_new_eval(
     weight_type: str,
     model_type: str,
     inference_framework: str,
+    debug: bool = False,
+    gpu_type: str = "NVIDIA-A100-PCIe-80GB",
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
+    if debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+    
     user_name = ""
     model_path = model
     if "/" in model:
@@ -110,17 +115,18 @@ def add_new_eval(
         "params": model_size,
         "license": license,
         "inference_framework": inference_framework,
+        "gpu_type": gpu_type
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}_{inference_framework}" in REQUESTED_MODELS:
+    if f"{model}_{revision}_{precision}_{inference_framework}_{gpu_type}" in REQUESTED_MODELS:
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
     # out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}_{inference_framework}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}_{inference_framework}_{gpu_type}.json"
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))