add choices for GPU and Solve leaderboard issue

app.py

@@ -217,7 +217,7 @@ with demo:
                         search_bar = gr.Textbox(
                             placeholder=" 🔍 Model search (separate multiple queries with `;`)",
                             show_label=False,
-                            elem_id="search-bar",
+                            elem_id="search-bar"
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
@@ -306,14 +306,14 @@ with demo:
                     filter_columns_size,
                     search_bar,
                 ],
-                leaderboard_table,
+                leaderboard_table
             )
 
             # Check query parameter once at startup and update search bar
-            # demo.load(load_query, inputs=[], outputs=[search_bar])
+            demo.load(load_query, inputs=[], outputs=[search_bar])
 
             for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size]:
-                selector.change(
+                selector.select(
                     update_table,
                     [
                         hidden_leaderboard_table_for_search,

backend-cli.py

@@ -16,13 +16,13 @@ from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PAT
 from src.backend.manage_requests import EvalRequest
 from src.leaderboard.read_evals import EvalResult
 
-from src.envs import QUEUE_REPO, RESULTS_REPO, API
+from src.envs import QUEUE_REPO, RESULTS_REPO, API, DEBUG_QUEUE_REPO
 from src.utils import my_snapshot_download, analyze_gpu_stats, parse_nvidia_smi, monitor_gpus
 
 from src.leaderboard.read_evals import get_raw_eval_results
 
 from typing import Optional
-
+import GPUtil
 import time
 
 import pprint
@@ -364,9 +364,22 @@ def maybe_refresh_results(thr: int, hard_task_lst: Optional[list[str]] = None) -
     return False
 
 
+def get_gpu_details():
+    gpus = GPUtil.getGPUs()
+    gpu = gpus[0]
+    name = gpu.name.replace(" ", "-")
+    # Convert memory from MB to GB and round to nearest whole number
+    memory_gb = round(gpu.memoryTotal / 1024)
+    memory = f"{memory_gb}GB"
+    formatted_name = f"{name}-{memory}"
+    return formatted_name
+
 def process_pending_requests() -> bool:
+    if args.debug:
+        QUEUE_REPO = DEBUG_QUEUE_REPO
+        
     sanity_checks()
-
+    print("Processing pending requests")
     current_pending_status = [PENDING_STATUS]
 
     # Get all eval request that are PENDING, if you want to run other evals, change this parameter
@@ -385,6 +398,12 @@ def process_pending_requests() -> bool:
 
     eval_request = eval_requests[0]
     pp.pprint(eval_request)
+    
+    gpu_type = eval_request.gpu_type
+    curr_gpu_type = get_gpu_details()
+    if gpu_type != curr_gpu_type:
+        print(f"GPU type mismatch: {gpu_type} vs {curr_gpu_type}")
+        return False
 
     my_snapshot_download(
         repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60
@@ -426,6 +445,7 @@ def get_args():
     parser.add_argument("--precision", type=str, default="float32,float16,8bit,4bit", help="Precision to debug")
     parser.add_argument("--inference-framework", type=str, default="hf-chat", help="Inference framework to debug")
     parser.add_argument("--limit", type=int, default=None, help="Limit for the number of samples")
+    parser.add_argument("--gpu-type", type=str, default="NVIDIA-A100-PCIe-80GB", help="GPU type")
     return parser.parse_args()
 
 
@@ -454,7 +474,8 @@ if __name__ == "__main__":
                             status="",
                             json_filepath="",
                             precision=precision,  # Use precision from arguments
-                            inference_framework=args.inference_framework  # Use inference framework from arguments
+                            inference_framework=args.inference_framework,  # Use inference framework from arguments
+                            gpu_type=args.gpu_type
                         )
                         results = process_evaluation(task, eval_request, limit=args.limit)
                     except Exception as e:

src/backend/manage_requests.py

@@ -28,6 +28,7 @@ class EvalRequest:
     params: Optional[int] = None
     license: Optional[str] = ""
     batch_size: Optional[int] = 1
+    gpu_type: Optional[str] = "NVIDIA-A100-PCIe-80GB"
 
     def get_model_args(self) -> str:
         model_args = f"pretrained={self.model},revision={self.revision},parallelize=True"  # ,max_length=4096"

src/display/utils.py

@@ -193,7 +193,7 @@ class InferenceFramework(Enum):
 class GPUType(Enum):
     H100_pcie = ModelDetails("NVIDIA-H100-PCIe-80GB")
     A100_pcie = ModelDetails("NVIDIA-A100-PCIe-80GB")
-    A5000 = ModelDetails("NVIDIA-A5000-24GB")
+    A5000 = ModelDetails("NVIDIA-RTX-A5000-24GB")
     Unknown = ModelDetails("?")
 
     def to_str(self):

src/submission/submit.py

@@ -115,7 +115,7 @@ def add_new_eval(
         "params": model_size,
         "license": license,
         "inference_framework": inference_framework,
-        "GPU_type": gpu_type
+        "gpu_type": gpu_type
     }
 
     # Check for duplicate submission