updates

app.py

@@ -102,11 +102,6 @@ models = original_df["model_name_for_query"].tolist() # needed for model backlin
 
 to_be_dumped = f"models = {repr(models)}\n"
 
-# with open("models_backlinks.py", "w") as f:
-#     f.write(to_be_dumped)
-
-# print(to_be_dumped)
-
 leaderboard_df = original_df.copy()
 (
     finished_eval_queue_df,
@@ -216,8 +211,8 @@ def change_tab(query_param: str):
 
 
 # Searching and filtering
-def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, size_query: list, show_deleted: bool, query: str):
-    filtered_df = filter_models(hidden_df, type_query, size_query, show_deleted)
+def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
     if query != "":
         filtered_df = search_table(filtered_df, query)
     df = select_columns(filtered_df, columns)
@@ -249,7 +244,7 @@ NUMERIC_INTERVALS = {
 }
 
 def filter_models(
-    df: pd.DataFrame, type_query: list, size_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
     # Show all models
     if show_deleted:
@@ -259,6 +254,7 @@ def filter_models(
 
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
 
     numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
@@ -277,6 +273,12 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
                             choices=[
@@ -310,11 +312,6 @@ with demo:
                             value=True, label="Show gated/private/deleted models", interactive=True
                         )
                 with gr.Column(min_width=320):
-                    search_bar = gr.Textbox(
-                        placeholder="🔍 Search for your model and press ENTER...",
-                        show_label=False,
-                        elem_id="search-bar",
-                    )
                     with gr.Box(elem_id="box-filter"):
                         filter_columns_type = gr.CheckboxGroup(
                             label="Model types",
@@ -323,16 +320,25 @@ with demo:
                                 ModelType.FT.to_str(),
                                 ModelType.IFT.to_str(),
                                 ModelType.RL.to_str(),
+                                ModelType.Unknown.to_str(),
                             ],
                             value=[
                                 ModelType.PT.to_str(),
                                 ModelType.FT.to_str(),
                                 ModelType.IFT.to_str(),
                                 ModelType.RL.to_str(),
+                                ModelType.Unknown.to_str(),
                             ],
                             interactive=True,
                             elem_id="filter-columns-type",
                         )
+                        filter_columns_precision = gr.CheckboxGroup(
+                            label="Precision",
+                            choices=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            value=["torch.float16", "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            interactive=True,
+                            elem_id="filter-columns-precision",
+                        )
                         filter_columns_size = gr.CheckboxGroup(
                             label="Model sizes",
                             choices=list(NUMERIC_INTERVALS.keys()),
@@ -375,6 +381,7 @@ with demo:
                     leaderboard_table,
                     shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -388,6 +395,7 @@ with demo:
                     leaderboard_table,
                     shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -402,6 +410,22 @@ with demo:
                     leaderboard_table,
                     shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                    search_bar,
+                ],
+                leaderboard_table,
+                queue=True,
+            )
+            filter_columns_precision.change(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -416,6 +440,7 @@ with demo:
                     leaderboard_table,
                     shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,
@@ -430,6 +455,7 @@ with demo:
                     leaderboard_table,
                     shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
                     search_bar,

model_info_cache.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c307938f15bda18b6c38af3d02cc0407d9d8d5345bc31f475af2cbbb33a4f8b5
-size 2895750
+oid sha256:5c80b745050df96eb1bc908e15b2406533b076c9160486a48b88c8a29f1ed312
+size 2985167

model_size_cache.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5b09d9f81d22f7849f92081950b675c2d68e3bfd320e5dfd1892d14602a29a2
+size 58166

requirements.txt

@@ -1,3 +1,4 @@
+accelerate==0.23.0
 aiofiles==23.1.0
 aiohttp==3.8.4
 aiosignal==1.3.1

src/display_models/get_model_metadata.py

@@ -8,6 +8,8 @@ from typing import List
 import huggingface_hub
 from huggingface_hub import HfApi
 from tqdm import tqdm
+from transformers import AutoModel, AutoConfig
+from accelerate import init_empty_weights
 
 from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
 from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
@@ -21,8 +23,13 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
     try:
         with open("model_info_cache.pkl", "rb") as f:
             model_info_cache = pickle.load(f)
-    except EOFError:
+    except (EOFError, FileNotFoundError):
         model_info_cache = {}
+    try:
+        with open("model_size_cache.pkl", "rb") as f:
+            model_size_cache = pickle.load(f)
+    except (EOFError, FileNotFoundError):
+        model_size_cache = {}
 
     for model_data in tqdm(leaderboard_data):
         model_name = model_data["model_name_for_query"]
@@ -37,16 +44,21 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
                 print("Repo not found!", model_name)
                 model_data[AutoEvalColumn.license.name] = None
                 model_data[AutoEvalColumn.likes.name] = None
-                model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
-                continue
+                if model_name not in model_size_cache:
+                    model_size_cache[model_name] = get_model_size(model_name, None)
+                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
 
         model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
         model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
-        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+        if model_name not in model_size_cache:
+            model_size_cache[model_name] = get_model_size(model_name, model_info)
+        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
     
     # save cache to disk in pickle format
     with open("model_info_cache.pkl", "wb") as f:
         pickle.dump(model_info_cache, f)
+    with open("model_size_cache.pkl", "wb") as f:
+        pickle.dump(model_size_cache, f)
 
 
 def get_model_license(model_info):
@@ -69,11 +81,17 @@ def get_model_size(model_name, model_info):
         return round(model_info.safetensors["total"] / 1e9, 3)
     except AttributeError:
         try:
-            size_match = re.search(size_pattern, model_name.lower())
-            size = size_match.group(0)
-            return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return 0
+            config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
+            with init_empty_weights():
+                model = AutoModel.from_config(config, trust_remote_code=False)
+            return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
+        except (EnvironmentError, ValueError, KeyError): # model config not found, likely private  
+            try:
+                size_match = re.search(size_pattern, model_name.lower())
+                size = size_match.group(0)
+                return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
+            except AttributeError:
+                return 0
 
 
 def get_model_type(leaderboard_data: List[dict]):

src/display_models/model_metadata_flags.py

@@ -7,6 +7,9 @@ FLAGGED_MODELS = {
     "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
     "TigerResearch/tigerbot-7b-sft-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/237",
     "gaodrew/gaodrew-gorgonzola-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/215",
+    "AIDC-ai-business/Marcoroni-70B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
+    "AIDC-ai-business/Marcoroni-13B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
+    "AIDC-ai-business/Marcoroni-7B": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/287",
 }
 
 # Models which have been requested by orgs to not be submitted on the leaderboard

src/display_models/model_metadata_type.py

@@ -14,7 +14,7 @@ class ModelType(Enum):
     FT = ModelInfo(name="fine-tuned", symbol="🔶")
     IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
     RL = ModelInfo(name="RL-tuned", symbol="🟦")
-    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
+    Unknown = ModelInfo(name="Unknown", symbol="?")
 
     def to_str(self, separator=" "):
         return f"{self.value.symbol}{separator}{self.value.name}"