Upload folder using huggingface_hub

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text
@@ -33,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+.idea/
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,12 +1,46 @@
 ---
-title: Msteb Leaderboard
-emoji: ⚡
-colorFrom: pink
-colorTo: yellow
+title: mSTEB Leaderboard
+emoji: 🥇
+colorFrom: green
+colorTo: indigo
 sdk: gradio
-sdk_version: 5.49.1
 app_file: app.py
-pinned: false
+pinned: true
+license: apache-2.0
+short_description: Leaderboard for mSTEB benchmark
+sdk_version: 5.19.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Start the configuration
+
+Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
+
+Results files should have the following format and be stored as json files:
+```json
+{
+    "config": {
+        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
+        "model_name": "path of the model on the hub: org/model",
+        "model_sha": "revision on the hub",
+    },
+    "results": {
+        "task_name": {
+            "metric_name": score,
+        },
+        "task_name2": {
+            "metric_name": score,
+        }
+    }
+}
+```
+
+Request files are created automatically by this tool.
+
+If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
+
+# Code logic for more complex edits
+
+You'll find 
+- the main table' columns names and properties in `src/display/utils.py`
+- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
+- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

app.py

@@ -0,0 +1,258 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS,
+    SPEECH_BENCHMARK_COLS,
+    COLS,
+    COLS_SPEECH,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    AutoEvalColumnSpeech,
+    ModelType,
+    fields,
+    WeightType,
+    Precision, REGION_MAP
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import handle_csv_submission
+
+text_sample_path = "src/submission_samples/model_name_text.csv"
+speech_sample_path = "src/submission_samples/model_name_speech.csv"
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+
+### Space initialisation
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+        token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30,
+        token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+
+def init_leaderboard(dataframe, result_type='text'):
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    column_class = AutoEvalColumn if result_type == "text" else AutoEvalColumnSpeech
+
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(column_class)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(column_class) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(column_class) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        # search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[column_class.model.name],
+        hide_columns=[c.name for c in fields(column_class) if c.hidden],
+        filter_columns=[
+            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
+            # ColumnFilter(
+            #     AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            # ),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+leaderboard_dataframes = {
+    region: get_leaderboard_df(
+        EVAL_RESULTS_PATH,
+        EVAL_REQUESTS_PATH,
+        COLS,
+        BENCHMARK_COLS,
+        region if region != "All" else None,
+        result_type="text"
+    )
+    for region in REGION_MAP.values()
+}
+
+leaderboard_dataframes_speech = {
+    region: get_leaderboard_df(
+        EVAL_RESULTS_PATH,
+        EVAL_REQUESTS_PATH,
+        COLS_SPEECH,
+        SPEECH_BENCHMARK_COLS,
+        region if region != "All" else None,
+        result_type="speech"
+    )
+    for region in REGION_MAP.values()
+}
+# Preload leaderboard blocks
+js_switch_code = """
+(displayRegion) => {
+    const regionMap = {
+        "All": "All",
+        "Africa": "Africa",
+        "Americas/Oceania": "Americas_Oceania",
+        "Asia (S)": "Asia_S",
+        "Asia (SE)": "Asia_SE",
+        "Asia (W, C)": "Asia_W_C",
+        "Asia (E)": "Asia_E",
+        "Europe (W, N, S)": "Europe_W_N_S",
+        "Europe (E)": "Europe_E"
+    };
+    const region = regionMap[displayRegion];
+    document.querySelectorAll('[id^="leaderboard-"]').forEach(el => el.classList.remove("visible"));
+    const target = document.getElementById("leaderboard-" + region);
+    if (target) {
+        target.classList.add("visible");
+        // 🧠 Trigger reflow to fix row cutoff
+        void target.offsetHeight;  // Trigger reflow
+        target.style.display = "none"; // Hide momentarily
+        requestAnimationFrame(() => {
+            target.style.display = "";
+        });
+    }
+}
+"""
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 mSTEB Text Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            with gr.Row():
+                region_dropdown = gr.Dropdown(
+                    choices=list(REGION_MAP.keys()),
+                    label="Select Region",
+                    value="All",
+                    interactive=True,
+                )
+
+            # Region-specific leaderboard containers
+            for display_name, region_key in REGION_MAP.items():
+                with gr.Column(
+                        elem_id=f"leaderboard-{region_key}",
+                        elem_classes=["visible"] if region_key == "All" else []
+                ):
+                    init_leaderboard(leaderboard_dataframes[region_key], result_type="text")
+
+            # JS hook to toggle visible leaderboard
+            region_dropdown.change(None, js=js_switch_code, inputs=[region_dropdown])
+
+        with gr.TabItem("🗣️ mSTEB Speech Benchmark", elem_id="speech-benchmark-tab-table", id=1):
+            with gr.Row():
+                speech_region_dropdown = gr.Dropdown(
+                    choices=list(REGION_MAP.keys()),
+                    label="Select Region",
+                    value="All",
+                    interactive=True,
+                )
+
+            for display_name, region_key in REGION_MAP.items():
+                with gr.Column(
+                        elem_id=f"speech-leaderboard-{region_key}",
+                        elem_classes=["visible"] if region_key == "All" else []
+                ):
+                    init_leaderboard(leaderboard_dataframes_speech[region_key], result_type='speech')
+
+            speech_region_dropdown.change(
+                None,
+                js=js_switch_code.replace("leaderboard-", "speech-leaderboard-"),
+                inputs=[speech_region_dropdown]
+            )
+            
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.File(
+                    label="📄 Sample Text CSV",
+                    value=text_sample_path,
+                    interactive=False,
+                    file_types=[".csv"]
+                )
+                gr.File(
+                    label="📄 Sample Speech CSV",
+                    value=speech_sample_path,
+                    interactive=False,
+                    file_types=[".csv"]
+                )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Column():
+                model_name_textbox = gr.Textbox(label="Model name")
+                result_type = gr.Radio(choices=["text", "speech"], label="Result Type", value="text")
+                csv_file = gr.File(label="Upload CSV File", file_types=[".csv"])
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+
+            submit_button.click(
+                handle_csv_submission,
+                [
+                    model_name_textbox,
+                    csv_file,
+                    result_type,
+                ],
+                submission_result,
+            )
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

notes.txt

@@ -0,0 +1,12 @@
+The submission flow for the project is as follows:
+
+When a csv is submitted, we store the csv in the msteb requests dataset in the folder that's appropriate
+based on text or speech results.
+
+Then this csv result is in the same flow converted into a json file and uploaded to results dataset.
+
+This helps the leaderboard parse those results and display it.
+
+There are validation checks for the csv being formatted correctly and that at least one result value is present.
+
+

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,17 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.13
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece
+pydantic==2.10.6

src/about.py

@@ -0,0 +1,73 @@
+from dataclasses import dataclass
+from enum import Enum
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    task0 = Task("lid", "acc", "LID")
+    task1 = Task("topic_classification", "acc", "TC")
+    task2 = Task("rc_qa", "acc", "RC-QA")
+    task3 = Task("nli", "acc", "NLI")
+    task4 = Task("machine_translation_xx_eng", "chrf", "MT (xx-en)")
+    task5 = Task("machine_translation_eng_xx", "chrf", "MT (en-xx)")
+
+class SpeechTasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
+    task0 = Task("lid", "acc", "LID")
+    task1 = Task("topic_classification", "acc", "TC")
+    task2 = Task("rc_qa", "acc", "RC-QA")
+    task3 = Task("asr", "cer", "ASR")
+    task4 = Task("s2tt_xx_eng", "chrf", "S2TT (xx-en)")
+    #task5 = Task("s2tt_eng_xx", "chrf", "S2TT (en-xx)")
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
+
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">mSTEB Leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+This leaderboard has the results of evaluation of models on mSTEB benchmark.
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+## Reproducibility
+To reproduce our results please look at the github page for mSTEB:
+
+https://github.com/McGill-NLP/mSTEB
+
+"""
+
+EVALUATION_QUEUE_TEXT = """
+## Submit your results
+Please provide the model name, csv file and select the appropriate result type to upload your evaluation results for mSTEB. 
+
+Kindly format the results in the same way as provided in the sample csv files below. 
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+@misc{beyene2025mstebmassivelymultilingualevaluation,
+  title  = {mSTEB: Massively Multilingual Evaluation of LLMs on Speech and Text Tasks}, 
+  author = {Luel Hagos Beyene and Vivek Verma and Min Ma and Jesujoba O. Alabi 
+            and Fabian David Schmidt and Joyce Nakatumba-Nabende and 
+            David Ifeoluwa Adelani},
+  year   = {2025},
+  eprint = {2506.08400},
+  archivePrefix = {arXiv},
+  primaryClass  = {cs.CL},
+  url = {https://arxiv.org/abs/2506.08400}
+}
+"""

src/display/css_html_js.py

@@ -0,0 +1,135 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+[id^="leaderboard-"] {
+    display: none;
+}
+#leaderboard-All.visible,
+#leaderboard-Africa.visible,
+#leaderboard-Americas_Oceania.visible,
+#leaderboard-Asia_S.visible,
+#leaderboard-Asia_SE.visible,
+#leaderboard-Asia_W_C.visible,
+#leaderboard-Asia_E.visible,
+#leaderboard-Europe_W_N_S.visible,
+#leaderboard-Europe_E.visible {
+    display: block;
+}
+
+
+[id^="speech-leaderboard-"] {
+    display: none;
+}
+#speech-leaderboard-All.visible,
+#speech-leaderboard-Africa.visible,
+#speech-leaderboard-Americas_Oceania.visible,
+#speech-leaderboard-Asia_S.visible,
+#speech-leaderboard-Asia_SE.visible,
+#speech-leaderboard-Asia_W_C.visible,
+#speech-leaderboard-Asia_E.visible,
+#speech-leaderboard-Europe_W_N_S.visible,
+#speech-leaderboard-Europe_E.visible {
+    display: block;
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,29 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+def has_at_least_one_benchmark(df, columns):
+    return df[columns].notna().any(axis=1)
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,138 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks
+from src.about import SpeechTasks
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Model Type", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+### Speech leaderboard columns
+auto_eval_column_dict_speech = []
+# Init
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("Model Type", "str", True, never_hidden=True)])
+auto_eval_column_dict_speech.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict_speech.append(["average", ColumnContent, ColumnContent("Average ⬆️ (Class. Tasks)", "number", True)])
+for task in SpeechTasks:
+    auto_eval_column_dict_speech.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
+
+# Model information
+# auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+# auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+# auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+AutoEvalColumnSpeech = make_dataclass("AutoEvalColumnSpeech", auto_eval_column_dict_speech, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_SPEECH = [c.name for c in fields(AutoEvalColumnSpeech) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+SPEECH_BENCHMARK_COLS = [t.value.col_name for t in SpeechTasks]
+
+REGION_MAP = {
+    "All": "All",
+    "Africa": "Africa",
+    "Americas/Oceania": "Americas_Oceania",
+    "Asia (S)": "Asia_S",
+    "Asia (SE)": "Asia_SE",
+    "Asia (W, C)": "Asia_W_C",
+    "Asia (E)": "Asia_E",
+    "Europe (W, N, S)": "Europe_W_N_S",
+    "Europe (E)": "Europe_E",
+}
+

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "McGill-NLP" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/msteb_leaderboard"
+QUEUE_REPO = f"{OWNER}/msteb_requests"
+RESULTS_REPO = f"{OWNER}/msteb_results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,241 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, SpeechTasks
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+    regions: dict = None
+
+    @classmethod
+    def init_from_json_file(self, json_filepath, result_type='speech'):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+        regions = data.get("regions", {})  # Parse regions from JSON
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+
+        task_enum = Tasks if result_type == "text" else SpeechTasks
+
+        for task in task_enum:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        regions_processed_results = {}
+        for region, region_results in regions.items():
+            processed = {}
+            for task in task_enum:
+                task = task.value
+
+                # We average all scores of a given metric (not all metrics are present in all files)
+                accs = np.array([v.get(task.metric, None) for k, v in region_results.items() if task.benchmark == k])
+                if accs.size == 0 or any([acc is None for acc in accs]):
+                    continue
+
+                mean_acc = np.mean(accs) * 100.0
+                processed[task.benchmark] = mean_acc
+            regions_processed_results[region] = processed
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+            regions=regions_processed_results
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self, region=None, result_type='text'):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        # print(self.results)
+        task_enum = Tasks if result_type == "text" else SpeechTasks
+
+        results = self.results if region is None else self.regions.get(region, {})
+        acc_values = [
+            results[task.value.benchmark]
+            for task in task_enum
+            if task.value.metric == "acc" and task.value.benchmark in results
+        ]
+        # print(acc_values)
+
+        average = sum(acc_values) / len(acc_values) if acc_values else None
+
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            # AutoEvalColumn.precision.name: self.precision.value.name,
+            # AutoEvalColumn.model_type.name: self.model_type.value.name,
+            # # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            # AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: self.full_model,
+            # AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.license.name: self.license,
+            # AutoEvalColumn.likes.name: self.likes,
+            # AutoEvalColumn.params.name: self.num_params,
+            # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in task_enum:
+            if task.value.benchmark in results:
+                data_dict[task.value.col_name] = results[task.value.benchmark]
+            else:
+                data_dict[task.value.col_name] = None  # or np.nan if preferred
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str, result_type: str = "text") -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    # result type
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if result_type == "text" and "msteb_text_results" not in root:
+            continue
+        if result_type == "speech" and "msteb_speech_results" not in root:
+            continue
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath,result_type)
+        # print('testing this one')
+        # print(eval_result)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+    # print('results')
+    # print(results)
+
+    return results

src/populate.py

@@ -0,0 +1,59 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model, has_at_least_one_benchmark
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, region=None, result_type="text") -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path, result_type=result_type)
+    # this here if region is none gets main results. I have to pass region value here to get region based results
+    # and they should come.
+    all_data_json = [v.to_dict(region, result_type) for v in raw_data]
+    # print('all_data_json', all_data_json)
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_at_least_one_benchmark(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,143 @@
+import json
+import os
+import pandas as pd
+from datetime import datetime, timezone
+
+from src.about import Tasks, SpeechTasks
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.display.utils import REGION_MAP
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO, RESULTS_REPO, EVAL_RESULTS_PATH
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+
+def handle_csv_submission(
+    model_name: str,
+    csv_file,  # uploaded file path
+    result_type: str,
+):
+    if model_name == "" or model_name is None:
+        return styled_error("Please provide a model name.")
+    if csv_file is None:
+        return styled_error("Please provide a CSV file with results.")
+
+    df = pd.read_csv(csv_file)
+
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Save uploaded CSV
+    subdir = os.path.join(EVAL_REQUESTS_PATH, result_type)
+    os.makedirs(subdir, exist_ok=True)
+
+    filename = f"{current_time}_{model_name}_{result_type}_results.csv"
+    remote_path = f"msteb_{result_type}_requests/{filename}"
+
+    csv_save_path = os.path.join(subdir,filename)
+    df.to_csv(csv_save_path, index=False)
+
+    print(f"Uploading to {QUEUE_REPO}/{remote_path}")
+    API.upload_file(
+        path_or_fileobj=csv_save_path,
+        path_in_repo=remote_path,
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",  # or "model" if you made the repo that way
+        commit_message=f"Add {result_type} request for {model_name} at {current_time}",
+    )
+
+    # Remove the local file
+    os.remove(csv_save_path)
+    # this converts dataframe to json and uploads it to results
+
+
+    try:
+        convert_csv_to_json_and_upload(df, model_name, result_type)
+    except ValueError as e:
+        return styled_error(f"{str(e)}")
+    return styled_message(f"Results CSV successfully submitted for `{model_name}`!")
+
+def find_task_by_col_name(col_name, enum_cls):
+    for task in enum_cls:
+        if task.value.col_name == col_name:
+            return task
+    return None
+def convert_csv_to_json_and_upload(df: pd.DataFrame, model_name: str, result_type: str):
+    task_enum = Tasks if result_type == "text" else SpeechTasks
+
+    task_display_names = {t.value.col_name for t in task_enum}
+    region_names = df["Region"].tolist()
+    average_row = "Average (Micro)"
+
+    # --- Validation ---
+    df_columns = set(df.columns[1:])  # exclude Region column
+    if not df_columns.issubset(task_display_names):
+        extra = df_columns - task_display_names
+        raise ValueError(f"Extra columns in CSV: {extra}")
+    if average_row not in df["Region"].values:
+        raise ValueError("Missing row for 'Average (Micro)'")
+
+    data_region_names = [r for r in region_names if r != average_row]
+
+    for region in data_region_names:
+        if region not in REGION_MAP:
+            raise ValueError(f"Region '{region}' not found in REGION_MAP keys.")
+
+    # --- Build JSON ---
+    # I go over the regions in the CSV and create a JSON object.
+    model_json = {
+        "config": {"model_name": model_name},
+        "results": {},
+        "regions": {},
+    }
+    at_least_one_number = False
+
+    for _, row in df.iterrows():
+        region_display = row["Region"]
+
+        if region_display == average_row:
+            for col, val in row.items():
+                if col == "Region":
+                    continue
+                task = find_task_by_col_name(col, task_enum)
+                if val is not None and not pd.isna(val) and isinstance(val, (int, float)):
+                    print(f" value {val}")
+                    at_least_one_number = True
+                    model_json["results"][task.value.benchmark] = {task.value.metric: val/100}
+        else:
+            model_json["regions"][REGION_MAP[region_display]] = {}
+            for col, val in row.items():
+                if col == "Region":
+                    continue
+                task = find_task_by_col_name(col, task_enum)
+                if val is not None and not pd.isna(val) and isinstance(val, (int, float)):
+                    model_json["regions"][REGION_MAP[region_display]][task.value.benchmark] = {task.value.metric: val/100}
+
+    # Check if at least one number is present in the results
+    print(at_least_one_number)
+    if at_least_one_number is False:
+        raise ValueError("No valid numeric results found in the CSV. Please check your input.")
+
+    # --- Save locally ---
+    subdir = os.path.join(EVAL_RESULTS_PATH, result_type)
+    os.makedirs(subdir, exist_ok=True)
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    filename = f"{current_time}_{model_name}_{result_type}.json"
+    json_save_path = os.path.join(subdir,filename)
+
+    with open(json_save_path, "w") as f:
+        json.dump(model_json, f, indent=2)
+
+    # --- Upload to HF Hub ---
+    remote_path = f"msteb_leaderboard/msteb_{result_type}_results/{filename}"
+    API.upload_file(
+        path_or_fileobj=json_save_path,
+        path_in_repo=remote_path,
+        repo_id=RESULTS_REPO,
+        repo_type="dataset",
+        commit_message=f"Upload results for {model_name} ({result_type}) at {current_time}",
+    )
+    os.remove(json_save_path)
+
+    print(f"Uploaded to {RESULTS_REPO}/{current_time}")
+
+    return f"Uploaded to {RESULTS_REPO}/{current_time}"

src/submission_samples/model_name_speech.csv

@@ -0,0 +1,10 @@
+Region,LID,TC,RC-QA,ASR,S2TT (xx-en),S2TT (en-xx)
+Africa,,,,,
+Americas/Oceania,,,,,
+Asia (S),,,,,
+Asia (SE),,,,,
+"Asia (W, C)",,,,,
+Asia (E),,,,,
+"Europe (W, N, S)",,,,,
+Europe (E),,,,,
+Average (Micro),,,,,

src/submission_samples/model_name_text.csv

@@ -0,0 +1,10 @@
+Region,LID,TC,RC-QA,NLI,MT (xx-en),MT (en-xx)
+Africa,,,,,,
+Americas/Oceania,,,,,,
+Asia (S),,,,,,
+Asia (SE),,,,,,
+"Asia (W, C)",,,,,,
+Asia (E),,,,,,
+"Europe (W, N, S)",,,,,,
+Europe (E),,,,,,
+Average (Micro),,,,,,