src/about.py

from dataclasses import dataclass
from enum import Enum

@dataclass
class Task:
    benchmark: str
    metric: str
    col_name: str

# Select your tasks here
# ---------------------------------------------------
class Tasks(Enum):
    # # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
    tsg_avg = Task("custom|3gpp:tsg|0", "em", "3GPP-TSG")
    # tele_EQ = Task("custom|telecom:math|0", "em", "TELE-EQ")
    tele_QnA = Task("custom|telecom:qna|0", "em", "TELE-QnA")
    math = Task("custom|math:problem_solving|0", "math_metric", "MATH-500")
    folio = Task("custom|folio:logical_reasoning|0", "em", "FOLIO")
    spider = Task("custom|spider:text2sql|0", "sql_metric", "Spider")



NUM_FEWSHOT = 0 # Change with your few shot
# ---------------------------------------------------



# Your leaderboard name
TITLE = """<h1 align="left" id="space-title">GSMA Open-Telco LLM Benchmarks</h1>"""
# TITLE = """<img src="https://avatars.githubusercontent.com/u/148767883?v=4" style="width:15%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">"""

BOTTOM_LOGO = """<img src="https://avatars.githubusercontent.com/u/148767883?v=4" style="width:20%;display:block;margin-left:auto;margin-right:auto;border-radius:15px;">"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
🌐 GSMA Open-Telco LLM Benchmarks : Evaluate and compare the performance of Telecom Large Language Models (LLMs).


When you submit a model on the "Submit here!" page, it will be evaluated on a set of benchmarks. 

More details about the benchmarks and the evaluation process is provided on the “About” page.
"""

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = f"""
# Context

Large Language Models (LLMs) have the potential to revolutionize the Sixth Generation (6G) communication networks. However, current mainstream LLMs generally lack the specialized
knowledge in telecom domain. We plan to collect and build  datasets to evaluate the capabilities of LLMs in telecom use-cases.

The benchmark provides an open-source, transparent evaluation framework for AI models in telecom, focusing on:

- Real-World Performance: Testing AI capabilities in customer support, network automation, and regulatory compliance.
- Holistic AI Assessment: Evaluating capability, energy efficiency, and safety, ensuring AI aligns with telecom's operational and sustainability goals.

By setting a unified approach for AI in telecom, the Open-Telco LLM Benchmark helps drive innovation and accelerate AI adoption in next-generation networks.


## Icons & Model types

🟢 : `pretrained` or `continuously pretrained`

🔶 : `fine-tuned on domain-specific datasets`

💬 : `chat models (RLHF, DPO, ORPO, ...)`

🤝 : `base merges and moerges`


If the icon is "?", it indicates that there is insufficient information about the model.
Please provide information about the model through an issue! 🤩

Note 1 : We reserve the right to correct any incorrect tags/icons after manual verification to ensure the accuracy and reliability of the leaderboard.

Note 2 ⚠️ : Some models might be widely discussed as subjects of caution by the community, implying that users should exercise restraint when using them. Models that have used the evaluation set for training to achieve a high leaderboard ranking, among others, may be selected as subjects of caution and might result in their deletion from the leaderboard.

## How it works
📈 We evaluate models using the impressive [LightEval](https://github.com/huggingface/lighteval), a unified and straightforward framework from the HuggingFace Eval Team to test and assess causal language models on a large number of different evaluation tasks.

To ensure a fair and unbiased assessment of the models' true capabilities, all evaluations are conducted in zero-shot settings `0-shots`. This approach eliminates any potential advantage from task-specific fine-tuning, providing a clear indication of how well the models can generalize to new tasks.

Please, consider reaching out to us through the discussions tab if you are working on benchmarks for Telecom LLMs and willing to see them on this leaderboard as well. Your benchmark might change the whole game for Telecom industry !

GPUs are provided by __[HuggingFace](https://huggingface.co/)__ and __[GSMA](https://www.gsma.com/)__ for the evaluations.

## Details and Logs
- Detailed numerical results in the `results` otellm dataset: https://huggingface.co/datasets/otellm/results
- Community queries and running status in the `requests` otellm dataset: https://huggingface.co/datasets/otellm/requests

## More resources
If you still have questions, you can check our FAQ [here](https://huggingface.co/spaces/otellm/Open-Telcom-LLM-Leaderboard/discussions/1)!
"""

EVALUATION_QUEUE_TEXT = """
## Some good practices before submitting a model

### 1) Make sure you can load your model and tokenizer using AutoClasses:

```python
from transformers import AutoConfig, AutoModel, AutoTokenizer
config = AutoConfig.from_pretrained("your model name", revision=revision)
model = AutoModel.from_pretrained("your model name", revision=revision)
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
```
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.

Note: make sure your model is public!
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!

### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

### 3) Make sure your model has an open license!
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗

### 4) Fill up your model card
When we add extra information about models to the leaderboard, it will be automatically taken from the model card

## In case of model failure
If your model is displayed in the `FAILED` category, its execution stopped.
Make sure you have followed the above steps first.
If everything is done, check you can launch the LightEval script on your model locally, using [this script](https://gist.github.com/alielfilali01/d486cfc962dca3ed4091b7c562a4377f).
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@misc{otellm,
  author = {Barriah, Lina and De Domenico, Antonio and Powell, Louis and Sana, Mohamed and Wang, Pierre and Piovesan, Nicola and Debbah, Merouane},
  title = {GSMA Open-Telco LLM Benchmarks},
  year = {2025},
  month= {February},
  publisher = {HuggingFace Community articles},
  howpublished = "\url{https://huggingface.co/blog/otellm/gsma-benchmarks}"
}

@article{maatouk2023teleqna,
  title={Teleqna: A benchmark dataset to assess large language models telecommunications knowledge},
  author={Maatouk, Ali and Ayed, Fadhel and Piovesan, Nicola and De Domenico, Antonio and Debbah, Merouane and Luo, Zhi-Quan},
  journal={arXiv preprint arXiv:2310.15051},
  year={2023}
}


@article{maatouk2024large,
  title={Large language models for telecom: Forthcoming impact on the industry},
  author={Maatouk, Ali and Piovesan, Nicola and Ayed, Fadhel and De Domenico, Antonio and Debbah, Merouane},
  journal={IEEE Communications Magazine},
  year={2024},
  publisher={IEEE}
}

@article{zou2024telecomgpt,
  title={TelecomGPT: A framework to build telecom-specfic large language models},
  author={Zou, Hang and Zhao, Qiyang and Tian, Yu and Bariah, Lina and Bader, Faouzi and Lestable, Thierry and Debbah, Merouane},
  journal={arXiv preprint arXiv:2407.09424},
  year={2024}
}

@misc{lighteval,
  author = {Fourrier, Clémentine and Habib, Nathan and Wolf, Thomas and Tunstall, Lewis},
  title = {LightEval: A lightweight framework for LLM evaluation},
  year = {2023},
  version = {0.3.0},
  url = {https://github.com/huggingface/lighteval}
}

@article{hendrycks2021measuring,
  title={Measuring mathematical problem solving with the math dataset},
  author={Hendrycks, Dan and Burns, Collin and Kadavath, Saurav and Arora, Akul and Basart, Steven and Tang, Eric and Song, Dawn and Steinhardt, Jacob},
  journal={arXiv preprint arXiv:2103.03874},
  year={2021}
}

@article{yu2018spider,
  title={Spider: A large-scale human-labeled dataset for complex and cross-domain semantic parsing and text-to-sql task},
  author={Yu, Tao and Zhang, Rui and Yang, Kai and Yasunaga, Michihiro and Wang, Dongxu and Li, Zifan and Ma, James and Li, Irene and Yao, Qingning and Roman, Shanelle and others},
  journal={arXiv preprint arXiv:1809.08887},
  year={2018}
}

@article{han2022folio,
  title={Folio: Natural language reasoning with first-order logic},
  author={Han, Simeng and Schoelkopf, Hailey and Zhao, Yilun and Qi, Zhenting and Riddell, Martin and Zhou, Wenfei and Coady, James and Peng, David and Qiao, Yujie and Benson, Luke and others},
  journal={arXiv preprint arXiv:2209.00840},
  year={2022}
}"""