import json
from pathlib import Path
import gradio as gr
import pandas as pd
from gradio_leaderboard import Leaderboard
from assets import custom_css
# override method to avoid bugg
Leaderboard.raise_error_if_incorrect_config = lambda self: None
abs_path = Path(__file__).parent
# Load the JSONL file into a pandas DataFrame using the json library
with open(abs_path / "results.jsonl", "r") as file:
json_data = file.read()
partially_fixed_json_data = json_data.replace("}\n{", "},\n{")
fixed_json_data = f"[{partially_fixed_json_data}]"
json_data = json.loads(fixed_json_data)
df = pd.DataFrame(json_data)
df["Model"] = df.apply(
lambda row: f'{row["Model"]}',
axis=1,
)
df = df[
["Model", "Median Inference Time", "Price per Image"]
+ [col for col in df.columns.tolist() if col not in ["URL", "Model", "Median Inference Time", "Price per Image"]]
]
df = df.sort_values(by="GenEval", ascending=False)
with gr.Blocks("ParityError/Interstellar", fill_width=True, css=custom_css) as demo:
gr.HTML(
"""
🏋️ InferBench 🏋️
A cost/quality/speed Leaderboard for Inference Providers!
"""
)
with gr.Tabs():
with gr.TabItem("FLUX.1 [dev] Leaderboard"):
Leaderboard(
value=df,
select_columns=df.columns.tolist(),
datatype=["markdown"] + ["number"] * (len(df.columns.tolist()) - 1),
)
with gr.Accordion("🌍 Join the Pruna AI community!", open=False):
gr.HTML(
"""
"""
)
with gr.Accordion("Citation", open=True):
gr.Markdown(
"""
```bibtex
@article{InferBench,
title={InferBench: A Leaderboard for Inference Providers},
author={PrunaAI},
year={2025},
howpublished={\\url{https://huggingface.co/spaces/PrunaAI/InferBench}}
}
```
"""
)
if __name__ == "__main__":
demo.launch()