Spaces:
Running
Running
import gradio as gr | |
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns | |
import pandas as pd | |
from apscheduler.schedulers.background import BackgroundScheduler | |
from huggingface_hub import snapshot_download | |
import numpy as np | |
import os | |
from src.about import ( | |
CITATION_BUTTON_LABEL, | |
CITATION_BUTTON_TEXT, | |
EVALUATION_QUEUE_TEXT, | |
INTRODUCTION_TEXT, | |
TITLE, | |
Tasks | |
) | |
from src.display.css_html_js import custom_css | |
from src.display.utils import ( | |
EVAL_COLS, | |
EVAL_TYPES, | |
AutoEvalColumn, | |
ModelType, | |
fields, | |
WeightType, | |
Precision, | |
AREA_DEFINITIONS, | |
AREA_AVG_COLUMN_MAP, | |
PLUE_GROUP_AREAS | |
) | |
from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN | |
from src.populate import get_evaluation_queue_df, get_leaderboard_df | |
from src.submission.submit import add_new_eval | |
# --- TESTE: Carregar dados locais --- | |
TEST_DATA_PATH = "output/leaderboard_results_1.csv" | |
#TEST_DATA_PATH = "output/leaderboard_data_20250413_002339.csv" # Ajuste o caminho se necessário | |
LOAD_TEST_DATA = True # Defina como False para usar dados do Hub | |
# ------- | |
def restart_space(): | |
API.restart_space(repo_id=REPO_ID) | |
### Space initialisation | |
if not LOAD_TEST_DATA: | |
try: | |
print(EVAL_REQUESTS_PATH) | |
snapshot_download( | |
repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN | |
) | |
except Exception as e: # Adicionar captura de exceção | |
print(f"Erro ao baixar EVAL_REQUESTS: {e}") | |
# Considerar restart_space() aqui também, dependendo da severidade | |
try: | |
print(EVAL_RESULTS_PATH) | |
snapshot_download( | |
repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN | |
) | |
except Exception as e: # Adicionar captura de exceção | |
print(f"Erro ao baixar EVAL_RESULTS: {e}") | |
# Considerar restart_space() aqui também | |
else: | |
print(f"Modo de teste: Carregando dados locais de {TEST_DATA_PATH}") | |
EVAL_RESULTS_PATH = None # Não precisamos do caminho do Hub para resultados | |
EVAL_REQUESTS_PATH = "data/eval_requests" # Manter ou ajustar se a fila ainda for lida do Hub | |
# Certifique-se de que o diretório da fila de requests existe se for usado | |
os.makedirs(EVAL_REQUESTS_PATH, exist_ok=True) | |
# Obter todas as colunas definidas | |
ALL_COLS = [c.name for c in fields(AutoEvalColumn)] | |
# Obter o leaderboard completo com as médias calculadas | |
try: | |
initial_df_for_test = None | |
if LOAD_TEST_DATA: | |
try: | |
initial_df_for_test = pd.read_csv(TEST_DATA_PATH) | |
# Renomear colunas do CSV para corresponder às chaves internas | |
rename_map = {} | |
# Mapear tasks (Nome no CSV -> Nome interno da Enum Task) | |
for task in Tasks: | |
rename_map[task.value.col_name] = task.name # Ex: {"Revalida": "REVALIDA"} | |
# Mapear outras colunas (Nome no CSV -> Nome interno de AutoEvalColumn) | |
# Verificar se a coluna existe no CSV antes de adicionar ao mapa | |
csv_columns = initial_df_for_test.columns | |
if "T" in csv_columns: rename_map["T"] = AutoEvalColumn.model_type_symbol.name | |
if "Modelo" in csv_columns: rename_map["Modelo"] = AutoEvalColumn.model.name | |
if "Tipo" in csv_columns: rename_map["Tipo"] = AutoEvalColumn.model_type.name | |
if "Arquitetura" in csv_columns: rename_map["Arquitetura"] = AutoEvalColumn.architecture.name | |
if "Tipo de Peso" in csv_columns: rename_map["Tipo de Peso"] = AutoEvalColumn.weight_type.name | |
if "Precisão" in csv_columns: rename_map["Precisão"] = AutoEvalColumn.precision.name | |
if "Licença" in csv_columns: rename_map["Licença"] = AutoEvalColumn.license.name | |
if "#Params (B)" in csv_columns: rename_map["#Params (B)"] = AutoEvalColumn.params.name | |
if "Hub Likes" in csv_columns: rename_map["Hub Likes"] = AutoEvalColumn.likes.name | |
if "Disponível no hub" in csv_columns: rename_map["Disponível no hub"] = AutoEvalColumn.still_on_hub.name | |
if "SHA do modelo" in csv_columns: rename_map["SHA do modelo"] = AutoEvalColumn.revision.name | |
# Mapear colunas de médias (já devem estar com nome correto se calculadas, mas por segurança) | |
if "Média Geral" in csv_columns: rename_map["Média Geral"] = AutoEvalColumn.average.name | |
if "Área Médica" in csv_columns: rename_map["Área Médica"] = AutoEvalColumn.area_medica_avg.name | |
if "Área do Direito" in csv_columns: rename_map["Área do Direito"] = AutoEvalColumn.area_direito_avg.name | |
if "Provas Militares" in csv_columns: rename_map["Provas Militares"] = AutoEvalColumn.provas_militares_avg.name | |
if "Computação" in csv_columns: rename_map["Computação"] = AutoEvalColumn.computacao_avg.name | |
if "Discurso de Ódio" in csv_columns: rename_map["Discurso de Ódio"] = AutoEvalColumn.discurso_odio_avg.name | |
if "Economia e Contabilidade" in csv_columns: rename_map["Economia e Contabilidade"] = AutoEvalColumn.economia_contabilidade_avg.name | |
if "Semântica e Inferência" in csv_columns: rename_map["Semântica e Inferência"] = AutoEvalColumn.semantica_inferencia_avg.name | |
if "Multidisciplinar" in csv_columns: rename_map["Multidisciplinar"] = AutoEvalColumn.multidisciplinar_avg.name | |
# Aplicar o rename | |
initial_df_for_test.rename(columns=rename_map, inplace=True) | |
print(f"Colunas após renomeação: {initial_df_for_test.columns.tolist()}") # Log para verificar | |
print("DataFrame de teste carregado e colunas renomeadas.") | |
except FileNotFoundError: | |
print(f"Erro: Arquivo de teste não encontrado em {TEST_DATA_PATH}") | |
initial_df_for_test = pd.DataFrame() | |
except Exception as e: | |
print(f"Erro ao carregar ou processar o arquivo de teste: {e}") | |
initial_df_for_test = pd.DataFrame() | |
LEADERBOARD_DF = get_leaderboard_df( | |
results_path=EVAL_RESULTS_PATH if not LOAD_TEST_DATA else None, | |
requests_path=EVAL_REQUESTS_PATH if not LOAD_TEST_DATA else None, | |
cols=ALL_COLS, | |
initial_df=initial_df_for_test | |
) | |
except Exception as e: | |
print(f"Erro ao gerar o DataFrame do Leaderboard: {e}") | |
LEADERBOARD_DF = pd.DataFrame() # Criar DataFrame vazio em caso de erro | |
# Obter DataFrames da fila de avaliação (pode precisar ser ajustado se LOAD_TEST_DATA=True) | |
# Se a fila também deve ser mockada/lida localmente, ajuste aqui | |
( | |
finished_eval_queue_df, | |
running_eval_queue_df, | |
pending_eval_queue_df, | |
) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) | |
def create_leaderboard_component(dataframe, displayed_cols, hidden_cols=None, cant_deselect_cols=None, title=None): | |
if dataframe is None or dataframe.empty: | |
return gr.Markdown(f"## {title or ''}\nNão há dados para exibir.") | |
if hidden_cols is None: | |
hidden_cols = [] | |
if cant_deselect_cols is None: | |
cant_deselect_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] | |
# Filtrar dataframe para conter apenas as colunas a serem exibidas (ou ocultas/não deselecionáveis) | |
all_required_cols = set(displayed_cols) | set(hidden_cols) | set(cant_deselect_cols) | {AutoEvalColumn.model_type.name, AutoEvalColumn.precision.name, AutoEvalColumn.params.name, AutoEvalColumn.still_on_hub.name} | |
available_cols = [col for col in all_required_cols if col in dataframe.columns] | |
filtered_df = dataframe[available_cols].copy() # Usar cópia para evitar SettingWithCopyWarning | |
# Garantir que as colunas 'always visible' estejam presentes | |
for col in cant_deselect_cols: | |
if col not in filtered_df.columns: | |
filtered_df[col] = np.nan # Ou algum valor padrão apropriado | |
# Construir lista de filtros, incluindo None para colunas ausentes | |
raw_filter_columns=[ | |
ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Tipos de Modelo") if AutoEvalColumn.model_type.name in filtered_df.columns else None, | |
ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precisão") if AutoEvalColumn.precision.name in filtered_df.columns else None, | |
ColumnFilter( | |
AutoEvalColumn.params.name, | |
type="slider", | |
min=0.01, | |
max=max(150, filtered_df[AutoEvalColumn.params.name].max(skipna=True) if AutoEvalColumn.params.name in filtered_df.columns and not filtered_df[AutoEvalColumn.params.name].dropna().empty else 150), # Ajustar max dinamicamente e ignorar NaN | |
label="Selecionar número de parâmetros (B)", | |
) if AutoEvalColumn.params.name in filtered_df.columns else None, | |
ColumnFilter( | |
AutoEvalColumn.still_on_hub.name, type="boolean", label="Deletado/incompleto", default=True | |
) if AutoEvalColumn.still_on_hub.name in filtered_df.columns else None, | |
] | |
# Filtrar Nones da lista de filtros | |
final_filter_columns = [f for f in raw_filter_columns if f is not None] | |
# --- Reordenar Colunas --- | |
current_cols = filtered_df.columns.tolist() | |
# Definir as colunas que devem vir primeiro | |
first_cols_desired = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] | |
# Garantir que elas existem no dataframe atual | |
first_cols_actual = [c for c in first_cols_desired if c in current_cols] | |
# Obter as outras colunas | |
other_cols = [c for c in current_cols if c not in first_cols_actual] | |
# Priorizar as colunas que deveriam ser exibidas por padrão (exceto as primeiras) | |
other_displayed_cols = [c for c in displayed_cols if c in other_cols] | |
# Obter as colunas restantes (ocultas por padrão ou não especificadas em displayed_cols) e ordená-las | |
remaining_cols = sorted([c for c in other_cols if c not in other_displayed_cols]) | |
# Montar a ordem final | |
final_order = first_cols_actual + other_displayed_cols + remaining_cols | |
# Aplicar a nova ordem | |
filtered_df = filtered_df[final_order] | |
# --- Fim Reordenar Colunas --- | |
# --- INÍCIO DA MODIFICAÇÃO --- | |
# print(f"--- Info for DataFrame passed to Leaderboard ({title}) ---") | |
# filtered_df.info() | |
# print("----------------------------------------------------------") | |
# --- FIM DA MODIFICAÇÃO --- | |
return Leaderboard( | |
value=filtered_df, # Passar o DataFrame reordenado | |
datatype=[c.type for c in fields(AutoEvalColumn) if c.name in filtered_df.columns], | |
select_columns=SelectColumns( | |
default_selection=displayed_cols, | |
cant_deselect=cant_deselect_cols, | |
label="Selecionar Benchmarks a Serem Exibidos:", | |
), | |
search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name] if AutoEvalColumn.license.name in filtered_df.columns else [AutoEvalColumn.model.name], | |
hide_columns=[c for c in hidden_cols if c in filtered_df.columns], # Ocultar apenas colunas existentes | |
filter_columns=final_filter_columns, # Usar a lista filtrada | |
bool_checkboxgroup_label="Ocultar modelos", | |
interactive=False, | |
) | |
# --- Definição do Grupo PLUE --- | |
PLUE_GENERAL_VIEW_NAME = "Conhecimentos Gerais para Língua Portuguesa" | |
# ------- | |
# Definição do tema verde | |
green_theme = gr.themes.Base(primary_hue=gr.themes.colors.green, secondary_hue=gr.themes.colors.blue, neutral_hue=gr.themes.colors.slate) | |
demo = gr.Blocks(css=custom_css, theme=green_theme) | |
with demo: | |
gr.HTML(TITLE) | |
gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") | |
with gr.Tabs(elem_classes="tab-buttons") as tabs: | |
# --- Definir Ordem das Abas --- | |
tab_index = 0 | |
# 1. Benchmark Geral | |
with gr.TabItem("📊 Resume", id=tab_index): | |
# Colunas a exibir: T, Modelo, Média Geral, PLUE, Energy, Reasoning | |
general_cols_to_display = [ | |
AutoEvalColumn.model_type_symbol.name, # T | |
AutoEvalColumn.model.name, # Modelo | |
AutoEvalColumn.average.name, # Média Geral | |
AutoEvalColumn.plue_avg.name, # Média PLUE | |
AutoEvalColumn.energy_avg.name, # Média Energy (Exibir por padrão) | |
AutoEvalColumn.reasoning_avg.name, # Média Reasoning (Exibir por padrão) | |
] | |
general_cols_to_display = [col for col in general_cols_to_display if col in LEADERBOARD_DF.columns] | |
# Colunas a ocultar: Tasks + Médias Individuais SOMENTE do grupo PLUE + detalhes | |
general_hidden_cols = [task.name for task in Tasks] + \ | |
[AREA_AVG_COLUMN_MAP[area] for area in PLUE_GROUP_AREAS if area in AREA_AVG_COLUMN_MAP] + \ | |
[ | |
AutoEvalColumn.model_type.name, | |
AutoEvalColumn.architecture.name, | |
AutoEvalColumn.weight_type.name, | |
AutoEvalColumn.precision.name, | |
AutoEvalColumn.license.name, | |
AutoEvalColumn.params.name, | |
AutoEvalColumn.likes.name, | |
AutoEvalColumn.still_on_hub.name, | |
AutoEvalColumn.revision.name | |
] | |
create_leaderboard_component( | |
LEADERBOARD_DF, | |
displayed_cols=general_cols_to_display, | |
hidden_cols=[col for col in general_hidden_cols if col in LEADERBOARD_DF.columns], | |
title="Benchmark Geral" | |
) | |
tab_index += 1 | |
# 2. PLUE (Agora apenas com as áreas originais + 3 adicionadas) | |
with gr.TabItem("📚 PLUE", id=tab_index) as plue_tab: | |
# --- Lógica interna da aba PLUE (ajustada) --- | |
gr.Markdown("## Selecione a visualização PLUE:") | |
# RECALCULAR choices e options com base na PLUE_GROUP_AREAS atualizada (sem Energy/Reasoning) | |
all_plue_options = [PLUE_GENERAL_VIEW_NAME] + sorted(PLUE_GROUP_AREAS) | |
plue_dropdown = gr.Dropdown( | |
choices=all_plue_options, | |
label="Visualização PLUE", | |
value=PLUE_GENERAL_VIEW_NAME | |
) | |
# Função auxiliar (lógica interna não muda, mas opera sobre PLUE_GROUP_AREAS atualizada) | |
def get_plue_leaderboard_config(selected_option): | |
if selected_option == PLUE_GENERAL_VIEW_NAME: | |
displayed_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name,] + [AREA_AVG_COLUMN_MAP[area] for area in PLUE_GROUP_AREAS if area in AREA_AVG_COLUMN_MAP] | |
hidden_cols = [task.name for task in Tasks] + [avg_col for area, avg_col in AREA_AVG_COLUMN_MAP.items() if area not in PLUE_GROUP_AREAS] + [AutoEvalColumn.average.name] + [AutoEvalColumn.plue_avg.name, AutoEvalColumn.model_type.name, AutoEvalColumn.architecture.name, AutoEvalColumn.weight_type.name, AutoEvalColumn.precision.name, AutoEvalColumn.license.name, AutoEvalColumn.params.name, AutoEvalColumn.likes.name, AutoEvalColumn.still_on_hub.name, AutoEvalColumn.revision.name] | |
title = PLUE_GENERAL_VIEW_NAME | |
else: | |
selected_area = selected_option | |
tasks_in_area = AREA_DEFINITIONS[selected_area] | |
displayed_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name,] + [task.name for task in tasks_in_area] | |
hidden_cols = list(AREA_AVG_COLUMN_MAP.values()) + [task.name for task in Tasks if task not in tasks_in_area] + [AutoEvalColumn.plue_avg.name, AutoEvalColumn.average.name, AutoEvalColumn.model_type.name, AutoEvalColumn.architecture.name, AutoEvalColumn.weight_type.name, AutoEvalColumn.precision.name, AutoEvalColumn.license.name, AutoEvalColumn.params.name, AutoEvalColumn.likes.name, AutoEvalColumn.still_on_hub.name, AutoEvalColumn.revision.name] | |
title = selected_area | |
final_hidden_cols = [col for col in hidden_cols if col in LEADERBOARD_DF.columns] | |
return displayed_cols, final_hidden_cols, title | |
# Pré-renderização (ATUALIZAR loop e containers com novas all_plue_options) | |
plue_containers = {} | |
for option in all_plue_options: | |
displayed_cols, hidden_cols, title = get_plue_leaderboard_config(option) | |
is_visible = (option == PLUE_GENERAL_VIEW_NAME) | |
with gr.Group(visible=is_visible) as plue_containers[option]: | |
create_leaderboard_component(LEADERBOARD_DF, displayed_cols=displayed_cols, hidden_cols=hidden_cols, title=title) | |
# Função de callback (ATUALIZAR loop com novas all_plue_options) | |
def switch_plue_view(selected_option): | |
update_list = [] | |
for option in all_plue_options: | |
update_list.append(gr.update(visible=(option == selected_option))) | |
return update_list | |
# Evento change (ATUALIZAR outputs com novas all_plue_options) | |
plue_dropdown.change(fn=switch_plue_view, inputs=[plue_dropdown], outputs=[plue_containers[option] for option in all_plue_options]) | |
# --- Fim Lógica PLUE --- | |
tab_index += 1 | |
# 3. Energy | |
with gr.TabItem("⚡️ Energy", id=tab_index): | |
# Exibir leaderboard com dados de Energy | |
energy_tasks = AREA_DEFINITIONS.get("Energy", []) | |
energy_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + [t.name for t in energy_tasks] | |
energy_hidden = [t.name for t in Tasks if t not in energy_tasks] + \ | |
list(AREA_AVG_COLUMN_MAP.values()) + \ | |
[AutoEvalColumn.plue_avg.name, AutoEvalColumn.average.name] + \ | |
[c.name for c in fields(AutoEvalColumn) if c.name not in energy_cols and c.name != AutoEvalColumn.model_type_symbol.name and c.name != AutoEvalColumn.model.name ] # Detalhes | |
create_leaderboard_component(LEADERBOARD_DF, displayed_cols=energy_cols, hidden_cols=[c for c in energy_hidden if c in LEADERBOARD_DF.columns], title="Energy") | |
tab_index += 1 | |
# 4. Reasoning | |
with gr.TabItem("🤔 Reasoning", id=tab_index): | |
# Exibir leaderboard com dados de Reasoning | |
reasoning_tasks = AREA_DEFINITIONS.get("Reasoning", []) | |
reasoning_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + [t.name for t in reasoning_tasks] | |
reasoning_hidden = [t.name for t in Tasks if t not in reasoning_tasks] + \ | |
list(AREA_AVG_COLUMN_MAP.values()) + \ | |
[AutoEvalColumn.plue_avg.name, AutoEvalColumn.average.name] + \ | |
[c.name for c in fields(AutoEvalColumn) if c.name not in reasoning_cols and c.name != AutoEvalColumn.model_type_symbol.name and c.name != AutoEvalColumn.model.name ] # Detalhes | |
create_leaderboard_component(LEADERBOARD_DF, displayed_cols=reasoning_cols, hidden_cols=[c for c in reasoning_hidden if c in LEADERBOARD_DF.columns], title="Reasoning") | |
tab_index += 1 | |
# 5. Submit | |
with gr.TabItem("📤 Submit!", id=tab_index): | |
with gr.Column(): | |
with gr.Row(): | |
gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") | |
with gr.Column(): | |
with gr.Accordion( | |
f"✅ Avaliações Concluídas ({len(finished_eval_queue_df)})", | |
open=False, | |
): | |
with gr.Row(): | |
finished_eval_table = gr.components.Dataframe( | |
value=finished_eval_queue_df, | |
headers=EVAL_COLS, | |
datatype=EVAL_TYPES, | |
row_count=5, | |
) | |
with gr.Accordion( | |
f"🔄 Fila de Avaliação em Execução ({len(running_eval_queue_df)})", | |
open=False, | |
): | |
with gr.Row(): | |
running_eval_table = gr.components.Dataframe( | |
value=running_eval_queue_df, | |
headers=EVAL_COLS, | |
datatype=EVAL_TYPES, | |
row_count=5, | |
) | |
with gr.Accordion( | |
f"⏳ Fila de Avaliação Pendente ({len(pending_eval_queue_df)})", | |
open=False, | |
): | |
with gr.Row(): | |
pending_eval_table = gr.components.Dataframe( | |
value=pending_eval_queue_df, | |
headers=EVAL_COLS, | |
datatype=EVAL_TYPES, | |
row_count=5, | |
) | |
with gr.Row(): | |
gr.Markdown("# ✉️✨ Submeta seu modelo aqui!", elem_classes="markdown-text") | |
with gr.Row(): | |
with gr.Column(): | |
model_name_textbox = gr.Textbox(label="Nome do Modelo") | |
revision_name_textbox = gr.Textbox(label="Commit da Revisão", placeholder="main") | |
model_type = gr.Dropdown( | |
choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown], | |
label="Tipo do Modelo", | |
multiselect=False, | |
value=None, | |
interactive=True, | |
) | |
with gr.Column(): | |
precision = gr.Dropdown( | |
choices=[i.value.name for i in Precision if i != Precision.Unknown], | |
label="Precisão", | |
multiselect=False, | |
value="float16", | |
interactive=True, | |
) | |
weight_type = gr.Dropdown( | |
choices=[i.value.name for i in WeightType], | |
label="Tipo dos Pesos", | |
multiselect=False, | |
value="Original", | |
interactive=True, | |
) | |
base_model_name_textbox = gr.Textbox(label="Modelo Base (para pesos delta ou adapter)") | |
submit_button = gr.Button("Submeter Avaliação") | |
submission_result = gr.Markdown() | |
submit_button.click( | |
add_new_eval, | |
[ | |
model_name_textbox, | |
base_model_name_textbox, | |
revision_name_textbox, | |
precision, | |
weight_type, | |
model_type, | |
], | |
submission_result, | |
) | |
with gr.Row(): | |
with gr.Accordion("📙 Citação", open=False): | |
citation_button = gr.Textbox( | |
value=CITATION_BUTTON_TEXT, | |
label=CITATION_BUTTON_LABEL, | |
lines=20, | |
elem_id="citation-button", | |
show_copy_button=True, | |
) | |
scheduler = BackgroundScheduler() | |
scheduler.add_job(restart_space, "interval", seconds=1800) | |
scheduler.start() | |
demo.queue(default_concurrency_limit=40).launch() |