scandeval_summary.py

import argparse
import pandas as pd

#Run ScandEval on model
#scandeval --model-id KennethTM/bert-base-uncased-danish

#Format results as markdown
#python scandeval_summary.py --name scandeval_benchmark_results.jsonl

def scandeval2md(file):
    df = pd.read_json(file, lines=True)

    df["total"] = [i["total"] for i in df["results"]]

    summary = []
    for i in df["total"]:

        summary_list = []

        for k, v in i.items():
            summary_string = ""

            if "_se" in k:
                summary_string += f" (±{round(v, 2)})\n"
            else:
                summary_string += f'{k.replace("test_", "")} = {round(v, 2)}'

            summary_list.append(summary_string)
        
        summary.append("".join(summary_list).strip())

    df["summary"] = summary

    md_table = df[["task", "dataset", "summary"]].rename(columns={"task": "Task", "dataset": "Dataset", "summary": "Score (±SE)"}).to_markdown(index=False)

    print(md_table)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Script for formatting ScandEval .jsonl result file as markdown to standard output')
    parser.add_argument('--name', help='ScandEval .jsonl result file')
    args = parser.parse_args()

    scandeval2md(args.name)