Adding updated app.py

Signed-off-by: Taejin Park <[email protected]>

app.py

@@ -1,12 +1,10 @@
 import os
 import json
-import csv
 import datetime
 from email.utils import parseaddr
 
 import gradio as gr
 import pandas as pd
-import numpy as np
 
 from datasets import load_dataset
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -16,19 +14,15 @@ from scorer import instruction_scorer
 from content import format_error, format_warning, format_log, TITLE, INTRODUCTION_TEXT, SUBMISSION_TEXT, CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, model_hyperlink
 
 TOKEN = os.environ.get("TOKEN", None)
-OWNER="ucla-contextual"
-TEST_DATASET = f"{OWNER}/contextual_test"
-VAL_DATASET = f"{OWNER}/contextual_val"
-SUBMISSION_DATASET = f"{OWNER}/submissions_internal"
-CONTACT_DATASET = f"{OWNER}/contact_info"
-RESULTS_DATASET = f"{OWNER}/results"
+OWNER="Taejin"
+REF_JSONS_SET = f"{OWNER}/speaker_tagging_lb_refs"
+RESULTS_DATASET = f"{OWNER}/spk_tag_results"
 LEADERBOARD_PATH = f"{OWNER}/leaderboard"
+SUBMISSION_DATASET = f"{OWNER}/submission_leaderboard"
 api = HfApi()
 
 YEAR_VERSION = "2024"
 
-results = {"dev": {"cpWER": 0, "W
-
 def read_json_file(filepath):
     with open(filepath) as infile:
         data_dict = json.load(infile)
@@ -40,50 +34,39 @@ def save_json_file(filepath, data_dict):
                                 
 os.makedirs("scored", exist_ok=True)
 
-# test_data_files = {"test": "contextual_test.csv"}
-# test_dataset = load_dataset(TEST_DATASET, data_files=test_data_files , token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
-
-# val_data_files = {"val": "contextual_val.csv"}
-# val_dataset = load_dataset(VAL_DATASET, data_files=val_data_files , token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
+results_data_files = {"dev": "dev_set_data.csv", "eval": "eval_set_data.csv"}
+results = load_dataset(RESULTS_DATASET, data_files=results_data_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
 
-# results_data_files = {"test": "contextual_test_results.csv", "val": "contextual_val_results.csv"}
-# results = load_dataset(RESULTS_DATASET, data_files=results_data_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
 
-# contacts_data_files = {"contacts": "contacts.csv"}
-# contact_infos = load_dataset(CONTACT_DATASET, data_files=contacts_data_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
+ref_json_files = {"dev_src": "err_dev.src.seglst.json", "dev_ref": "err_dev.ref.seglst.json", "eval_src": "err_eval.src.seglst.json", "eval_ref": "err_eval.ref.seglst.json"}
+ref_jsons = load_dataset(REF_JSONS_SET, data_files=ref_json_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
 
 def get_dataframe_from_results(results, split):
     df = results[split].to_pandas()
-    df.drop(columns=['URL'], inplace=True)
-    df = df.sort_values(by=["All"], ascending=False)
+    df = df.sort_values(by=["cpWER"], ascending=True)
     return df
 
-# test_dataset_dataframe = test_dataset["test"].to_pandas()
-# val_dataset_dataframe = val_dataset["val"].to_pandas()
-
-# contacts_dataframe = contact_infos["contacts"].to_pandas()
-
-# val_results_dataframe = get_dataframe_from_results(results=results, split="val")
-# test_results_dataframe = get_dataframe_from_results(results=results, split="test")
 
 def restart_space():
     api.restart_space(repo_id=LEADERBOARD_PATH, token=TOKEN)
 
-TYPES = ["markdown", "markdown", "markdown", "number", "number", "number","number", "number", "number", "number", "number", "number"]
+TYPES = ["markdown", "markdown", "markdown", "number", "number", "number"]
+
+dev_dataset_dataframe= get_dataframe_from_results(results=results, split="dev")
+eval_dataset_dataframe= get_dataframe_from_results(results=results, split="eval")
 
 def add_new_eval(
-    model: str,
+    system_name: str,
     method: str,
-    url: str,
     path_to_file: str,
     organisation: str,
     mail: str,
 ):
-    print("printing all inputs:", model, method, url, path_to_file, organisation, mail)
+    print(f"printing all inputs system_name: {system_name},  method: {method}, path_to_file: {path_to_file}, organisation: {organisation}, mail: {mail}")
 
-    if len(model)==0:
-        print("model none")
-        raise gr.Error("Please provide a model name. Field empty!")
+    if len(system_name)==0:
+        print("system_name none")
+        raise gr.Error("Please provide a system_name name. Field empty!")
     
     if len(method)==0:
         print("method none")
@@ -99,121 +82,123 @@ def add_new_eval(
         print("email here")
         raise gr.Error("Please provide a valid email address.")
     
-
-    # Check if the combination model/org already exists and prints a warning message if yes
-    if model.lower() in set([m.lower() for m in results["val"]["Model"]]) and organisation.lower() in set([o.lower() for o in results["val"]["Organisation"]]):
-        print("model org combo here")
-        raise gr.Error("This model has been already submitted.")
+    # Check if the combination system_name/org already exists and prints a warning message if yes
+    # if system_name.lower() in set([m.lower() for m in results["dev"]["System_name"]]) and organisation.lower() in set([o.lower() for o in results["dev"]["Organisation"]]):
+    #     print("system_name org combo here")
+    #     raise gr.Error("This system_name has been already submitted.")
 
     if path_to_file is None:
         print("file missing here")
         raise gr.Error("Please attach a file.")
 
-    tmp_file_output = read_json_file(path_to_file.name)
-    
-    if len(tmp_file_output.keys())!=1:
-        print("file format wrong here")
-        raise gr.Error("Submission file format incorrect. Please refer to the format description!")
-    
-    tmp_output_key = list(tmp_file_output.keys())[0]
-    if len(tmp_file_output[tmp_output_key].keys())!=100:
-        print("file not 100 here")
-        raise gr.Error("File must contain exactly 100 predictions.")
     
     # Save submitted file
     time_atm = datetime.datetime.today()
     api.upload_file(
         repo_id=SUBMISSION_DATASET, 
         path_or_fileobj=path_to_file.name, 
-        path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_raw_{time_atm}.json",
+        path_in_repo=f"{organisation}/{system_name}/{YEAR_VERSION}_raw_{time_atm}.json",
         repo_type="dataset", 
         token=TOKEN
     )
 
     # Compute score
-    file_path = path_to_file.name        
-    scores = instruction_scorer(val_dataset_dataframe, file_path , model)
-
-    path_or_fileobj=f"scored/{organisation}_{model}.json"
-    save_json_file(path_or_fileobj, scores)
+    if "err_dev.hyp.seglst.json" in path_to_file.name:
+        ref_file_path="seglst_files/err_dev.ref.seglst.json"
+        mode = "dev"
+    elif "err_eval.hyp.seglst.json" in path_to_file.name:
+        ref_file_path="seglst_files/err_eval.ref.seglst.json"
+        mode = "eval"
+    else:
+        basename = os.path.basename(path_to_file.name)
+        raise gr.Error(f"{basename} is NOT a valid name. It should be either err_dev.hyp.seglst.json or err_eval.hyp.seglst.json")
+    scores =  instruction_scorer(file_path_input= path_to_file.name, ref_file_path=ref_file_path,  system_name=system_name)
+
+    path_or_fileobj=f"scored/{organisation}_{system_name}.json"
+    scores_and_info = {
+            "system_name": system_name,
+            "method":method,
+            "organisation": organisation,
+            "email": mail, 
+            "cpWER": scores["cpWER"],
+            "errors": scores["errors"],
+            "length": scores["length"],
+        }
+    save_json_file(path_or_fileobj, data_dict=scores_and_info)
 
     # Save scored file
     api.upload_file(
         repo_id=SUBMISSION_DATASET, 
         path_or_fileobj=path_or_fileobj,
-        path_in_repo=f"{organisation}/{model}/{YEAR_VERSION}_scored_{time_atm}.json", 
+        path_in_repo=f"{organisation}/{system_name}/{YEAR_VERSION}_scored_{time_atm}.json", 
         repo_type="dataset", 
         token=TOKEN
     )
 
     # Actual submission
     eval_entry = {
-        "Model": model,
-        "Method":method,
-        "Organisation": organisation,
-        "URL": url,
-        "All":scores["average"],
-        "Time":scores["time"],
-        "Shopping":scores["shopping"],
-        "Navigation":scores["navigation-transportation"],
-        "Abstract":scores["abstract"],
-        "Application Usage":scores["app"],
-        "Web Usage":scores["web"],
-        "Infographic":scores["infographics"],
-        "Miscellaneous Natural Scenes": scores["misc"]
-    }
-
-    val_results_dataframe = get_dataframe_from_results(results=results, split="val")
-    val_results_dataframe = pd.concat([val_results_dataframe, pd.DataFrame([eval_entry])], ignore_index=True)
-    val_results_dataframe.to_csv('contextual_val_results.csv', index=False)
-
-    api.upload_file(
-        repo_id=RESULTS_DATASET, 
-        path_or_fileobj="contextual_val_results.csv",
-        path_in_repo=f"contextual_val_results.csv", 
-        repo_type="dataset", 
-        token=TOKEN
-    )
-
-    contact_info = {
-        "Model": model,
-        "URL": url,
-        "Organisation": organisation,
-        "Mail": mail,
+        "system_name": system_name,
+        "method":method,
+        "organisation": organisation,
+        "cpWER":scores["cpWER"],
+        "errors":scores["errors"],
+        "length":scores["length"],
     }
+    
+    if mode == "dev":
+        dev_set_data_csv = "dev_set_data.csv"
+        dev_dataset_dataframe = get_dataframe_from_results(results=results, split="dev")
+        dev_dataset_dataframe = pd.concat([dev_dataset_dataframe, pd.DataFrame([eval_entry])], ignore_index=True)
+        dev_dataset_dataframe.to_csv(dev_set_data_csv, index=False)
+
+        api.upload_file(
+            repo_id=RESULTS_DATASET, 
+            path_or_fileobj=dev_set_data_csv,
+            path_in_repo=dev_set_data_csv,
+            repo_type="dataset", 
+            token=TOKEN
+        )
+    elif mode == "eval":
+        eval_set_data_csv = "eval_set_data.csv"
+        eval_dataset_dataframe = get_dataframe_from_results(results=results, split="eval")
+        eval_dataset_dataframe = pd.concat([eval_dataset_dataframe, pd.DataFrame([eval_entry])], ignore_index=True)
+        eval_dataset_dataframe.to_csv(eval_set_data_csv, index=False)
+
+        api.upload_file(
+            repo_id=RESULTS_DATASET, 
+            path_or_fileobj=eval_set_data_csv,
+            path_in_repo=eval_set_data_csv,
+            repo_type="dataset", 
+            token=TOKEN
+        )
 
-    contacts_dataframe = contact_infos["contacts"].to_pandas()
-    contacts_dataframe = pd.concat([contacts_dataframe, pd.DataFrame([contact_info])], ignore_index=True)
-    contacts_dataframe.to_csv('contacts.csv', index=False)
-
-    api.upload_file(
-        repo_id=CONTACT_DATASET, 
-        path_or_fileobj="contacts.csv",
-        path_in_repo=f"contacts.csv", 
-        repo_type="dataset", 
-        token=TOKEN
-    )
-
-    return format_log(f"Model {model} submitted by {organisation} successfully! \nPlease refresh the val leaderboard, and wait a bit to see the score displayed")
+    return format_log(f"system_name {system_name} submitted by {organisation} successfully! \nPlease refresh the val leaderboard, and wait a bit to see the score displayed")
 
 
 def refresh():
-    results_data_files = {"test": "contextual_test_results.csv", "val": "contextual_val_results.csv"}
-    results = load_dataset(RESULTS_DATASET, data_files=
-    results_data_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
-    val_results_dataframe = get_dataframe_from_results(results=results, split="val")
-    test_results_dataframe = get_dataframe_from_results(results=results, split="test")
-    return val_results_dataframe, test_results_dataframe
+    results_data_files = {"dev": "dev_set_data.csv", "eval": "eval_set_data.csv"}
+    results = load_dataset(RESULTS_DATASET, data_files=results_data_files, token=TOKEN, download_mode="force_redownload", ignore_verifications=True)
+    dev_results_dataframe = get_dataframe_from_results(results=results, split="dev")
+    eval_results_dataframe = get_dataframe_from_results(results=results, split="eval")
+    return dev_results_dataframe, eval_results_dataframe
 
 def upload_file(files):
     file_paths = [file.name for file in files]
     return file_paths
 
+for file_key in ['dev_src', 'dev_ref', 'eval_src', 'eval_ref']:
+    ref_jsons[file_key].to_json(path_or_buf=f"seglst_files/{file_key}.json")
+    buff_list = [x.strip() for x in open(f"seglst_files/{file_key}.json").readlines()]
+    buff_str = ",\n".join(buff_list)
+    seglst_json = f"[\n{buff_str}\n]"
+    split, datatype = file_key.split("_")
+    with open(f"seglst_files/err_{split}.{datatype}.seglst.json", "w") as f:
+        f.write(seglst_json)
 
 demo = gr.Blocks()
 with demo:
     gr.HTML(TITLE)
-    # gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Row():
         with gr.Accordion("🧐 Introduction", open=False):
@@ -230,14 +215,15 @@ with demo:
                 label=CITATION_BUTTON_LABEL,
                 elem_id="citation-button",
             ) 
-    with gr.Tab("Results: Test"):
-        leaderboard_table_test = gr.components.Dataframe(
-            value=test_results_dataframe, datatype=TYPES, interactive=False,
+    with gr.Tab("Results: Dev"):
+        leaderboard_table_dev  = gr.components.Dataframe(
+            value=dev_dataset_dataframe, datatype=TYPES, interactive=False,
             column_widths=["20%"] 
         )
-    with gr.Tab("Results: Val"):
-        leaderboard_table_val = gr.components.Dataframe(
-            value=val_results_dataframe, datatype=TYPES, interactive=False,
+        
+    with gr.Tab("Results: Eval"):
+        leaderboard_table_eval = gr.components.Dataframe(
+            value=eval_dataset_dataframe, datatype=TYPES, interactive=False,
             column_widths=["20%"] 
         )
 
@@ -246,18 +232,17 @@ with demo:
         refresh,
         inputs=[],
         outputs=[
-            leaderboard_table_val,
-            leaderboard_table_test,
+            leaderboard_table_dev,
+            leaderboard_table_eval,
         ],
     )
-    with gr.Accordion("Submit a new model for evaluation"):
+    with gr.Accordion("Submit a new system_name for evaluation"):
         with gr.Row():
             with gr.Column():
-                model_name_textbox = gr.Textbox(label="Model name", type='text')
-                method_textbox = gr.Textbox(label="Method (LMM or Aug LLM or any other)", type='text')
-                url_textbox = gr.Textbox(label="URL to model information", type='text')
+                system_name_textbox = gr.Textbox(label="System name", type='text')
+                method_textbox = gr.Textbox(label="Method (LLM with prompt, beam-search, etc)", type='text')
             with gr.Column():
-                organisation = gr.Textbox(label="Organisation", type='text')
+                organisation = gr.Textbox(label="Organisation or Team Name", type='text')
                 mail = gr.Textbox(label="Contact email (will be stored privately, & used if there is an issue with your submission)", type='email')
                 file_output = gr.File()
 
@@ -267,9 +252,8 @@ with demo:
         submit_button.click(
             add_new_eval,
             [
-                model_name_textbox,
+                system_name_textbox,
                 method_textbox,
-                url_textbox,
                 file_output,
                 organisation,
                 mail

content.py

@@ -76,14 +76,7 @@ There should be 506 predictions, corresponding to the 506 urls of the test set.
 
 
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""@misc{wadhawan2024contextual,
-      title={ConTextual: Evaluating Context-Sensitive Text-Rich Visual Reasoning in Large Multimodal Models}, 
-      author={Rohan Wadhawan and Hritik Bansal and Kai-Wei Chang and Nanyun Peng},
-      year={2024},
-      eprint={2401.13311},
-      archivePrefix={arXiv},
-      primaryClass={cs.CV}
-}"""
+CITATION_BUTTON_TEXT = ""
 
 
 def format_error(msg):

scorer.py

@@ -16,15 +16,15 @@ def instruction_scorer(file_path_input, ref_file_path,  system_name):
     subprocess.run(cmd_hyp)
 
     # Read the JSON file and print the cpWER
-    asrdiar_file_name="err_dev"
-    output_cpwer_hyp_json_file = os.path.join(f"{asrdiar_file_name}.hyp.seglst_cpwer.json")
+    print("file_path_input:", file_path_input)
+    output_cpwer_hyp_json_file = file_path_input.replace(".hyp.seglst.json", ".hyp.seglst_cpwer.json")
+    
     with open(output_cpwer_hyp_json_file, "r") as temp_file:
         data_h = json.load(temp_file)
         print("Hypothesis cpWER:", data_h["error_rate"])
     cpwer = data_h["error_rate"]
     logging.info(f"-> HYPOTHESIS cpWER={cpwer:.4f}")
-
-    scores_dict = {"cpWER": cpwer, "WER": cpwer}
+    scores_dict = {"cpWER": cpwer, "errors": data_h["errors"], "length": data_h["length"]}
     return scores_dict