refactor

Dockerfile

@@ -34,4 +34,4 @@ SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
 COPY --chown=1000:1000 . /app/
 RUN python setup.py install
 
-CMD python -c "import competitions; print(competitions.__version__)"
+CMD competitions run

competitions/__init__.py

@@ -1,4 +1,14 @@
-from . import config
+import os
+
+from .info import CompetitionInfo
 
 
 __version__ = "0.0.1"
+
+MOONLANDING_URL = os.getenv("MOONLANDING_URL")
+COMPETITION_ID = os.getenv("COMPETITION_ID")
+AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
+AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
+AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
+
+competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=AUTOTRAIN_TOKEN)

competitions/app.py

@@ -1,160 +0,0 @@
-import uuid
-from datetime import datetime
-from functools import partial
-
-import config
-import gradio as gr
-import pandas as pd
-import utils
-from huggingface_hub import HfApi
-from huggingface_hub.utils._errors import EntryNotFoundError
-
-
-SUBMISSION_TEXT = f"""You can make upto {config.competition_info.submission_limit} submissions per day.
-The test data has been divided into public and private splits.
-Your score on the public split will be shown on the leaderboard.
-Your final score will be based on your private split performance.
-The final rankings will be based on the private split performance.
-"""
-
-SUBMISSION_ERROR = """Submission is not in a proper format.
-Please check evaluation instructions for more details."""
-
-SUBMISSION_LIMIT_TEXT = f"""You can select upto {config.competition_info.selection_limit}
- submissions for private leaderboard."""
-
-
-def get_subs(user_info, private=False):
-    # get user submissions
-    user_id = user_info["id"]
-    try:
-        user_submissions = utils.fetch_submissions(user_id)
-    except EntryNotFoundError:
-        return_value = "No submissions found"
-        return [gr.Markdown.update(visible=True, value=return_value), gr.DataFrame.update(visible=False)]
-    submissions_df = pd.DataFrame(user_submissions)
-    if not private:
-        submissions_df = submissions_df.drop(columns=["private_score"])
-        submissions_df = submissions_df[
-            ["date", "submission_id", "public_score", "submission_comment", "selected", "status"]
-        ]
-    else:
-        submissions_df = submissions_df[
-            ["date", "submission_id", "public_score", "private_score", "submission_comment", "selected", "status"]
-        ]
-    return [gr.Markdown.update(visible=False), gr.DataFrame.update(visible=True, value=submissions_df)]
-
-
-def my_submissions(user_token):
-    if user_token != "":
-        user_info = utils.user_authentication(token=user_token)
-        if "error" in user_info:
-            return_value = "Invalid token"
-            return [gr.Markdown.update(visible=True, value=return_value), gr.DataFrame.update(visible=False)]
-
-        if user_info["emailVerified"] is False:
-            return_value = "Please verify your email on Hugging Face Hub"
-            return [gr.Markdown.update(visible=True, value=return_value), gr.DataFrame.update(visible=False)]
-
-        current_date_time = datetime.now()
-        private = False
-        if current_date_time >= config.competition_info.end_date:
-            private = True
-        subs = get_subs(user_info, private=private)
-        return subs
-    return [gr.Markdown.update(visible=True, value="Invalid token"), gr.DataFrame.update(visible=False)]
-
-
-def new_submission(user_token, uploaded_file):
-    if uploaded_file is not None and user_token != "":
-        # verify token
-        user_info = utils.user_authentication(token=user_token)
-        if "error" in user_info:
-            return "Invalid token"
-
-        if user_info["emailVerified"] is False:
-            return "Please verify your email on Hugging Face Hub"
-
-        # check if user can submit to the competition
-        if utils.check_user_submission_limit(user_info) is False:
-            return "You have reached your submission limit for today"
-        with open(uploaded_file.name, "rb") as f:
-            bytes_data = f.read()
-        # verify file is valid
-        if not utils.verify_submission(bytes_data):
-            return "Invalid submission"
-            # gr.Markdown(SUBMISSION_ERROR)
-            # write a horizontal html line
-            # gr.Markdown("<hr/>", unsafe_allow_html=True)
-        else:
-            user_id = user_info["id"]
-            submission_id = str(uuid.uuid4())
-            file_extension = uploaded_file.orig_name.split(".")[-1]
-            # upload file to hf hub
-            api = HfApi()
-            api.upload_file(
-                path_or_fileobj=bytes_data,
-                path_in_repo=f"submissions/{user_id}-{submission_id}.{file_extension}",
-                repo_id=config.COMPETITION_ID,
-                repo_type="dataset",
-                token=config.AUTOTRAIN_TOKEN,
-            )
-            # update submission limit
-            submissions_made = utils.increment_submissions(
-                user_id=user_id,
-                submission_id=submission_id,
-                submission_comment="",
-            )
-            # schedule submission for evaluation
-            utils.create_project(
-                project_id=f"{submission_id}",
-                dataset=f"{config.COMPETITION_ID}",
-                submission_dataset=user_id,
-                model="generic_competition",
-            )
-        return_text = f"Submission scheduled for evaluation. You have {config.competition_info.submission_limit - submissions_made} submissions left for today."
-        return return_text
-    return "Error"
-
-
-with gr.Blocks() as demo:
-    with gr.Tab("Overview"):
-        gr.Markdown(f"# Welcome to {config.competition_info.competition_name}! 👋")
-
-        gr.Markdown(f"{config.competition_info.competition_description}")
-
-        gr.Markdown("## Dataset")
-        gr.Markdown(f"{config.competition_info.dataset_description}")
-
-    with gr.Tab("Public Leaderboard"):
-        output_markdown = gr.Markdown("")
-        fetch_lb = gr.Button("Fetch Leaderboard")
-        fetch_lb_partial = partial(utils.fetch_leaderboard, private=False)
-        fetch_lb.click(fn=fetch_lb_partial, outputs=[output_markdown])
-        # lb = utils.fetch_leaderboard(private=False)
-        # gr.Markdown(lb.to_markdown())
-    with gr.Tab("Private Leaderboard"):
-        current_date_time = datetime.now()
-        if current_date_time >= config.competition_info.end_date:
-            lb = utils.fetch_leaderboard(private=True)
-            gr.Markdown(lb)
-        else:
-            gr.Markdown("Private Leaderboard will be available after the competition ends")
-    with gr.Tab("New Submission"):
-        gr.Markdown(SUBMISSION_TEXT)
-        user_token = gr.Textbox(max_lines=1, value="hf_XXX", label="Please enter your Hugging Face token")
-        uploaded_file = gr.File()
-        output_text = gr.Markdown(visible=True, show_label=False)
-        new_sub_button = gr.Button("Upload Submission")
-        new_sub_button.click(fn=new_submission, inputs=[user_token, uploaded_file], outputs=[output_text])
-
-    with gr.Tab("My Submissions"):
-        gr.Markdown(SUBMISSION_LIMIT_TEXT)
-        user_token = gr.Textbox(max_lines=1, value="hf_XXX", label="Please enter your Hugging Face token")
-        output_text = gr.Markdown(visible=True, show_label=False)
-        output_df = gr.Dataframe(visible=False)
-        my_subs_button = gr.Button("Fetch Submissions")
-        my_subs_button.click(fn=my_submissions, inputs=[user_token], outputs=[output_text, output_df])
-
-if __name__ == "__main__":
-    demo.launch()

competitions/cli/__init__.py

@@ -0,0 +1,13 @@
+from abc import ABC, abstractmethod
+from argparse import ArgumentParser
+
+
+class BaseCompetitionsCommand(ABC):
+    @staticmethod
+    @abstractmethod
+    def register_subcommand(parser: ArgumentParser):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def run(self):
+        raise NotImplementedError()

competitions/cli/competitions.py

@@ -0,0 +1,42 @@
+import argparse
+import sys
+
+from loguru import logger
+
+from .. import __version__
+from .run import RunCompetitionsAppCommand
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        "Competitions CLI",
+        usage="competitions <command> [<args>]",
+        epilog="For more information about a command, run: `competitions <command> --help`",
+    )
+    parser.add_argument("--version", "-v", help="Display competitions version", action="store_true")
+    commands_parser = parser.add_subparsers(help="commands")
+
+    # Register commands
+    RunCompetitionsAppCommand.register_subcommand(commands_parser)
+
+    args = parser.parse_args()
+
+    if args.version:
+        print(__version__)
+        exit(0)
+
+    if not hasattr(args, "func"):
+        parser.print_help()
+        exit(1)
+
+    command = args.func(args)
+
+    try:
+        command.run()
+    except Exception as e:
+        logger.error(e)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

competitions/cli/run.py

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+
+from . import BaseCompetitionsCommand
+
+
+def run_app_command_factory(args):
+    return RunCompetitionsAppCommand()
+
+
+class RunCompetitionsAppCommand(BaseCompetitionsCommand):
+    @staticmethod
+    def register_subcommand(parser: ArgumentParser):
+        create_project_parser = parser.add_parser("run", description="✨ Run competitions app")
+        # create_project_parser.add_argument("--name", type=str, default=None, required=True, help="The project's name")
+        create_project_parser.set_defaults(func=run_app_command_factory)
+
+    # def __init__(self):
+    #     self._name = name
+    #     self._task = task
+    #     self._lang = language
+    #     self._max_models = max_models
+    #     self._hub_model = hub_model
+
+    def run(self):
+        from ..competitions import demo
+
+        demo.launch()

competitions/competitions.py

@@ -0,0 +1,61 @@
+import gradio as gr
+
+from . import competition_info
+from .leaderboard import Leaderboard
+from .submissions import Submissions
+
+
+# with gr.Blocks() as demo:
+#     with gr.Tab("Overview"):
+#         gr.Markdown(f"# Welcome to {config.competition_info.competition_name}! 👋")
+
+#         gr.Markdown(f"{config.competition_info.competition_description}")
+
+#         gr.Markdown("## Dataset")
+#         gr.Markdown(f"{config.competition_info.dataset_description}")
+
+#     with gr.Tab("Public Leaderboard"):
+#         output_markdown = gr.Markdown("")
+#         fetch_lb = gr.Button("Fetch Leaderboard")
+#         fetch_lb_partial = partial(utils.fetch_leaderboard, private=False)
+#         fetch_lb.click(fn=fetch_lb_partial, outputs=[output_markdown])
+#         # lb = utils.fetch_leaderboard(private=False)
+#         # gr.Markdown(lb.to_markdown())
+#     with gr.Tab("Private Leaderboard"):
+#         current_date_time = datetime.now()
+#         if current_date_time >= config.competition_info.end_date:
+#             lb = utils.fetch_leaderboard(private=True)
+#             gr.Markdown(lb)
+#         else:
+#             gr.Markdown("Private Leaderboard will be available after the competition ends")
+#     with gr.Tab("New Submission"):
+#         gr.Markdown(SUBMISSION_TEXT)
+#         user_token = gr.Textbox(max_lines=1, value="hf_XXX", label="Please enter your Hugging Face token")
+#         uploaded_file = gr.File()
+#         output_text = gr.Markdown(visible=True, show_label=False)
+#         new_sub_button = gr.Button("Upload Submission")
+#         new_sub_button.click(fn=new_submission, inputs=[user_token, uploaded_file], outputs=[output_text])
+
+#     with gr.Tab("My Submissions"):
+#         gr.Markdown(SUBMISSION_LIMIT_TEXT)
+#         user_token = gr.Textbox(max_lines=1, value="hf_XXX", label="Please enter your Hugging Face token")
+#         output_text = gr.Markdown(visible=True, show_label=False)
+#         output_df = gr.Dataframe(visible=False)
+#         my_subs_button = gr.Button("Fetch Submissions")
+#         my_subs_button.click(fn=my_submissions, inputs=[user_token], outputs=[output_text, output_df])
+
+with gr.Blocks() as demo:
+    with gr.Tabs() as tab_container:
+        with gr.TabItem("Overview", id="overview"):
+            gr.Markdown(f"# Welcome to {competition_info.competition_name}! 👋")
+            gr.Markdown(f"{competition_info.competition_description}")
+            gr.Markdown("## Dataset")
+            gr.Markdown(f"{competition_info.dataset_description}")
+        with gr.TabItem("Public Leaderboard", id="public_leaderboard"):
+            pass
+        with gr.TabItem("Private Leaderboard", id="private_leaderboard"):
+            pass
+        with gr.TabItem("New Submission", id="new_submission"):
+            pass
+        with gr.TabItem("My Submissions", id="my_submissions"):
+            pass

competitions/errors.py

@@ -0,0 +1,14 @@
+class AuthenticationError(Exception):
+    pass
+
+
+class NoSubmissionError(Exception):
+    pass
+
+
+class SubmissionError(Exception):
+    pass
+
+
+class SubmissionLimitError(Exception):
+    pass

competitions/{config.py → info.py}

@@ -1,37 +1,29 @@
 import json
-import os
+from dataclasses import dataclass
 from datetime import datetime
-from pathlib import Path
 
-from dotenv import load_dotenv
 from huggingface_hub import hf_hub_download
 from huggingface_hub.utils._errors import EntryNotFoundError
+from loguru import logger
 
 
-if Path("../.env").is_file():
-    load_dotenv("../.env")
-
-
-MOONLANDING_URL = os.getenv("MOONLANDING_URL")
-COMPETITION_ID = os.getenv("COMPETITION_ID")
-AUTOTRAIN_USERNAME = os.getenv("AUTOTRAIN_USERNAME")
-AUTOTRAIN_TOKEN = os.getenv("AUTOTRAIN_TOKEN")
-AUTOTRAIN_BACKEND_API = os.getenv("AUTOTRAIN_BACKEND_API")
-
-
+@dataclass
 class CompetitionInfo:
-    def __init__(self):
+    competition_id: str
+    autotrain_token: str
+
+    def __post_init__(self):
         try:
             config_fname = hf_hub_download(
-                repo_id=COMPETITION_ID,
+                repo_id=self.competition_id,
                 filename="conf",
-                use_auth_token=AUTOTRAIN_TOKEN,
+                use_auth_token=self.autotrain_token,
                 repo_type="dataset",
             )
         except EntryNotFoundError:
             raise Exception("Competition config not found. Please check the competition id.")
         except Exception as e:
-            print(e)
+            logger.error(e)
             raise Exception("Hugging Face Hub is unreachable, please try again later.")
 
         self.config = self.load_config(config_fname)
@@ -78,6 +70,3 @@ class CompetitionInfo:
     @property
     def dataset_description(self):
         return self.config["DATASET_DESCRIPTION"]
-
-
-competition_info = CompetitionInfo()

competitions/leaderboard.py

@@ -0,0 +1,99 @@
+import glob
+import json
+import os
+from dataclasses import dataclass
+
+import pandas as pd
+from huggingface_hub import snapshot_download
+
+
+@dataclass
+class Leaderboard:
+    end_date: str
+    eval_higher_is_better: bool
+    competition_id: str
+    autotrain_token: str
+
+    def __post_init__(self):
+        self.private_columns = [
+            "rank",
+            "name",
+            "private_score",
+            "submission_datetime",
+        ]
+        self.public_columns = [
+            "rank",
+            "name",
+            "public_score",
+            "submission_datetime",
+        ]
+
+    def _download_submissions(self, private):
+        submissions_folder = snapshot_download(
+            repo_id=self.competition_id,
+            allow_patterns="*.json",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        submissions = []
+        for submission in glob.glob(os.path.join(submissions_folder, "*.json")):
+            with open(submission, "r") as f:
+                submission_info = json.load(f)
+            if self.eval_higher_is_better:
+                submission_info["submissions"].sort(
+                    key=lambda x: x["private_score"] if private else x["public_score"],
+                    reverse=True,
+                )
+            else:
+                submission_info["submissions"].sort(key=lambda x: x["private_score"] if private else x["public_score"])
+
+            # select only the best submission
+            submission_info["submissions"] = submission_info["submissions"][0]
+            temp_info = {
+                "id": submission_info["id"],
+                "name": submission_info["name"],
+                "submission_id": submission_info["submissions"]["submission_id"],
+                "submission_comment": submission_info["submissions"]["submission_comment"],
+                "status": submission_info["submissions"]["status"],
+                "selected": submission_info["submissions"]["selected"],
+                "public_score": submission_info["submissions"]["public_score"],
+                "private_score": submission_info["submissions"]["private_score"],
+                "submission_date": submission_info["submissions"]["date"],
+                "submission_time": submission_info["submissions"]["time"],
+            }
+            submissions.append(temp_info)
+        return submissions
+
+    def fetch(self, private=False):
+        submissions = self._download_submissions(private)
+
+        if len(submissions) == 0:
+            return pd.DataFrame()
+
+        df = pd.DataFrame(submissions)
+        # convert submission date and time to datetime
+        df["submission_datetime"] = pd.to_datetime(
+            df["submission_date"] + " " + df["submission_time"], format="%Y-%m-%d %H:%M:%S"
+        )
+        # convert datetime column to string
+        df["submission_datetime"] = df["submission_datetime"].dt.strftime("%Y-%m-%d %H:%M:%S")
+
+        # sort by submission datetime
+        # sort by public score and submission datetime
+        if self.eval_higher_is_better:
+            df = df.sort_values(
+                by=["public_score", "submission_datetime"],
+                ascending=[False, True],
+            )
+        else:
+            df = df.sort_values(
+                by=["public_score", "submission_datetime"],
+                ascending=[True, True],
+            )
+
+        # reset index
+        df = df.reset_index(drop=True)
+        df["rank"] = df.index + 1
+
+        columns = self.public_columns if not private else self.private_columns
+        return df[columns]

competitions/submissions.py

@@ -0,0 +1,310 @@
+import io
+import json
+import time
+import uuid
+from dataclasses import dataclass
+from datetime import datetime
+
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.utils._errors import EntryNotFoundError
+from loguru import logger
+
+from .errors import AuthenticationError, NoSubmissionError, SubmissionError, SubmissionLimitError
+from .utils import http_get, http_post, user_authentication
+
+
+@dataclass
+class Submissions:
+    competition_id: str
+    submission_limit: str
+    end_date: datetime
+    autotrain_username: str
+    autotrain_token: str
+    autotrain_backend_api: str
+
+    def __post_init__(self):
+        self.public_sub_columns = [
+            "date",
+            "submission_id",
+            "public_score",
+            "submission_comment",
+            "selected",
+        ]
+        self.private_sub_columns = [
+            "date",
+            "submission_id",
+            "public_score",
+            "private_score",
+            "submission_comment",
+            "selected",
+            "status",
+        ]
+
+    def _verify_submission(self, bytes_data):
+        return True
+
+    def _add_new_user(self, user_info):
+        api = HfApi()
+        user_submission_info = {}
+        user_submission_info["name"] = user_info["name"]
+        user_submission_info["id"] = user_info["id"]
+        user_submission_info["submissions"] = []
+        # convert user_submission_info to BufferedIOBase file object
+        user_submission_info_json = json.dumps(user_submission_info)
+        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+
+        api.upload_file(
+            path_or_fileobj=user_submission_info_json_buffer,
+            path_in_repo=f"{user_info['id']}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+            token=self.autotrain_token,
+        )
+
+    def _check_user_submission_limit(self, user_info):
+        user_id = user_info["id"]
+        try:
+            user_fname = hf_hub_download(
+                repo_id=self.competition_id,
+                filename=f"{user_id}.json",
+                use_auth_token=self.autotrain_token,
+                repo_type="dataset",
+            )
+        except EntryNotFoundError:
+            self._add_new_user(user_info)
+            user_fname = hf_hub_download(
+                repo_id=self.competition_id,
+                filename=f"{user_id}.json",
+                use_auth_token=self.autotrain_token,
+                repo_type="dataset",
+            )
+        except Exception as e:
+            logger.error(e)
+            raise Exception("Hugging Face Hub is unreachable, please try again later.")
+
+        with open(user_fname, "r") as f:
+            user_submission_info = json.load(f)
+
+        todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
+        if len(user_submission_info["submissions"]) == 0:
+            user_submission_info["submissions"] = []
+
+        # count the number of times user has submitted today
+        todays_submissions = 0
+        for sub in user_submission_info["submissions"]:
+            if sub["date"] == todays_date:
+                todays_submissions += 1
+        if todays_submissions >= self.submission_limit:
+            return False
+        return True
+
+    def _increment_submissions(self, user_id, submission_id, submission_comment):
+        user_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"{user_id}.json",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        with open(user_fname, "r") as f:
+            user_submission_info = json.load(f)
+        todays_date = datetime.now().strftime("%Y-%m-%d")
+        current_time = datetime.now().strftime("%H:%M:%S")
+
+        # here goes all the default stuff for submission
+        user_submission_info["submissions"].append(
+            {
+                "date": todays_date,
+                "time": current_time,
+                "submission_id": submission_id,
+                "submission_comment": submission_comment,
+                "status": "pending",
+                "selected": False,
+                "public_score": -1,
+                "private_score": -1,
+            }
+        )
+        # count the number of times user has submitted today
+        todays_submissions = 0
+        for sub in user_submission_info["submissions"]:
+            if sub["date"] == todays_date:
+                todays_submissions += 1
+
+        # convert user_submission_info to BufferedIOBase file object
+        user_submission_info_json = json.dumps(user_submission_info)
+        user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
+        user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
+        api = HfApi()
+        api.upload_file(
+            path_or_fileobj=user_submission_info_json_buffer,
+            path_in_repo=f"{user_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+            token=self.autotrain_token,
+        )
+        return todays_submissions
+
+    def _download_user_subs(self, user_id):
+        user_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"{user_id}.json",
+            use_auth_token=self.autotrain_token,
+            repo_type="dataset",
+        )
+        with open(user_fname, "r") as f:
+            user_submission_info = json.load(f)
+        return user_submission_info["submissions"]
+
+    def _get_user_subs(self, user_info, private=False):
+        # get user submissions
+        user_id = user_info["id"]
+        try:
+            user_submissions = self._download_user_subs(user_id)
+        except EntryNotFoundError:
+            return NoSubmissionError("No submissions found ")
+
+        submissions_df = pd.DataFrame(user_submissions)
+        if not private:
+            submissions_df = submissions_df.drop(columns=["private_score"])
+            submissions_df = submissions_df[self.public_sub_columns]
+        else:
+            submissions_df = submissions_df[self.private_sub_columns]
+        return submissions_df
+
+    def _get_user_info(self, user_token):
+        user_info = user_authentication(token=user_token)
+        if "error" in user_info:
+            return AuthenticationError("Invalid token")
+
+        if user_info["emailVerified"] is False:
+            return AuthenticationError("Please verify your email on Hugging Face Hub")
+        return user_info
+
+    def _create_autotrain_project(self, project_id, submission_dataset, model, dataset):
+        project_config = {}
+        project_config["dataset_name"] = "lewtun/imdb-dummy"
+        project_config["dataset_config"] = "lewtun--imdb-dummy"
+        project_config["dataset_split"] = "train"
+        project_config["col_mapping"] = {"text": "text", "label": "target"}
+
+        payload = {
+            "username": self.autotrain_username,
+            "proj_name": project_id,
+            "task": 1,
+            "config": {
+                "language": "en",
+                "max_models": 5,
+                "benchmark": {
+                    "dataset": dataset,
+                    "model": model,
+                    "submission_dataset": submission_dataset,
+                    "create_prediction_repo": False,
+                },
+            },
+        }
+
+        project_json_resp = http_post(
+            path="/projects/create",
+            payload=payload,
+            token=self.autotrain_token,
+            domain=self.autotrain_backend_api,
+        ).json()
+
+        time.sleep(5)
+        # Upload data
+        payload = {
+            "split": 4,
+            "col_mapping": project_config["col_mapping"],
+            "load_config": {"max_size_bytes": 0, "shuffle": False},
+            "dataset_id": project_config["dataset_name"],
+            "dataset_config": project_config["dataset_config"],
+            "dataset_split": project_config["dataset_split"],
+        }
+
+        _ = http_post(
+            path=f"/projects/{project_json_resp['id']}/data/dataset",
+            payload=payload,
+            token=self.autotrain_token,
+            domain=self.autotrain_backend_api,
+        ).json()
+        logger.info("💾💾💾 Dataset creation done 💾💾💾")
+
+        # Process data
+        _ = http_post(
+            path=f"/projects/{project_json_resp['id']}/data/start_processing",
+            token=self.autotrain_token,
+            domain=self.autotrain_backend_api,
+        ).json()
+
+        logger.info("⏳ Waiting for data processing to complete ...")
+        is_data_processing_success = False
+        while is_data_processing_success is not True:
+            project_status = http_get(
+                path=f"/projects/{project_json_resp['id']}",
+                token=self.autotrain_token,
+                domain=self.autotrain_backend_api,
+            ).json()
+            # See database.database.enums.ProjectStatus for definitions of `status`
+            if project_status["status"] == 3:
+                is_data_processing_success = True
+                logger.info("✅ Data processing complete!")
+            time.sleep(5)
+
+        # Approve training job
+        _ = http_post(
+            path=f"/projects/{project_json_resp['id']}/start_training",
+            token=self.autotrain_token,
+            domain=self.autotrain_backend_api,
+        ).json()
+
+    def my_submissions(self, user_token):
+        user_info = self._get_user_info(user_token)
+        current_date_time = datetime.now()
+        private = False
+        if current_date_time >= self.end_date:
+            private = True
+        subs = self._get_user_subs(user_info, private=private)
+        return subs
+
+    def new_submission(self, user_token, uploaded_file):
+        # verify token
+        user_info = self._get_user_info(user_token)
+
+        # check if user can submit to the competition
+        if self._check_user_submission_limit(user_info) is False:
+            return SubmissionLimitError("Submission limit reached")
+
+        with open(uploaded_file.name, "rb") as f:
+            bytes_data = f.read()
+        # verify file is valid
+        if not self._verify_submission(bytes_data):
+            return SubmissionError("Invalid submission file")
+        else:
+            user_id = user_info["id"]
+            submission_id = str(uuid.uuid4())
+            file_extension = uploaded_file.orig_name.split(".")[-1]
+            # upload file to hf hub
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=bytes_data,
+                path_in_repo=f"submissions/{user_id}-{submission_id}.{file_extension}",
+                repo_id=self.competition_id,
+                repo_type="dataset",
+                token=self.autotrain_token,
+            )
+            # update submission limit
+            submissions_made = self._increment_submissions(
+                user_id=user_id,
+                submission_id=submission_id,
+                submission_comment="",
+            )
+            # schedule submission for evaluation
+            self._create_autotrain_project(
+                project_id=f"{submission_id}",
+                dataset=f"{self.competition_id}",
+                submission_dataset=user_id,
+                model="generic_competition",
+            )
+        remaining_submissions = self.submission_limit - submissions_made
+        return remaining_submissions

competitions/text.py

@@ -0,0 +1,11 @@
+SUBMISSION_TEXT = """You can make upto {} submissions per day.
+The test data has been divided into public and private splits.
+Your score on the public split will be shown on the leaderboard.
+Your final score will be based on your private split performance.
+The final rankings will be based on the private split performance.
+"""
+
+SUBMISSION_ERROR = """Submission is not in a proper format.
+Please check evaluation instructions for more details."""
+
+SUBMISSION_LIMIT_TEXT = """You can select upto {} submissions for private leaderboard."""

competitions/utils.py

@@ -1,15 +1,7 @@
-import datetime
-import glob
-import io
-import json
-import os
-import time
-
-import config
-import pandas as pd
 import requests
-from huggingface_hub import HfApi, hf_hub_download, snapshot_download
-from huggingface_hub.utils._errors import EntryNotFoundError
+from loguru import logger
+
+from . import MOONLANDING_URL
 
 
 def get_auth_headers(token: str, prefix: str = "Bearer"):
@@ -23,7 +15,7 @@ def http_post(path: str, token: str, payload=None, domain: str = None, params=No
             url=domain + path, json=payload, headers=get_auth_headers(token=token), allow_redirects=True, params=params
         )
     except requests.exceptions.ConnectionError:
-        print("❌ Failed to reach AutoNLP API, check your internet connection")
+        logger.error("❌ Failed to reach AutoNLP API, check your internet connection")
     response.raise_for_status()
     return response
 
@@ -33,85 +25,11 @@ def http_get(path: str, token: str, domain: str = None) -> requests.Response:
     try:
         response = requests.get(url=domain + path, headers=get_auth_headers(token=token), allow_redirects=True)
     except requests.exceptions.ConnectionError:
-        print("❌ Failed to reach AutoNLP API, check your internet connection")
+        logger.error("❌ Failed to reach AutoNLP API, check your internet connection")
     response.raise_for_status()
     return response
 
 
-def create_project(project_id, submission_dataset, model, dataset):
-    project_config = {}
-    project_config["dataset_name"] = "lewtun/imdb-dummy"
-    project_config["dataset_config"] = "lewtun--imdb-dummy"
-    project_config["dataset_split"] = "train"
-    project_config["col_mapping"] = {"text": "text", "label": "target"}
-
-    payload = {
-        "username": config.AUTOTRAIN_USERNAME,
-        "proj_name": project_id,
-        "task": 1,
-        "config": {
-            "language": "en",
-            "max_models": 5,
-            "benchmark": {
-                "dataset": dataset,
-                "model": model,
-                "submission_dataset": submission_dataset,
-                "create_prediction_repo": False,
-            },
-        },
-    }
-
-    project_json_resp = http_post(
-        path="/projects/create", payload=payload, token=config.AUTOTRAIN_TOKEN, domain=config.AUTOTRAIN_BACKEND_API
-    ).json()
-    time.sleep(5)
-    # Upload data
-    payload = {
-        "split": 4,
-        "col_mapping": project_config["col_mapping"],
-        "load_config": {"max_size_bytes": 0, "shuffle": False},
-        "dataset_id": project_config["dataset_name"],
-        "dataset_config": project_config["dataset_config"],
-        "dataset_split": project_config["dataset_split"],
-    }
-
-    _ = http_post(
-        path=f"/projects/{project_json_resp['id']}/data/dataset",
-        payload=payload,
-        token=config.AUTOTRAIN_TOKEN,
-        domain=config.AUTOTRAIN_BACKEND_API,
-    ).json()
-    print("💾💾💾 Dataset creation done 💾💾💾")
-
-    # Process data
-    _ = http_post(
-        path=f"/projects/{project_json_resp['id']}/data/start_processing",
-        token=config.AUTOTRAIN_TOKEN,
-        domain=config.AUTOTRAIN_BACKEND_API,
-    ).json()
-
-    print("⏳ Waiting for data processing to complete ...")
-    is_data_processing_success = False
-    while is_data_processing_success is not True:
-        project_status = http_get(
-            path=f"/projects/{project_json_resp['id']}",
-            token=config.AUTOTRAIN_TOKEN,
-            domain=config.AUTOTRAIN_BACKEND_API,
-        ).json()
-        # See database.database.enums.ProjectStatus for definitions of `status`
-        if project_status["status"] == 3:
-            is_data_processing_success = True
-            print("✅ Data processing complete!")
-        time.sleep(10)
-
-    # Approve training job
-    _ = http_post(
-        path=f"/projects/{project_json_resp['id']}/start_training",
-        token=config.AUTOTRAIN_TOKEN,
-        domain=config.AUTOTRAIN_BACKEND_API,
-    ).json()
-
-
 def user_authentication(token):
     headers = {}
     cookies = {}
@@ -121,185 +39,12 @@ def user_authentication(token):
         cookies = {"token": token}
     try:
         response = requests.get(
-            config.MOONLANDING_URL + "/api/whoami-v2",
+            MOONLANDING_URL + "/api/whoami-v2",
             headers=headers,
             cookies=cookies,
             timeout=3,
         )
     except (requests.Timeout, ConnectionError) as err:
-        print(f"Failed to request whoami-v2 - {repr(err)}")
+        logger.error(f"Failed to request whoami-v2 - {repr(err)}")
         raise Exception("Hugging Face Hub is unreachable, please try again later.")
     return response.json()
-
-
-def add_new_user(user_info):
-    api = HfApi()
-    user_submission_info = {}
-    user_submission_info["name"] = user_info["name"]
-    user_submission_info["id"] = user_info["id"]
-    user_submission_info["submissions"] = []
-    # convert user_submission_info to BufferedIOBase file object
-    user_submission_info_json = json.dumps(user_submission_info)
-    user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
-    user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
-
-    api.upload_file(
-        path_or_fileobj=user_submission_info_json_buffer,
-        path_in_repo=f"{user_info['id']}.json",
-        repo_id=config.COMPETITION_ID,
-        repo_type="dataset",
-        token=config.AUTOTRAIN_TOKEN,
-    )
-
-
-def check_user_submission_limit(user_info):
-    user_id = user_info["id"]
-    try:
-        user_fname = hf_hub_download(
-            repo_id=config.COMPETITION_ID,
-            filename=f"{user_id}.json",
-            use_auth_token=config.AUTOTRAIN_TOKEN,
-            repo_type="dataset",
-        )
-    except EntryNotFoundError:
-        add_new_user(user_info)
-        user_fname = hf_hub_download(
-            repo_id=config.COMPETITION_ID,
-            filename=f"{user_id}.json",
-            use_auth_token=config.AUTOTRAIN_TOKEN,
-            repo_type="dataset",
-        )
-    except Exception as e:
-        print(e)
-        raise Exception("Hugging Face Hub is unreachable, please try again later.")
-
-    with open(user_fname, "r") as f:
-        user_submission_info = json.load(f)
-
-    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    if len(user_submission_info["submissions"]) == 0:
-        user_submission_info["submissions"] = []
-
-    # count the number of times user has submitted today
-    todays_submissions = 0
-    for sub in user_submission_info["submissions"]:
-        if sub["date"] == todays_date:
-            todays_submissions += 1
-    if todays_submissions >= config.competition_info.submission_limit:
-        return False
-    return True
-
-
-def increment_submissions(user_id, submission_id, submission_comment):
-    user_fname = hf_hub_download(
-        repo_id=config.COMPETITION_ID,
-        filename=f"{user_id}.json",
-        use_auth_token=config.AUTOTRAIN_TOKEN,
-        repo_type="dataset",
-    )
-    with open(user_fname, "r") as f:
-        user_submission_info = json.load(f)
-    todays_date = datetime.datetime.now().strftime("%Y-%m-%d")
-    current_time = datetime.datetime.now().strftime("%H:%M:%S")
-    # here goes all the default stuff for submission
-    user_submission_info["submissions"].append(
-        {
-            "date": todays_date,
-            "time": current_time,
-            "submission_id": submission_id,
-            "submission_comment": submission_comment,
-            "status": "pending",
-            "selected": False,
-            "public_score": -1,
-            "private_score": -1,
-        }
-    )
-    # count the number of times user has submitted today
-    todays_submissions = 0
-    for sub in user_submission_info["submissions"]:
-        if sub["date"] == todays_date:
-            todays_submissions += 1
-    # convert user_submission_info to BufferedIOBase file object
-    user_submission_info_json = json.dumps(user_submission_info)
-    user_submission_info_json_bytes = user_submission_info_json.encode("utf-8")
-    user_submission_info_json_buffer = io.BytesIO(user_submission_info_json_bytes)
-    api = HfApi()
-    api.upload_file(
-        path_or_fileobj=user_submission_info_json_buffer,
-        path_in_repo=f"{user_id}.json",
-        repo_id=config.COMPETITION_ID,
-        repo_type="dataset",
-        token=config.AUTOTRAIN_TOKEN,
-    )
-    return todays_submissions
-
-
-def verify_submission(bytes_data):
-    return True
-
-
-def fetch_submissions(user_id):
-    user_fname = hf_hub_download(
-        repo_id=config.COMPETITION_ID,
-        filename=f"{user_id}.json",
-        use_auth_token=config.AUTOTRAIN_TOKEN,
-        repo_type="dataset",
-    )
-    with open(user_fname, "r") as f:
-        user_submission_info = json.load(f)
-    return user_submission_info["submissions"]
-
-
-def fetch_leaderboard(private=False):
-    submissions_folder = snapshot_download(
-        repo_id=config.COMPETITION_ID,
-        allow_patterns="*.json",
-        use_auth_token=config.AUTOTRAIN_TOKEN,
-        repo_type="dataset",
-    )
-    submissions = []
-    for submission in glob.glob(os.path.join(submissions_folder, "*.json")):
-        with open(submission, "r") as f:
-            submission_info = json.load(f)
-        if config.competition_info.eval_higher_is_better:
-            submission_info["submissions"].sort(
-                key=lambda x: x["private_score"] if private else x["public_score"], reverse=True
-            )
-        else:
-            submission_info["submissions"].sort(key=lambda x: x["private_score"] if private else x["public_score"])
-        # select only the best submission
-        submission_info["submissions"] = submission_info["submissions"][0]
-        temp_info = {
-            "id": submission_info["id"],
-            "name": submission_info["name"],
-            "submission_id": submission_info["submissions"]["submission_id"],
-            "submission_comment": submission_info["submissions"]["submission_comment"],
-            "status": submission_info["submissions"]["status"],
-            "selected": submission_info["submissions"]["selected"],
-            "public_score": submission_info["submissions"]["public_score"],
-            "private_score": submission_info["submissions"]["private_score"],
-            "submission_date": submission_info["submissions"]["date"],
-            "submission_time": submission_info["submissions"]["time"],
-        }
-        submissions.append(temp_info)
-
-    df = pd.DataFrame(submissions)
-    # convert submission date and time to datetime
-    df["submission_datetime"] = pd.to_datetime(
-        df["submission_date"] + " " + df["submission_time"], format="%Y-%m-%d %H:%M:%S"
-    )
-    # sort by submission datetime
-    # sort by public score and submission datetime
-    if config.competition_info.eval_higher_is_better:
-        df = df.sort_values(by=["public_score", "submission_datetime"], ascending=[False, True])
-    else:
-        df = df.sort_values(by=["public_score", "submission_datetime"], ascending=[True, True])
-    # reset index
-    df = df.reset_index(drop=True)
-    df["rank"] = df.index + 1
-
-    if private:
-        columns = ["rank", "name", "private_score", "submission_datetime"]
-    else:
-        columns = ["rank", "name", "public_score", "submission_datetime"]
-    return df[columns].to_markdown()

requirements.txt

@@ -1,5 +1,5 @@
 gradio==3.9.1
+loguru==0.6.0
 pandas==1.5.0
-python-dotenv==0.20.0
 huggingface_hub==0.10.1
 tabulate==0.9.0

setup.py

@@ -44,6 +44,7 @@ setup(
     url="https://github.com/huggingface/competitions",
     download_url="https://github.com/huggingface/competitions/tags",
     packages=find_packages("."),
+    entry_points={"console_scripts": ["competitions=competitions.cli.competitions:main"]},
     install_requires=INSTALL_REQUIRES,
     extras_require=EXTRAS_REQUIRE,
     python_requires=">=3.8",