add socket kit

Dockerfile

@@ -3,7 +3,14 @@ FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive \
     TZ=UTC
 
-RUN pip install pip==23.3.2
+RUN apt-get update &&  \
+    apt-get upgrade -y &&  \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    wget \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
 
 WORKDIR /app
 RUN mkdir -p /app/.cache
@@ -29,4 +36,8 @@ RUN conda create -p /app/env -y python=3.10 \
 SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
 
 COPY --chown=1000:1000 . /app/
+RUN make socket-kit.so
+
+ENV PATH="/app:${PATH}"
+
 RUN pip install -e .

Makefile

@@ -1,3 +1,4 @@
+CFLAGS += -std=c99 -Wall
 .PHONY: quality style
 
 quality:
@@ -7,4 +8,16 @@ quality:
 
 style:
 	python -m black --line-length 119 --target-version py38 .
-	python -m isort .
+	python -m isort .
+
+docker:
+	docker build -t competitions:latest .
+	docker tag competitions:latest huggingface/competitions:latest
+	docker push huggingface/competitions:latest
+
+
+socket-kit.so: socket-kit.c
+	gcc $(CFLAGS) -shared -fPIC $^ -o $@ -ldl
+
+clean:
+	rm *.so

competitions/evaluate.py

@@ -1,5 +1,7 @@
 import argparse
 import json
+import os
+import shutil
 import subprocess
 
 from huggingface_hub import HfApi, snapshot_download
@@ -22,12 +24,11 @@ def upload_submission_file(params, file_path):
 
 
 def generate_submission_file(params):
-    base_user = params.competition_id.split("/")[0]
     logger.info("Downloading submission dataset")
     submission_dir = snapshot_download(
-        repo_id=f"{base_user}/{params.submission_id}",
+        repo_id=params.submission_repo,
         local_dir=params.output_path,
-        token=params.token,
+        token=os.environ.get("USER_TOKEN"),
         repo_type="model",
     )
     # submission_dir has a script.py file
@@ -35,7 +36,13 @@ def generate_submission_file(params):
     # the script.py will generate a submission.csv file in the submission_dir
     # push the submission.csv file to the repo using upload_submission_file
     logger.info("Generating submission file")
-    subprocess.run(["python", "script.py"], cwd=submission_dir)
+    # copy socket-kit.so to submission_dir
+    shutil.copyfile("socket-kit.so", f"{submission_dir}/socket-kit.so")
+    cmd = "python script.py"
+    socket_kit_path = os.path.abspath(f"{submission_dir}/socket-kit.so")
+    env = os.environ.copy()
+    env["LD_PRELOAD"] = socket_kit_path
+    subprocess.run(cmd, cwd=submission_dir, shell=True, check=True, env=env)
 
     api = HfApi(token=params.token)
     api.upload_file(
@@ -48,6 +55,8 @@ def generate_submission_file(params):
 
 @utils.monitor
 def run(params):
+    logger.info(params)
+    logger.info(f"User token: {os.environ.get('USER_TOKEN')}")
     if isinstance(params, dict):
         params = EvalParams(**params)
 

competitions/runner.py

@@ -2,13 +2,11 @@ import glob
 import io
 import json
 import os
-import random
-import string
 import time
 from dataclasses import dataclass
 
 import pandas as pd
-from huggingface_hub import HfApi, snapshot_download
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
 from loguru import logger
 
 from competitions.info import CompetitionInfo
@@ -18,7 +16,7 @@ from competitions.utils import run_evaluation
 _DOCKERFILE = """
 FROM huggingface/competitions:latest
 
-CMD uvicorn competitions.app:app --port 7860 --host 0.0.0.0
+CMD uvicorn competitions.api:api --port 7860 --host 0.0.0.0
 """
 
 # format _DOCKERFILE
@@ -59,6 +57,8 @@ class JobRunner:
                             "submission_id": sub["submission_id"],
                             "datetime": sub["datetime"],
                             "submission_repo": sub["submission_repo"],
+                            "space_id": sub["space_id"],
+                            "space_status": sub["space_status"],
                         }
                     )
         if len(pending_submissions) == 0:
@@ -105,23 +105,8 @@ class JobRunner:
         _readme = io.BytesIO(_readme.encode())
         return _readme
 
-    def create_space(self, team_id, submission_id, submission_repo):
-        project_name = "".join(
-            random.choices(
-                string.ascii_lowercase + string.digits,
-                k=10,
-            )
-        )
+    def create_space(self, team_id, submission_id, submission_repo, space_id):
         api = HfApi(token=self.token)
-        username = self.competition_id.split("/")[0]
-        repo_id = f"{username}/competitions-{project_name}"
-        api.create_repo(
-            repo_id=repo_id,
-            repo_type="space",
-            space_sdk="docker",
-            space_hardware="cpu-basic",
-            private=True,
-        )
         params = {
             "competition_id": self.competition_id,
             "competition_type": self.competition_type,
@@ -136,13 +121,13 @@ class JobRunner:
             "submission_repo": submission_repo,
         }
 
-        api.add_space_secret(repo_id=repo_id, key="PARAMS", value=json.dumps(params))
+        api.add_space_secret(repo_id=space_id, key="PARAMS", value=json.dumps(params))
 
-        readme = self._create_readme(project_name)
+        readme = self._create_readme(space_id.split("/")[-1])
         api.upload_file(
             path_or_fileobj=readme,
             path_in_repo="README.md",
-            repo_id=repo_id,
+            repo_id=space_id,
             repo_type="space",
         )
 
@@ -150,10 +135,35 @@ class JobRunner:
         api.upload_file(
             path_or_fileobj=_dockerfile,
             path_in_repo="Dockerfile",
-            repo_id=repo_id,
+            repo_id=space_id,
             repo_type="space",
         )
-        return repo_id
+
+        # update space_status in submission_info
+        team_fname = hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"submission_info/{team_id}.json",
+            token=self.token,
+            repo_type="dataset",
+        )
+        with open(team_fname, "r", encoding="utf-8") as f:
+            team_submission_info = json.load(f)
+
+        for submission in team_submission_info["submissions"]:
+            if submission["submission_id"] == submission_id:
+                submission["space_status"] = 1
+                break
+
+        team_submission_info_json = json.dumps(team_submission_info, indent=4)
+        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
+        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+        api = HfApi(token=self.token)
+        api.upload_file(
+            path_or_fileobj=team_submission_info_json_buffer,
+            path_in_repo=f"submission_info/{team_id}.json",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
 
     def run(self):
         while True:
@@ -168,5 +178,8 @@ class JobRunner:
                     team_id = row["team_id"]
                     submission_id = row["submission_id"]
                     submission_repo = row["submission_repo"]
-                    self.create_space(team_id, submission_id, submission_repo)
+                    space_id = row["space_id"]
+                    space_status = row["space_status"]
+                    if space_status == 0:
+                        self.create_space(team_id, submission_id, submission_repo, space_id)
             time.sleep(5)

competitions/submissions.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from datetime import datetime
 
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.utils._errors import EntryNotFoundError
 from loguru import logger
 
@@ -133,9 +133,20 @@ class Submissions:
                 todays_submissions += 1
         return todays_submissions
 
-    def _increment_submissions(self, team_id, user_id, submission_id, submission_comment, submission_repo=None):
+    def _increment_submissions(
+        self,
+        team_id,
+        user_id,
+        submission_id,
+        submission_comment,
+        submission_repo=None,
+        space_id=None,
+        space_status=0,
+    ):
         if submission_repo is None:
             submission_repo = ""
+        if space_id is None:
+            space_id = ""
         team_fname = hf_hub_download(
             repo_id=self.competition_id,
             filename=f"submission_info/{team_id}.json",
@@ -153,11 +164,13 @@ class Submissions:
                 "submission_id": submission_id,
                 "submission_comment": submission_comment,
                 "submission_repo": submission_repo,
+                "space_id": space_id,
                 "submitted_by": user_id,
                 "status": "pending",
                 "selected": False,
                 "public_score": -1,
                 "private_score": -1,
+                "space_status": space_status,
             }
         )
         # count the number of times user has submitted today
@@ -227,9 +240,7 @@ class Submissions:
             repo_type="dataset",
         )
 
-    def _get_team_subs(self, user_info, private=False):
-        user_id = user_info["id"]
-        team_id = self._get_team_id(user_id)
+    def _get_team_subs(self, team_id, private=False):
         try:
             team_submissions = self._download_team_subs(team_id)
         except EntryNotFoundError:
@@ -319,7 +330,8 @@ class Submissions:
         private = False
         if current_date_time >= self.end_date:
             private = True
-        success_subs, failed_subs = self._get_team_subs(user_info, private=private)
+        team_id = self._get_team_id(user_info["id"])
+        success_subs, failed_subs = self._get_team_subs(team_id, private=private)
         return success_subs, failed_subs
 
     def _get_team_id(self, user_id):
@@ -381,33 +393,48 @@ class Submissions:
                 user_id=user_id,
                 submission_id=submission_id,
                 submission_comment=submission_comment,
-                submission_repo="",
             )
         else:
-            submission_repo = snapshot_download(
-                repo_id=uploaded_file,
-                local_dir=submission_id,
-                token=user_token,
-                repo_type="model",
-            )
+            # Download the submission repo and upload it to the competition repo
+            # submission_repo = snapshot_download(
+            #     repo_id=uploaded_file,
+            #     local_dir=submission_id,
+            #     token=user_token,
+            #     repo_type="model",
+            # )
+            # api = HfApi(token=self.token)
+            # competition_user = self.competition_id.split("/")[0]
+            # api.create_repo(
+            #     repo_id=f"{competition_user}/{submission_id}",
+            #     repo_type="model",
+            #     private=True,
+            # )
+            # api.upload_folder(
+            #     folder_path=submission_repo,
+            #     repo_id=f"{competition_user}/{submission_id}",
+            #     repo_type="model",
+            # )
+            # create barebones submission runner space
+            competition_organizer = self.competition_id.split("/")[0]
+            space_id = f"{competition_organizer}/comp-{submission_id}"
             api = HfApi(token=self.token)
-            competition_user = self.competition_id.split("/")[0]
             api.create_repo(
-                repo_id=f"{competition_user}/{submission_id}",
-                repo_type="model",
+                repo_id=space_id,
+                repo_type="space",
+                space_sdk="docker",
+                space_hardware="cpu-basic",
                 private=True,
             )
-            api.upload_folder(
-                folder_path=submission_repo,
-                repo_id=f"{competition_user}/{submission_id}",
-                repo_type="model",
-            )
+
+            api.add_space_secret(repo_id=space_id, key="USER_TOKEN", value=user_token)
             submissions_made = self._increment_submissions(
                 team_id=team_id,
                 user_id=user_id,
                 submission_id=submission_id,
                 submission_comment=submission_comment,
                 submission_repo=uploaded_file,
+                space_id=space_id,
+                space_status=0,
             )
         remaining_submissions = self.submission_limit - submissions_made
         return remaining_submissions

requirements.txt

@@ -4,4 +4,7 @@ loguru==0.7.2
 pandas==2.1.4
 huggingface_hub==0.20.1
 tabulate==0.9.0
-markdown==3.5.1
+markdown==3.5.1
+psutil==5.9.0
+scikit-learn==1.3.2
+transformers

socket-kit.c

@@ -0,0 +1,8 @@
+#include <errno.h>
+#include <sys/socket.h>
+
+int connect(int fd, const struct sockaddr *addr, socklen_t len)
+{
+    errno = ENETDOWN;
+    return -1;
+}