feat: update log feature

.gitignore

@@ -147,4 +147,7 @@ dmypy.json
 **.tfvars
 
 # Alembic / database artifcats
-**.db
+**.db
+
+# lock
+submission_lock/*

Dockerfile

@@ -1,12 +1,9 @@
-FROM nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04
+FROM python:3.11.13-bullseye
 
 ENV DEBIAN_FRONTEND=noninteractive \
     TZ=UTC \
     HF_HUB_ENABLE_HF_TRANSFER=1
 
-ENV PATH="${HOME}/miniconda3/bin:${PATH}"
-ARG PATH="${HOME}/miniconda3/bin:${PATH}"
-
 RUN mkdir -p /tmp/model && \
     chown -R 1000:1000 /tmp/model && \
     mkdir -p /tmp/data && \
@@ -30,7 +27,6 @@ RUN apt-get update &&  \
     git-lfs \
     libgl1 \
     unzip \
-    openjdk-11-jre-headless \
     libseccomp-dev \
     && rm -rf /var/lib/apt/lists/* && \
     apt-get clean
@@ -48,34 +44,10 @@ ENV HOME=/app
 
 ENV PYTHONPATH=$HOME/app \
     PYTHONUNBUFFERED=1 \
-    GRADIO_ALLOW_FLAGGING=never \
-    GRADIO_NUM_PORTS=1 \
-    GRADIO_SERVER_NAME=0.0.0.0 \
-    GRADIO_THEME=huggingface \
     SYSTEM=spaces
 
-
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-    && sh Miniconda3-latest-Linux-x86_64.sh -b -p /app/miniconda \
-    && rm -f Miniconda3-latest-Linux-x86_64.sh
-ENV PATH /app/miniconda/bin:$PATH
-
-RUN conda create -p /app/env -y python=3.10
-
-SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
-
-RUN conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia && \
-    conda clean -ya && \
-    conda install -c "nvidia/label/cuda-12.1.0" cuda-nvcc && conda clean -ya
-
 COPY --chown=1000:1000 . /app/
-RUN make sandbox
-
-# give permissions to run sandbox
-RUN chmod +x /app/sandbox
 
-ENV PATH="/app:${PATH}"
+RUN pip install -r requirements.txt
 
-RUN pip install -U pip
-RUN pip install -e .
-RUN pip install -r requirements_docker.txt
+CMD ["python", "-m", "uvicorn", "competitions.app:app", "--workers", "1", "--port", "7860", "--host", "0.0.0.0"]

competitions/app.py

@@ -4,9 +4,10 @@ import threading
 import time
 import io
 from typing import Dict, Any
+from urllib.parse import urlencode
 
-from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, UploadFile, Body
-from fastapi.responses import HTMLResponse, JSONResponse, FileResponse
+from fastapi import Depends, FastAPI, Form, HTTPException, Request, UploadFile, Body, Query
+from fastapi.responses import HTMLResponse, JSONResponse, FileResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from huggingface_hub import hf_hub_download
@@ -25,8 +26,8 @@ from competitions.oauth import attach_oauth
 from competitions.runner import JobRunner
 from competitions.submissions import Submissions
 from competitions.text import SUBMISSION_SELECTION_TEXT, SUBMISSION_TEXT
-from competitions.utils import team_file_api, submission_api, leaderboard_api
-from competitions.enums import SubmissionStatus
+from competitions.utils import team_file_api, submission_api, leaderboard_api, error_log_api
+from competitions.enums import SubmissionStatus, ErrorMessage
 
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
@@ -137,7 +138,9 @@ def use_oauth(request: Request, user_token: str = Depends(utils.user_authenticat
     if user_token:
         result["is_login"] = True
         result["is_admin"] = utils.is_user_admin(user_token, comp_org)
-        result["is_registered"] = utils.team_file_api.get_team_info(user_token) is not None
+        team_info = utils.team_file_api.get_team_info(user_token)
+        result["is_registered"] = team_info is not None
+        result["is_white_team"] = team_info and team_info["id"] in team_file_api.get_team_white_list()
     return {"response": result}
 
 
@@ -185,7 +188,9 @@ def get_rules(request: Request):
 
 
 @app.get("/submission_info", response_class=JSONResponse)
-def get_submission_info(request: Request):
+def get_submission_info(request: Request, user_token: str = Depends(utils.user_authentication)):
+    if utils.team_file_api.get_team_info(user_token) is None:
+        return {"response": "Please register your team first."}
     competition_info = CompetitionInfo(competition_id=COMPETITION_ID, autotrain_token=HF_TOKEN)
     info = competition_info.submission_desc
     resp = {"response": info}
@@ -213,6 +218,10 @@ def my_submissions(request: Request, user_token: str = Depends(utils.user_authen
                 "team_name": "",
             }
         }
+    team_info = utils.team_file_api.get_team_info(user_token)
+    if team_info["id"] not in team_file_api.get_team_white_list():
+        raise HTTPException(status_code=403, detail="You are not allowed to access this resource.")
+
     sub = Submissions(
         end_date=competition_info.end_date,
         submission_limit=competition_info.submission_limit,
@@ -240,7 +249,13 @@ def my_submissions(request: Request, user_token: str = Depends(utils.user_authen
     submission_text = SUBMISSION_TEXT.format(competition_info.submission_limit)
     submission_selection_text = SUBMISSION_SELECTION_TEXT.format(competition_info.selection_limit)
 
-    team_name = utils.team_file_api.get_team_info(user_token)["name"]
+    team_name = team_info["name"]
+
+    for sub in subs:
+        if sub["error_message"] in {ErrorMessage.RUNTIME_ERROR, ErrorMessage.BUILD_SPACE_FAILED}:
+            sub["log_file_url"] = "/error_log?" + urlencode({"token": error_log_api.generate_log_token(sub["submission_id"])})
+        else:
+            sub["log_file_url"] = ""
 
     resp = {
         "response": {
@@ -273,6 +288,8 @@ def new_submission(
             return {"response": "Competition has not started yet!"}
 
     team_id = team_file_api.get_team_info(user_token)["id"]
+    if team_id not in team_file_api.get_team_white_list():
+        return {"response": "You are not allowed to make submissions."}
 
     lock = FileLock(f"./submission_lock/{team_id}.lock", blocking=False)
     try:
@@ -348,7 +365,7 @@ def update_team_name(
         return {"success": False, "error": "Team name cannot be empty."}
 
     try:
-        utils.team_file_api.update_and_create_team_name(user_token, new_team_name)
+        utils.team_file_api.update_team_name(user_token, new_team_name)
         return {"success": True, "error": ""}
     except Exception as e:
         return {"success": False, "error": str(e)}
@@ -464,3 +481,19 @@ def register(
     )
 
     return {"success": True, "response": "Team created successfully."}
+
+
+@app.get("/error_log")
+def get_error_log(token: str = Query(..., description="Token to access the error log file.")):
+    try:
+        log_content = error_log_api.get_log_by_token(token)
+        log_file = io.BytesIO(log_content.encode('utf-8'))
+        log_file.seek(0)
+        return StreamingResponse(
+            log_file,
+            media_type="text/plain",
+            headers={"Content-Disposition": "attachment; filename=error_log.txt"}
+        )
+    except Exception as e:
+        logger.error(f"Error while fetching log file: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error.")

competitions/enums.py

@@ -12,3 +12,12 @@ class SubmissionStatus(enum.Enum):
 class CompetitionType(enum.Enum):
     GENERIC = 1
     SCRIPT = 2
+
+
+
+class ErrorMessage(str, enum.Enum):
+    FAILED_TO_CREATE_SPACE = "FAILED_TO_CREATE_SPACE"
+    START_SPACE_FAILED = "START_SPACE_FAILED"
+    BUILD_SPACE_FAILED = "BUILD_SPACE_FAILED"
+    RUNTIME_ERROR = "RUNTIME_ERROR"
+    SPACE_TIMEOUT = "SPACE_TIMEOUT"

competitions/runner.py

@@ -13,7 +13,7 @@ import pandas as pd
 from huggingface_hub import HfApi, hf_hub_download, snapshot_download
 from loguru import logger
 
-from competitions.enums import SubmissionStatus
+from competitions.enums import SubmissionStatus, ErrorMessage
 from competitions.info import CompetitionInfo
 from competitions.utils import user_token_api, space_cleaner
 
@@ -221,7 +221,7 @@ class JobRunner:
             all_submissions = self._get_all_submissions()
 
             # Clean up spaces every 100 iterations
-            if cur % 100 == 0:
+            if cur % 100 == 1:
                 logger.info("Cleaning up spaces...")
                 for space in all_submissions:
                     if space["status"] in {SubmissionStatus.QUEUED.value, SubmissionStatus.PROCESSING.value}:
@@ -247,7 +247,7 @@ class JobRunner:
                     f"Failed to create space for {first_pending_sub['submission_id']}: {e}"
                 )
                 # mark submission as failed
-                self.mark_submission_failed(first_pending_sub['team_id'], first_pending_sub['submission_id'], "Failed to create space")
+                self.mark_submission_failed(first_pending_sub['team_id'], first_pending_sub['submission_id'], ErrorMessage.FAILED_TO_CREATE_SPACE.value)
                 try:
                     space_cleaner.delete_space(first_pending_sub["space_id"])
                 except Exception as e:

competitions/templates/index.html

@@ -150,15 +150,17 @@
                         if (data.response.submissions && data.response.submissions.length > 0 && data.response.error.length == 0) {
                             // Start building the table HTML
                             let tableHTML = teamNameDiv;
-                            tableHTML += '<table border="1"><tr><th>Datetime</th><th>Submission ID</th><th>Score</th><th>Status</th></tr>';
+                            tableHTML += '<table border="1"><tr><th width: 17%;>Datetime</th><th style="width: 40%;">Submission ID</th><th style="width: 20%;">Score</th><th style="width: 8%;">Status</th><th style="width: 15%;">Error Code</th></tr>';
 
                             // Iterate over each submission and add it to the table
                             data.response.submissions.forEach(submission => {
+                                logFileLink = submission.log_file_url === "" ? "" : `<a href="${submission.log_file_url}" target="_blank">log_file.txt</a>`;
                                 tableHTML += `<tr>
                                     <td>${submission.datetime}</td>
                                     <td>${submission.submission_id}</td>
                                     <td>${submission.score}</td>
                                     <td>${submission.status}</td>
+                                    <td>${submission.error_message} <br> ${logFileLink}</td>
                                   </tr>`;
                             });
 
@@ -369,7 +371,7 @@
 
         function checkOAuth() {
             var url = "/login_status";
-            makeApiRequest(url, function ({is_login, is_admin, is_registered}) {
+            makeApiRequest(url, function ({is_login, is_admin, is_registered, is_white_team}) {
                 const registerRemoveStyle = ["pointer-events-none", "text-gray-400", "cursor-not-allowed"];
                 const registerNewStyle = ["ext-gray-900", "hover:bg-gray-100"]
                 if (is_login) {
@@ -383,12 +385,16 @@
                     document.getElementById("admin").classList.remove("hidden");
                 }
                 if (is_registered) {
+                    document.getElementById("submission_info").classList.remove(...registerRemoveStyle);
+                    document.getElementById("submission_info").classList.add(...registerNewStyle);
+                } else {
+                    document.getElementById("registerButton").style.display = "block";
+                }
+                if (is_white_team) {
                     document.getElementById("my_submissions").classList.remove(...registerRemoveStyle);
                     document.getElementById("my_submissions").classList.add(...registerNewStyle);
                     document.getElementById("new_submission").classList.remove(...registerRemoveStyle);
                     document.getElementById("new_submission").classList.add(...registerNewStyle);
-                } else {
-                    document.getElementById("registerButton").style.display = "block";
                 }
             });
         }
@@ -478,7 +484,7 @@
                     <ul id="submissions-dropdown" class="py-2 space-y-2">
                         <li>
                             <a href="#" id="submission_info"
-                                class="flex items-center w-full p-2 text-gray-900 transition duration-75 rounded-lg pl-11 group hover:bg-gray-100">Submission
+                                class="flex items-center w-full p-2 transition duration-75 rounded-lg pl-11 group pointer-events-none text-gray-400 cursor-not-allowed">Submission
                                 information</a>
                         </li>
                         <li>

competitions/utils.py

@@ -8,11 +8,13 @@ import threading
 import uuid
 import base64
 import glob
-from typing import Optional, Dict, Any, List
+from typing import Optional, Dict, Any, List, Literal
 from collections import defaultdict
+from datetime import datetime, timezone, timedelta
 
 import requests
 import pandas as pd
+import jwt
 from fastapi import Request
 from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.utils._errors import RepositoryNotFoundError
@@ -21,7 +23,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from huggingface_hub import SpaceStage
 from cachetools import cached, TTLCache
 
-from competitions.enums import SubmissionStatus
+from competitions.enums import SubmissionStatus, ErrorMessage
 from competitions.params import EvalParams
 
 from . import HF_URL
@@ -400,6 +402,19 @@ class TeamFileApi:
                 repo_type="dataset",
             )
         return new_team_name
+    
+    @cached(cache=TTLCache(maxsize=1, ttl=600))
+    def get_team_white_list(self) -> List[str]:
+        file = hf_hub_download(
+            repo_id=self.competition_id,
+            filename="team_id_whitelist.json",
+            token=self.hf_token,
+            repo_type="dataset",
+        )
+
+        with open(file, "r", encoding="utf-8") as f:
+            team_white_list = json.load(f)
+        return team_white_list
 
 
 team_file_api = TeamFileApi(
@@ -551,12 +566,83 @@ submission_api = SubmissionApi(
 )
 
 
+class ErrorLogApi:
+    def __init__(self, hf_token: str, competition_id: str, encode_key: str):
+        self.hf_token = hf_token
+        self.competition_id = competition_id
+        self.api = HfApi(token=hf_token)
+        self.encode_key = encode_key
+
+    def save_error_log(self, submission_id: str, content: str):
+        """Save the error log of a space to the submission."""
+        content_buffer = io.BytesIO(content.encode())
+        self.api.upload_file(
+            path_or_fileobj=content_buffer,
+            path_in_repo=f"error_logs/{submission_id}.txt",
+            repo_id=self.competition_id,
+            repo_type="dataset",
+        )
+
+    def get_log(self, space_id: str, kind: Literal["run", "build"], tail: int = -1) -> str:
+        """Get the build log of a space."""
+        url = f"https://huggingface.co/api/spaces/{space_id}/logs/{kind}"
+        headers = {
+            "Authorization": f"Bearer {self.hf_token}"
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            raise RuntimeError(f"Failed to get logs: {response.status_code}\n{response.text}")
+        content = response.text
+        line_str_list = []
+        start_index = 0 if tail == -1 else max(0, len(content.split('\n')) - tail)
+        for line in content.split('\n')[start_index:]:
+            if line.startswith("data:"):
+                line_json = json.loads(line[5:].strip())
+                line_str_list.append(f"{line_json['timestamp']}: {line_json['data']}")
+        return "\n".join(line_str_list)
+
+    def generate_log_token(self, submission_id: str) -> str:
+        payload = {
+            "submission_id": submission_id,
+            "exp": datetime.now(timezone.utc) + timedelta(hours=1)
+        }
+        token = jwt.encode(payload, self.encode_key, algorithm="HS256")
+        return token
+    
+    def get_log_by_token(self, token: str) -> str:
+        try:
+            payload = jwt.decode(token, self.encode_key, algorithms=["HS256"])
+            submission_id = payload["submission_id"]
+        except jwt.ExpiredSignatureError:
+            raise RuntimeError("Token has expired.")
+        except jwt.InvalidTokenError as e:
+            raise RuntimeError(f"Invalid token: {e}")
+
+        log_file_path = self.api.hf_hub_download(
+            repo_id=self.competition_id,
+            filename=f"error_logs/{submission_id}.txt",
+            repo_type="dataset",
+        )
+        with open(log_file_path, 'r') as f:
+            file_content = f.read()
+        
+        return file_content
+
+
+error_log_api = ErrorLogApi(
+    hf_token=os.environ.get("HF_TOKEN", None),
+    competition_id=os.environ.get("COMPETITION_ID"),
+    encode_key=os.environ.get("ERROR_LOG_ENCODE_KEY", "key")
+)
+
+
 class SpaceCleaner:
-    def __init__(self, hf_token: str):
+    def __init__(self, hf_token: str, competition_id: str):
         self.hf_token = hf_token
+        self.competition_id = competition_id
         self.api = HfApi(token=hf_token)
         self.space_build_error_count = defaultdict(int)
-    
+
     def delete_space(self, space_id: str):
         """Delete a space by its ID."""
         self.api.delete_repo(repo_id=space_id, repo_type="space")
@@ -565,38 +651,53 @@ class SpaceCleaner:
         try:
             space_info = self.api.space_info(repo_id=space_id)
         except RepositoryNotFoundError:
-            submission_api.update_submission_status(
+            submission_api.update_submission_data(
                 team_id=team_id,
                 submission_id=submission_id,
-                data={"status": SubmissionStatus.FAILED.value, "error_message": "start space failed."},
+                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.START_SPACE_FAILED.value},
+            )
+            return
 
+        if (datetime.now(timezone.utc) - space_info.created_at).total_seconds() > 60 * 60 * 1.5:
+            # If the space is older than 1.5 hours, delete it
+            self.delete_space(space_id)
+            submission_api.update_submission_data(
+                team_id=team_id,
+                submission_id=submission_id,
+                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.SPACE_TIMEOUT.value},
             )
             return
+
         if space_info.runtime.stage == SpaceStage.BUILD_ERROR:
             self.space_build_error_count[space_id] += 1
             if self.space_build_error_count[space_id] >= 3:
+                log_content = error_log_api.get_log(space_id, kind="build")
+                error_log_api.save_error_log(submission_id, log_content)
                 self.delete_space(space_id)
-                submission_api.update_submission_status(
+                submission_api.update_submission_data(
                     team_id=team_id,
                     submission_id=submission_id,
-                    status=SubmissionStatus.FAILED.value
+                    data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.BUILD_SPACE_FAILED.value},
                 )
             else:
                 self.api.restart_space(repo_id=space_id)
             return
         
         if space_info.runtime.stage == SpaceStage.RUNTIME_ERROR:
+            log_content = error_log_api.get_log(space_id, kind="run")
+            error_log_api.save_error_log(submission_id, log_content)
             self.delete_space(space_id)
-            submission_api.update_submission_status(
+            submission_api.update_submission_data(
                 team_id=team_id,
                 submission_id=submission_id,
-                status=SubmissionStatus.FAILED.value
+                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.RUNTIME_ERROR.value},
             )
             return
 
 
 space_cleaner = SpaceCleaner(
     os.environ.get("HF_TOKEN", None),
+    os.environ.get("COMPETITION_ID")
 )
 
 

requirements.txt

@@ -14,6 +14,7 @@ pydantic==2.8.2
 gradio==4.37.2
 authlib==1.3.1
 itsdangerous==2.2.0
-hf-transfer
-filelock>=3.17.0
+hf-transfer>=0.1.9
+filelock>=3.13.3
 cachetools==6.0.0
+PyJWT>=2.10.1