update nav

.gitattributes

@@ -32,4 +32,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -151,3 +151,4 @@ dmypy.json
 
 # lock
 submission_lock/*
+test_gt_datas

Dockerfile

@@ -50,4 +50,4 @@ COPY --chown=1000:1000 . /app/
 
 RUN pip install -r requirements.txt
 
-CMD ["python", "-m", "uvicorn", "competitions.app:app", "--workers", "1", "--port", "7860", "--host", "0.0.0.0"]
+CMD ["bash", "-c", "python download_pre_datas.py && python -m uvicorn competitions.app:app --workers 1 --port 7860 --host 0.0.0.0"]

README.md

@@ -1,5 +1,5 @@
 ---
-title: Hugsim Competition
+title: ICCV2025-RealADSim-NVS Competition
 emoji: 🏃
 colorFrom: blue
 colorTo: pink

competitions/app.py

@@ -27,8 +27,8 @@ from competitions.oauth import attach_oauth
 from competitions.runner import JobRunner
 from competitions.submissions import Submissions
 from competitions.text import SUBMISSION_SELECTION_TEXT, SUBMISSION_TEXT
-from competitions.utils import team_file_api, submission_api, leaderboard_api, error_log_api
-from competitions.enums import SubmissionStatus, ErrorMessage
+from competitions.utils import team_file_api, submission_api, leaderboard_api
+from competitions.enums import SubmissionStatus
 
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
@@ -294,10 +294,7 @@ def my_submissions(request: Request, user_token: str = Depends(utils.user_authen
     team_name = team_info["name"]
 
     for sub in subs:
-        if sub["error_message"] in {ErrorMessage.RUNTIME_ERROR, ErrorMessage.BUILD_SPACE_FAILED}:
-            sub["log_file_url"] = "/error_log?" + urlencode({"token": error_log_api.generate_log_token(sub["submission_id"])})
-        else:
-            sub["log_file_url"] = ""
+        sub.pop("final_score", "")
 
     resp = {
         "response": {
@@ -523,22 +520,6 @@ def register(
     return {"success": True, "response": "Team created successfully."}
 
 
-@app.get("/error_log")
-def get_error_log(token: str = Query(..., description="Token to access the error log file.")):
-    try:
-        log_content = error_log_api.get_log_by_token(token)
-        log_file = io.BytesIO(log_content.encode('utf-8'))
-        log_file.seek(0)
-        return StreamingResponse(
-            log_file,
-            media_type="text/plain",
-            headers={"Content-Disposition": "attachment; filename=error_log.txt"}
-        )
-    except Exception as e:
-        logger.error(f"Error while fetching log file: {e}")
-        raise HTTPException(status_code=500, detail="Internal server error.")
-
-
 @app.get("/register_page", response_class=HTMLResponse)
 def register_page(request: Request):
     """

competitions/runner.py

@@ -1,21 +1,48 @@
-import glob
-import io
 import json
 import os
 import time
-import uuid
+import io
 import shutil
+from uuid import uuid4
+import glob
 from dataclasses import dataclass
 from typing import List, Dict, Any
-from collections import defaultdict
 
+import torch
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+import lpips
+import numpy as np
+from huggingface_hub import HfApi, snapshot_download
 from loguru import logger
+from PIL import Image
+from torchvision.transforms.functional import to_tensor
+from torchmetrics.image import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure
 
-from competitions.enums import SubmissionStatus, ErrorMessage
+from competitions.enums import SubmissionStatus
 from competitions.info import CompetitionInfo
-from competitions.utils import user_token_api, space_cleaner, dockerfile_modifier
+from competitions.utils import submission_api, user_token_api
+
+
+def _psnr_mask(img1, img2, mask):
+
+    # Flatten mask
+    mask_flat = mask.reshape(-1)
+    img1_flat = img1.reshape(-1)
+    img2_flat = img2.reshape(-1)
+
+    # Non-zero indices
+    nonzero_indices = torch.nonzero(~mask_flat).squeeze()
+
+    # Only keep non-zero pixel
+    img1_nonzero = torch.index_select(img1_flat, 0, nonzero_indices)
+    img2_nonzero = torch.index_select(img2_flat, 0, nonzero_indices)
+
+    # MSE
+    mse = ((img1_nonzero - img2_nonzero) ** 2).mean()
+
+    # PSNR
+    psnr_value = 20 * torch.log10(1.0 / torch.sqrt(mse))
+    return psnr_value
 
 
 @dataclass
@@ -57,8 +84,6 @@ class JobRunner:
                         "datetime": sub["datetime"],
                         "status": sub["status"],
                         "submission_repo": sub["submission_repo"],
-                        "space_id": sub["space_id"],
-                        "server_url": sub["server_url"],
                         "hardware": sub["hardware"],
                     }
                 )
@@ -78,187 +103,161 @@ class JobRunner:
         pending_submissions = pending_submissions.sort_values("datetime")
         pending_submissions = pending_submissions.reset_index(drop=True)
         return pending_submissions
+    
+    def _avg_score(self, score_list: List[Dict[str, Any]]) -> Dict[str, Any]:
+        psnr, ssim, lpips  = [], [], []
+        for score in score_list:
+            psnr.append(score['psnr'])
+            ssim.append(score['ssim'])
+            lpips.append(score['lpips'])
+        return {'psnr': sum(psnr)/len(psnr), 'ssim': sum(ssim)/len(ssim), 'lpips': sum(lpips)/len(lpips)}
 
-    def _get_server_active_count(self, submissions: List[Dict[str, Any]]) -> Dict[str, int]:
-        server_active_count = defaultdict(int)
-        for sub in submissions:
-            if sub["status"] in {SubmissionStatus.PROCESSING.value, SubmissionStatus.QUEUED.value}:
-                server_active_count[sub["server_url"]] += 1
-        return server_active_count
-
-    def _queue_submission(self, team_id, submission_id):
-        team_fname = hf_hub_download(
-            repo_id=self.competition_id,
-            filename=f"submission_info/{team_id}.json",
-            token=self.token,
-            repo_type="dataset",
-        )
-        with open(team_fname, "r", encoding="utf-8") as f:
-            team_submission_info = json.load(f)
+    def _calculate_score(self, results: Dict[str, Any]) -> Dict[str, Any]:
+        all_scores, level1, level2, level3 = [], [], [], []
+        for im_name, scores in results.items():
+            all_scores.append(scores)
+            if "level1" in im_name:
+                level1.append(scores)
+            if "level2" in im_name:
+                level2.append(scores)
+            if "level3" in im_name:
+                level3.append(scores)
 
-        for submission in team_submission_info["submissions"]:
-            if submission["submission_id"] == submission_id:
-                submission["status"] = SubmissionStatus.QUEUED.value
-                break
+        return {
+            "all": self._avg_score(all_scores),
+            "level1": self._avg_score(level1),
+            "level2": self._avg_score(level2),
+            "level3": self._avg_score(level3),
+        }
 
-        team_submission_info_json = json.dumps(team_submission_info, indent=4)
-        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
-        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+    def _process_submission(self, submission: Dict[str, Any]):
         api = HfApi(token=self.token)
+        user_repo = submission["submission_repo"]
+        team_id = submission["team_id"]
+        submission_id = submission["submission_id"]
+
+        user_token = user_token_api.get(team_id)
+        client_commits = api.list_repo_commits(user_repo, repo_type="dataset")
+        client_code_local_dir = f"/tmp/data/client_repo/{uuid4().hex}"
+        try:
+            api.snapshot_download(
+                repo_id=user_repo,
+                repo_type="dataset",
+                revision=client_commits[0].commit_id,
+                token=user_token,
+                local_dir=client_code_local_dir,
+                allow_patterns=["*"],
+            )
+            evel_result = self._eval("./test_gt_datas", client_code_local_dir)
+        finally:
+            shutil.rmtree(client_code_local_dir, ignore_errors=True)
+        evel_result_json_string = json.dumps(evel_result, indent=2)
+        evel_result_json_bytes = evel_result_json_string.encode("utf-8")
+        evel_result_json_buffer = io.BytesIO(evel_result_json_bytes)
         api.upload_file(
-            path_or_fileobj=team_submission_info_json_buffer,
-            path_in_repo=f"submission_info/{team_id}.json",
+            path_or_fileobj=evel_result_json_buffer,
+            path_in_repo=f"eval_results/{submission_id}.json",
             repo_id=self.competition_id,
             repo_type="dataset",
         )
-
-    def mark_submission_failed(self, team_id: str, submission_id: str, error_message: str):
-        team_fname = hf_hub_download(
-            repo_id=self.competition_id,
-            filename=f"submission_info/{team_id}.json",
-            token=self.token,
-            repo_type="dataset",
+        final_score = self._calculate_score(evel_result)
+        score = final_score["all"]["psnr"] * 0.4 + final_score["all"]["ssim"] * 0.3 - final_score["all"]["lpips"] * 0.3
+        submission_api.update_submission_data(
+            team_id=team_id,
+            submission_id=submission_id,
+            data={
+                "status": SubmissionStatus.SUCCESS.value,
+                "final_score": final_score,
+                "score": np.round(score, 3),
+            }
         )
-        with open(team_fname, "r", encoding="utf-8") as f:
-            team_submission_info = json.load(f)
 
-        for submission in team_submission_info["submissions"]:
-            if submission["submission_id"] == submission_id:
-                submission["status"] = SubmissionStatus.FAILED.value
-                submission["error_message"] = error_message
+    def _eval(self, gt_folder_path: str, test_folder_path: str) -> Dict[str, Any]:
+        # list all files
+        files1 = sorted(glob.glob(os.path.join(gt_folder_path, '*/*/images', "*")))
+        files2 = sorted(glob.glob(os.path.join(test_folder_path, '*/*/images', "*")))
 
-        team_submission_info_json = json.dumps(team_submission_info, indent=4)
-        team_submission_info_json_bytes = team_submission_info_json.encode("utf-8")
-        team_submission_info_json_buffer = io.BytesIO(team_submission_info_json_bytes)
+        # filter by extensions
+        image_extensions = ('.png', '.jpg', '.jpeg')
+        images1 = [os.path.relpath(f, gt_folder_path) for f in files1 if f.lower().endswith(image_extensions)]
+        images2 = [os.path.relpath(f, test_folder_path) for f in files2 if f.lower().endswith(image_extensions)]
 
-        api = HfApi(token=self.token)
-        api.upload_file(
-            path_or_fileobj=team_submission_info_json_buffer,
-            path_in_repo=f"submission_info/{team_id}.json",
-            repo_id=self.competition_id,
-            repo_type="dataset",
-        )
+        # format check
+        if set(images1) != set(images2):
+            raise ValueError("Submission Format Error")
+        
+        # metrics
+        ssim_metric = StructuralSimilarityIndexMeasure(data_range=1.0).to("cuda" if torch.cuda.is_available() else "cpu")
+        lpips_metric = lpips.LPIPS(net='alex').to("cuda" if torch.cuda.is_available() else "cpu")
 
-    def _create_readme(self, project_name: str) -> str:
-        _readme = "---\n"
-        _readme += f"title: {project_name}\n"
-        _readme += "emoji: 🚀\n"
-        _readme += "colorFrom: green\n"
-        _readme += "colorTo: indigo\n"
-        _readme += "sdk: docker\n"
-        _readme += "pinned: false\n"
-        _readme += "---\n"
-        return _readme
+        results = {}
 
-    def create_space(self, team_id, submission_id, submission_repo, space_id, server_url, hardware):
-        user_token = user_token_api.get(team_id)
+        for img_name in images1:
+            path1 = os.path.join(gt_folder_path, img_name)
+            path2 = os.path.join(test_folder_path, img_name)
 
-        api = HfApi(token=self.token)
-        params = {
-            "space_id": space_id,
-            "client_space_id": space_id,
-            "competition_id": self.competition_id,
-            "team_id": team_id,
-            "submission_id": submission_id,
-            "output_path": self.output_path,
-            "submission_repo": submission_repo,
-            "time_limit": self.time_limit,
-            "dataset": self.dataset,
-            "submission_filenames": self.submission_filenames,
-        }
-        token_info_json = json.dumps(params, indent=4)
-        token_info_json_bytes = token_info_json.encode("utf-8")
-        token_info_json_buffer = io.BytesIO(token_info_json_bytes)
+            try:
+                # load images
+                img1 = Image.open(path1).convert("RGB")
+                img2 = Image.open(path2).convert("RGB")
+                if os.path.exists(path1.replace('images', 'masks')):
+                    dynamic_mask = Image.open(path1.replace('images', 'masks'))
+                else:
+                    dynamic_mask = Image.open(path1.replace('images', 'masks').replace('.jpg', '.png'))    
+                
+                # to tensor
+                tensor1 = to_tensor(img1).unsqueeze(0)
+                tensor2 = to_tensor(img2).unsqueeze(0)
+                dynamic_mask = to_tensor(dynamic_mask).unsqueeze(0).bool()
+                dynamic_mask = dynamic_mask.expand(-1, 3, -1, -1)
+                tensor1[dynamic_mask] *= 0
+                tensor2[dynamic_mask] *= 0
 
-        api = HfApi(token=self.token)
-        client_token = uuid.uuid4().hex + uuid.uuid4().hex
+                # move to devices
+                tensor1 = tensor1.to("cuda" if torch.cuda.is_available() else "cpu")
+                tensor2 = tensor2.to("cuda" if torch.cuda.is_available() else "cpu")
 
-        api.upload_file(
-            path_or_fileobj=token_info_json_buffer,
-            path_in_repo=f"token_data_info/{client_token}.json",
-            repo_id=self.competition_id,
-            repo_type="dataset",
-        )
-        api.create_repo(
-            repo_id=space_id,
-            repo_type="space",
-            space_sdk="docker",
-            space_hardware=hardware,
-            private=True,
-        )
+                # metrics
+                psnr_val = _psnr_mask(tensor1, tensor2, dynamic_mask).item()
+                ssim_val = ssim_metric(tensor1, tensor2).item()
+                lpips_val = lpips_metric(tensor1 * 2 - 1, tensor2 * 2 - 1).item()
 
-        api.add_space_secret(repo_id=space_id, key="HUGSIM_API_TOKEN", value=client_token)
-        api.add_space_secret(repo_id=space_id, key="HUGSIM_SERVER_HOST", value=server_url)
-        client_code_local_dir = f"/tmp/data/client_repo/{space_id}"
-        client_commits = api.list_repo_commits(submission_repo, repo_type="model")
-        api.snapshot_download(
-            repo_id=submission_repo,
-            repo_type="model",
-            revision=client_commits[0].commit_id,
-            token=user_token,
-            local_dir=client_code_local_dir,
-            allow_patterns=["*"],
-        )
-        with open(f"{client_code_local_dir}/README.md", "w", encoding="utf-8") as f:
-            f.write(self._create_readme(space_id))
-        shutil.copyfile("./other_files/network_filter.so", os.path.join(client_code_local_dir, "network_filter.so"))
-        for filename in os.listdir(client_code_local_dir):
-            if filename.lower() == "dockerfile":
-                filepath = os.path.join(client_code_local_dir, filename)
-                with open(filepath, "r", encoding="utf-8") as f:
-                    dockerfile_content = f.read()
-                with open(filepath, "w", encoding="utf-8") as f:
-                    f.write(dockerfile_modifier.modify_dockerfile_content(dockerfile_content)[0])
-        try:
-            api.upload_folder(
-                repo_id=space_id,
-                repo_type="space",
-                folder_path=client_code_local_dir,
-            )
-        finally:
-            shutil.rmtree(client_code_local_dir, ignore_errors=True)
-        self._queue_submission(team_id, submission_id)
+                results[img_name] = {
+                    "psnr": psnr_val,
+                    "ssim": ssim_val,
+                    "lpips": lpips_val
+                }
+            except Exception:
+                raise RuntimeError
+        return results
 
     def run(self):
-        cur = 0
         while True:
             time.sleep(5)
-            if cur == 10000:
-                cur = 0
-            cur += 1
-            all_submissions = self._get_all_submissions()
-
-            # Clean up spaces every 100 iterations
-            if cur % 100 == 1:
-                logger.info("Cleaning up spaces...")
-                for space in all_submissions:
-                    if space["status"] in {SubmissionStatus.QUEUED.value, SubmissionStatus.PROCESSING.value}:
-                        logger.info(f"Cleaning up space {space['space_id']} for submission {space['submission_id']}")
-                        space_cleaner.clean_space(
-                            space["space_id"],
-                            space["team_id"],
-                            space["submission_id"],
-                        )
 
+            all_submissions = self._get_all_submissions()
             pending_submissions = self._get_pending_subs(all_submissions)
             if pending_submissions is None:
                 continue
             first_pending_sub = pending_submissions.iloc[0]
-            server_active_count = self._get_server_active_count(all_submissions)
-
-            if server_active_count[first_pending_sub["server_url"]] >= 1:
-                continue
+            submission_api.update_submission_status(
+                team_id=first_pending_sub['team_id'],
+                submission_id=first_pending_sub['submission_id'],
+                status=SubmissionStatus.PROCESSING.value,
+            )
             try:
-                self.create_space(first_pending_sub["team_id"], first_pending_sub["submission_id"], first_pending_sub["submission_repo"], first_pending_sub["space_id"], first_pending_sub["server_url"], first_pending_sub["hardware"])
+                self._process_submission(first_pending_sub)
             except Exception as e:
                 logger.error(
-                    f"Failed to create space for {first_pending_sub['submission_id']}: {e}"
+                    f"Failed to process {first_pending_sub['submission_id']}: {e}"
+                )
+                submission_api.update_submission_data(
+                    team_id=first_pending_sub['team_id'],
+                    submission_id=first_pending_sub['submission_id'],
+                    data={
+                        "status": SubmissionStatus.FAILED.value,
+                        "error_message": str(e)
+                    }
                 )
-                # mark submission as failed
-                self.mark_submission_failed(first_pending_sub['team_id'], first_pending_sub['submission_id'], ErrorMessage.FAILED_TO_CREATE_SPACE.value)
-                try:
-                    space_cleaner.delete_space(first_pending_sub["space_id"])
-                except Exception as e:
-                    logger.error(f"Failed to delete space {first_pending_sub['space_id']}: {e}")
-                logger.error(f"Marked submission {first_pending_sub['submission_id']} as failed.")
+                raise e
                 continue

competitions/submissions.py

@@ -5,12 +5,12 @@ from dataclasses import dataclass
 from datetime import datetime
 
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download, SpaceHardware
+from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.utils._errors import EntryNotFoundError
 
 from competitions.enums import SubmissionStatus
 from competitions.errors import AuthenticationError, PastDeadlineError, SubmissionError, SubmissionLimitError
-from competitions.utils import token_information, team_file_api, user_token_api, server_manager
+from competitions.utils import token_information, team_file_api, user_token_api
 
 
 @dataclass
@@ -57,12 +57,9 @@ class Submissions:
         submission_id,
         submission_comment,
         submission_repo=None,
-        space_id=None,
     ):
         if submission_repo is None:
             submission_repo = ""
-        if space_id is None:
-            space_id = ""
         team_fname = hf_hub_download(
             repo_id=self.competition_id,
             filename=f"submission_info/{team_id}.json",
@@ -80,15 +77,13 @@ class Submissions:
                 "submission_id": submission_id,
                 "submission_comment": submission_comment,
                 "submission_repo": submission_repo,
-                "space_id": space_id,
                 "submitted_by": user_id,
                 "status": SubmissionStatus.PENDING.value,
                 "selected": False,
                 "public_score": {},
                 "private_score": {},
-                "score": {},
+                "score": 0,
                 "error_message": "",
-                "server_url": server_manager.get_next_server(),
                 "hardware": self.hardware,
             }
         )
@@ -155,7 +150,6 @@ class Submissions:
 
         # stringify public_score column
         submissions_df["public_score"] = submissions_df["public_score"].apply(json.dumps)
-        submissions_df["score"] = submissions_df["score"].apply(json.dumps)
 
         if private:
             submissions_df["private_score"] = submissions_df["private_score"].apply(json.dumps)
@@ -220,33 +214,8 @@ class Submissions:
         if self._is_submission_allowed(team_id) is False:
             raise SubmissionLimitError("Submission limit reached")
 
-        if self.competition_type == "generic":
-            bytes_data = uploaded_file.file.read()
-            # verify file is valid
-            if not self._verify_submission(bytes_data):
-                raise SubmissionError("Invalid submission file")
-
-            file_extension = uploaded_file.filename.split(".")[-1]
-            # upload file to hf hub
-            api = HfApi(token=self.token)
-            api.upload_file(
-                path_or_fileobj=bytes_data,
-                path_in_repo=f"submissions/{team_id}-{submission_id}.{file_extension}",
-                repo_id=self.competition_id,
-                repo_type="dataset",
-            )
-            submissions_made = self._increment_submissions(
-                team_id=team_id,
-                user_id=user_id,
-                submission_id=submission_id,
-                submission_comment=submission_comment,
-            )
-            return self.submission_limit - submissions_made
-
         user_api = HfApi(token=user_token)
         submission_id = user_api.model_info(repo_id=uploaded_file).sha + "__" + submission_id
-        competition_organizer = self.competition_id.split("/")[0]
-        space_id = f"{competition_organizer}/comp-{submission_id}"
 
         user_token_api.put(team_id, user_token)
         submissions_made = self._increment_submissions(
@@ -255,7 +224,6 @@ class Submissions:
             submission_id=submission_id,
             submission_comment=submission_comment,
             submission_repo=uploaded_file,
-            space_id=space_id,
         )
         remaining_submissions = self.submission_limit - submissions_made
         return remaining_submissions

competitions/templates/index.html

@@ -150,17 +150,16 @@
                         if (data.response.submissions && data.response.submissions.length > 0 && data.response.error.length == 0) {
                             // Start building the table HTML
                             let tableHTML = teamNameDiv;
-                            tableHTML += '<table border="1"><tr><th width: 17%;>Datetime</th><th style="width: 40%;">Submission ID</th><th style="width: 20%;">Score</th><th style="width: 8%;">Status</th><th style="width: 15%;">Error Code</th></tr>';
+                            tableHTML += '<table border="1"><tr><th width: 17%;>Datetime</th><th style="width: 40%;">Submission ID</th><th style="width: 20%;">Score</th><th style="width: 8%;">Status</th><th style="width: 15%;">Error</th></tr>';
 
                             // Iterate over each submission and add it to the table
                             data.response.submissions.forEach(submission => {
-                                logFileLink = submission.log_file_url === "" ? "" : `<a href="${submission.log_file_url}" target="_blank">log_file.txt</a>`;
                                 tableHTML += `<tr>
                                     <td>${submission.datetime}</td>
                                     <td>${submission.submission_id}</td>
                                     <td>${submission.score}</td>
                                     <td>${submission.status}</td>
-                                    <td>${submission.error_message} <br> ${logFileLink}</td>
+                                    <td>${submission.error_message}</td>
                                   </tr>`;
                             });
 
@@ -580,7 +579,7 @@
                 {% endif %}
                 {% if competition_type == 'script' %}
                 <div class="form-group">
-                    <label for="hub_model" class="text-sm font-medium text-gray-700">Hub model
+                    <label for="hub_model" class="text-sm font-medium text-gray-700">Hub Dataset
                     </label>
                     <input type="text" name="hub_model" id="hub_model"
                         class="mt-1 block w-full border border-gray-300 px-3 py-1.5 bg-white rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500"
@@ -840,7 +839,7 @@
             } else if (competitionType === 'script') {
                 var hubModel = document.getElementById('hub_model').value;
                 if (!hubModel) {
-                    alert('Hub model is required.');
+                    alert('Hub Dataset is required.');
                     return;
                 }
                 formData.append('hub_model', hubModel);

competitions/utils.py

@@ -3,7 +3,6 @@ import json
 import os
 import shlex
 import subprocess
-import re
 import threading
 import uuid
 import base64
@@ -14,6 +13,7 @@ from datetime import datetime, timezone, timedelta
 
 import requests
 import pandas as pd
+import numpy as np
 import jwt
 from fastapi import Request
 from huggingface_hub import HfApi, hf_hub_download
@@ -470,22 +470,6 @@ user_token_api = UserTokenApi(
 )
 
 
-class ServerManager:
-    def __init__(self, server_url_list: List[str]):
-        self.server_url_list = server_url_list
-        self._cur_index = 0
-        self._lock = threading.Lock()
-    
-    def get_next_server(self) -> str:
-        with self._lock:
-            server_url = self.server_url_list[self._cur_index]
-            self._cur_index = (self._cur_index + 1) % len(self.server_url_list)
-        return server_url
-    
-
-server_manager = ServerManager(["https://xdimlab-hugsim-web-server-0.hf.space"])
-
-
 class SubmissionApi:
     def __init__(self, hf_token: str, competition_id: str):
         self.hf_token = hf_token
@@ -558,141 +542,6 @@ submission_api = SubmissionApi(
 )
 
 
-class ErrorLogApi:
-    def __init__(self, hf_token: str, competition_id: str, encode_key: str):
-        self.hf_token = hf_token
-        self.competition_id = competition_id
-        self.api = HfApi(token=hf_token)
-        self.encode_key = encode_key
-
-    def save_error_log(self, submission_id: str, content: str):
-        """Save the error log of a space to the submission."""
-        content_buffer = io.BytesIO(content.encode())
-        self.api.upload_file(
-            path_or_fileobj=content_buffer,
-            path_in_repo=f"error_logs/{submission_id}.txt",
-            repo_id=self.competition_id,
-            repo_type="dataset",
-        )
-
-    def get_log(self, space_id: str, kind: Literal["run", "build"], tail: int = -1) -> str:
-        """Get the build log of a space."""
-        url = f"https://huggingface.co/api/spaces/{space_id}/logs/{kind}"
-        headers = {
-            "Authorization": f"Bearer {self.hf_token}"
-        }
-        response = requests.get(url, headers=headers)
-        if response.status_code != 200:
-            raise RuntimeError(f"Failed to get logs: {response.status_code}\n{response.text}")
-        content = response.text
-        line_str_list = []
-        start_index = 0 if tail == -1 else max(0, len(content.split('\n')) - tail)
-        for line in content.split('\n')[start_index:]:
-            if line.startswith("data:"):
-                line_json = json.loads(line[5:].strip())
-                line_str_list.append(f"{line_json['timestamp']}: {line_json['data']}")
-        return "\n".join(line_str_list)
-
-    def generate_log_token(self, submission_id: str) -> str:
-        payload = {
-            "submission_id": submission_id,
-            "exp": datetime.now(timezone.utc) + timedelta(hours=1)
-        }
-        token = jwt.encode(payload, self.encode_key, algorithm="HS256")
-        return token
-    
-    def get_log_by_token(self, token: str) -> str:
-        try:
-            payload = jwt.decode(token, self.encode_key, algorithms=["HS256"])
-            submission_id = payload["submission_id"]
-        except jwt.ExpiredSignatureError:
-            raise RuntimeError("Token has expired.")
-        except jwt.InvalidTokenError as e:
-            raise RuntimeError(f"Invalid token: {e}")
-
-        log_file_path = self.api.hf_hub_download(
-            repo_id=self.competition_id,
-            filename=f"error_logs/{submission_id}.txt",
-            repo_type="dataset",
-        )
-        with open(log_file_path, 'r') as f:
-            file_content = f.read()
-        
-        return file_content
-
-
-error_log_api = ErrorLogApi(
-    hf_token=os.environ.get("HF_TOKEN", None),
-    competition_id=os.environ.get("COMPETITION_ID"),
-    encode_key=os.environ.get("ERROR_LOG_ENCODE_KEY", "key")
-)
-
-
-class SpaceCleaner:
-    def __init__(self, hf_token: str, competition_id: str):
-        self.hf_token = hf_token
-        self.competition_id = competition_id
-        self.api = HfApi(token=hf_token)
-        self.space_build_error_count = defaultdict(int)
-
-    def delete_space(self, space_id: str):
-        """Delete a space by its ID."""
-        self.api.delete_repo(repo_id=space_id, repo_type="space")
-
-    def clean_space(self, space_id: str, team_id: str, submission_id: str):
-        try:
-            space_info = self.api.space_info(repo_id=space_id)
-        except RepositoryNotFoundError:
-            submission_api.update_submission_data(
-                team_id=team_id,
-                submission_id=submission_id,
-                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.START_SPACE_FAILED.value},
-            )
-            return
-
-        if (datetime.now(timezone.utc) - space_info.created_at).total_seconds() > 60 * 60 * 1.5:
-            # If the space is older than 1.5 hours, delete it
-            self.delete_space(space_id)
-            submission_api.update_submission_data(
-                team_id=team_id,
-                submission_id=submission_id,
-                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.SPACE_TIMEOUT.value},
-            )
-            return
-
-        if space_info.runtime.stage == SpaceStage.BUILD_ERROR:
-            self.space_build_error_count[space_id] += 1
-            if self.space_build_error_count[space_id] >= 3:
-                log_content = error_log_api.get_log(space_id, kind="build")
-                error_log_api.save_error_log(submission_id, log_content)
-                self.delete_space(space_id)
-                submission_api.update_submission_data(
-                    team_id=team_id,
-                    submission_id=submission_id,
-                    data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.BUILD_SPACE_FAILED.value},
-                )
-            else:
-                self.api.restart_space(repo_id=space_id)
-            return
-        
-        if space_info.runtime.stage == SpaceStage.RUNTIME_ERROR:
-            log_content = error_log_api.get_log(space_id, kind="run")
-            error_log_api.save_error_log(submission_id, log_content)
-            self.delete_space(space_id)
-            submission_api.update_submission_data(
-                team_id=team_id,
-                submission_id=submission_id,
-                data={"status": SubmissionStatus.FAILED.value, "error_message": ErrorMessage.RUNTIME_ERROR.value},
-            )
-            return
-
-
-space_cleaner = SpaceCleaner(
-    os.environ.get("HF_TOKEN", None),
-    os.environ.get("COMPETITION_ID")
-)
-
-
 class LeaderboardApi:
     def __init__(self, hf_token: str, competition_id: str):
         self.hf_token = hf_token
@@ -708,12 +557,12 @@ class LeaderboardApi:
         """
         all_scores = self._get_all_scores()
         if not all_scores:
-            return pd.DataFrame(columns=["team_id", "team_name", "rc", "hdscore"])
+            return pd.DataFrame(columns=["team_id", "team_name", "score"])
 
         df = pd.DataFrame(all_scores)
-        df = df.sort_values(by=["hdscore", "rc"], ascending=[False, False])
+        df = df.sort_values(by=["score"], ascending=[False])
         df = df.groupby("team_id").first()
-        df = df.sort_values(by=["hdscore", "rc"], ascending=[False, False])
+        df = df.sort_values(by=["score"], ascending=[False])
         df['rank'] = range(1, len(df) + 1)
         df.insert(0, 'rank', df.pop('rank'))
         df.reset_index(drop=True, inplace=True)
@@ -745,8 +594,7 @@ class LeaderboardApi:
                 all_scores.append({
                     "team_id": team_id,
                     "team_name": team_metadata[team_id]["name"],
-                    "rc": sub["score"]["rc"],
-                    "hdscore": sub["score"]["hdscore"],
+                    "score": sub["score"],
                 })
         return all_scores
 
@@ -755,163 +603,3 @@ leaderboard_api = LeaderboardApi(
     hf_token=os.environ.get("HF_TOKEN", None),
     competition_id=os.environ.get("COMPETITION_ID")
 )
-
-
-
-class DockerfileModifier:
-    def __init__(self, allowed_hosts: str, source_so_path: str = "./network_filter.so"):
-        self.allowed_hosts = allowed_hosts
-        self.source_so_path = source_so_path
-        self.tatget_so_dir = "/_app_extensions"
-        self.tatget_so_path = os.path.join(self.tatget_so_dir, "network_filter.so")
-        self.preload_prefix = f'LD_PRELOAD={self.tatget_so_path} ALLOWED_HOSTS="{allowed_hosts}"'
-        
-    def parse_dockerfile_line(self, line: str) -> Tuple[str, str, str]:
-        """
-        解析 Dockerfile 行，返回 (指令名, 原始命令, 格式类型)
-        格式类型: 'exec' (JSON数组) 或 'shell' (shell命令)
-        """
-        line = line.strip()
-        
-        # 匹配 CMD 或 ENTRYPOINT
-        cmd_match = re.match(r'^(CMD|ENTRYPOINT)\s+(.+)$', line, re.IGNORECASE)
-        if not cmd_match:
-            return "", "", ""
-            
-        instruction = cmd_match.group(1).upper()
-        command_part = cmd_match.group(2).strip()
-        
-        # 判断是 exec 格式 (JSON数组) 还是 shell 格式
-        if command_part.startswith('[') and command_part.endswith(']'):
-            return instruction, command_part, "exec"
-        else:
-            return instruction, command_part, "shell"
-    
-    def modify_shell_format(self, command: str) -> str:
-        """修改 shell 格式的命令"""
-        # 在原命令前添加环境变量
-        return f'{self.preload_prefix} {command}'
-    
-    def modify_exec_format(self, command: str) -> str:
-        """修改 exec 格式 (JSON数组) 的命令"""
-        try:
-            # 解析 JSON 数组格式
-            # 移除外层的方括号
-            inner = command[1:-1].strip()
-            
-            # 简单的 JSON 数组解析
-            parts = []
-            current = ""
-            in_quotes = False
-            escape_next = False
-            
-            for char in inner:
-                if escape_next:
-                    current += char
-                    escape_next = False
-                elif char == '\\':
-                    current += char
-                    escape_next = True
-                elif char == '"' and not escape_next:
-                    in_quotes = not in_quotes
-                    current += char
-                elif char == ',' and not in_quotes:
-                    parts.append(current.strip())
-                    current = ""
-                else:
-                    current += char
-            
-            if current.strip():
-                parts.append(current.strip())
-            
-            # 移除引号并处理转义
-            cleaned_parts = []
-            for part in parts:
-                part = part.strip()
-                if part.startswith('"') and part.endswith('"'):
-                    part = part[1:-1]
-                # 处理基本的转义字符
-                part = part.replace('\\"', '"').replace('\\\\', '\\')
-                cleaned_parts.append(part)
-            
-            if not cleaned_parts:
-                return command
-            
-            # 构建新的命令
-            # 第一个元素通常是 shell (/bin/sh, /bin/bash 等)
-            # 如果第一个元素是 shell，修改执行的命令
-            if len(cleaned_parts) >= 3 and cleaned_parts[0] in ['/bin/sh', '/bin/bash', 'sh', 'bash']:
-                if cleaned_parts[1] == '-c':
-                    # 格式: ["/bin/sh", "-c", "command"]
-                    original_cmd = cleaned_parts[2]
-                    new_cmd = f'{self.preload_prefix} {original_cmd}'
-                    new_parts = [cleaned_parts[0], cleaned_parts[1], new_cmd] + cleaned_parts[3:]
-                else:
-                    # 直接在现有命令前添加环境变量，通过 shell 执行
-                    original_cmd = ' '.join(cleaned_parts[1:])
-                    new_cmd = f'{self.preload_prefix} {original_cmd}'
-                    new_parts = [cleaned_parts[0], '-c', new_cmd]
-            else:
-                # 直接执行的命令，需要通过 shell 包装
-                original_cmd = ' '.join(cleaned_parts)
-                new_parts = ['/bin/sh', '-c', f'{self.preload_prefix} {original_cmd}']
-            
-            # 重新构建 JSON 数组
-            escaped_parts = []
-            for part in new_parts:
-                # 转义引号和反斜杠
-                escaped = part.replace('\\', '\\\\').replace('"', '\\"')
-                escaped_parts.append(f'"{escaped}"')
-            
-            return '[' + ', '.join(escaped_parts) + ']'
-            
-        except Exception as e:
-            print(f"警告: 解析 exec 格式失败: {e}")
-            print(f"原始命令: {command}")
-            # 如果解析失败，转换为 shell 格式
-            return f'{self.preload_prefix} {command}'
-    
-    def modify_dockerfile_content(self, content: str) -> Tuple[str, List[str]]:
-        """
-        修改 Dockerfile 内容
-        返回: (修改后的内容, 修改日志)
-        """
-        lines = content.splitlines()
-        modified_lines = []
-        changes = []
-        
-        for i, line in enumerate(lines, 1):
-            instruction, command, format_type = self.parse_dockerfile_line(line)
-            
-            if instruction in ['CMD', 'ENTRYPOINT'] and command:
-                if format_type == "shell":
-                    new_command = self.modify_shell_format(command)
-                    new_line = f'{instruction} {new_command}'
-                elif format_type == "exec":
-                    new_command = self.modify_exec_format(command)
-                    new_line = f'{instruction} {new_command}'
-                else:
-                    new_line = line
-                
-                changes.append(f"第 {i} 行: {instruction} 指令已修改")
-                changes.append(f"  原始: {line}")
-                changes.append(f"  修改: {new_line}")
-                modified_lines.append(new_line)
-            else:
-                modified_lines.append(line)
-
-        last_user = None
-        for line in modified_lines[::-1]:
-            if line.startswith("USER"):
-                last_user = line.split()[1].strip()
-
-        if last_user is None:
-            modified_lines.insert(-1, f"COPY {self.source_so_path}" + f" {self.tatget_so_path}")
-        else:
-            modified_lines.insert(-1, f"COPY --chown={last_user}:{last_user} {self.source_so_path} {self.tatget_so_path}")
-            modified_lines.insert(-1, f"RUN chown -R {last_user}:{last_user} {self.tatget_so_dir}")
-
-        return '\n'.join(modified_lines), changes
-
-
-dockerfile_modifier = DockerfileModifier("127.0.0.1,xdimlab-hugsim-web-server-0.hf.space")

download_pre_datas.py

@@ -0,0 +1,10 @@
+from huggingface_hub import snapshot_download
+import os
+
+snapshot_download(
+    repo_id='XDimLab/nvs-gt-datas',
+    revision='main',
+    local_dir='./test_gt_datas',
+    repo_type='dataset',
+    token=os.getenv("HF_TOKEN", ""),
+)

requirements.txt

@@ -18,3 +18,8 @@ hf-transfer>=0.1.9
 filelock>=3.13.3
 cachetools==6.0.0
 PyJWT>=2.10.1
+pillow==10.4.0
+torch==2.7.1
+torchvision==0.22.1
+torchmetrics==1.7.3
+lpips==0.1.4