add timeout

competitions/evaluate.py

@@ -36,13 +36,32 @@ def generate_submission_file(params):
     # the script.py will generate a submission.csv file in the submission_dir
     # push the submission.csv file to the repo using upload_submission_file
     logger.info("Generating submission file")
-    # copy socket-kit.so to submission_dir
+
+    # Copy socket-kit.so to submission_dir
     shutil.copyfile("socket-kit.so", f"{submission_dir}/socket-kit.so")
+
+    # Define your command
     cmd = "python script.py"
     socket_kit_path = os.path.abspath(f"{submission_dir}/socket-kit.so")
+
+    # Copy the current environment and modify it
     env = os.environ.copy()
     env["LD_PRELOAD"] = socket_kit_path
-    subprocess.run(cmd, cwd=submission_dir, shell=True, check=True, env=env)
+
+    # Start the subprocess
+    process = subprocess.Popen(cmd, cwd=submission_dir, shell=True, env=env)
+
+    # Wait for the process to complete or timeout
+    try:
+        process.wait(timeout=params.time_limit)
+    except subprocess.TimeoutExpired:
+        logger.info(f"Process exceeded {params.time_limit} seconds time limit. Terminating...")
+        process.kill()
+        process.wait()
+
+    # Check if process terminated due to timeout
+    if process.returncode and process.returncode != 0:
+        logger.error("Subprocess didn't terminate successfully")
 
     api = HfApi(token=params.token)
     api.upload_file(

competitions/info.py

@@ -121,3 +121,7 @@ class CompetitionInfo:
     @property
     def submission_rows(self):
         return self.config["SUBMISSION_ROWS"]
+
+    @property
+    def time_limit(self):
+        return self.config["TIME_LIMIT"]

competitions/params.py

@@ -16,6 +16,7 @@ class EvalParams(BaseModel):
     submission_rows: int
     output_path: str
     submission_repo: str
+    time_limit: int
 
     class Config:
         protected_namespaces = ()

competitions/runner.py

@@ -36,6 +36,7 @@ class JobRunner:
         self.submission_id_col = self.competition_info.submission_id_col
         self.submission_cols = self.competition_info.submission_cols
         self.submission_rows = self.competition_info.submission_rows
+        self.time_limit = self.competition_info.time_limit
 
     def get_pending_subs(self):
         submission_jsons = snapshot_download(
@@ -62,7 +63,6 @@ class JobRunner:
                         }
                     )
         if len(pending_submissions) == 0:
-            logger.info("No pending submissions.")
             return None
         logger.info(f"Found {len(pending_submissions)} pending submissions.")
         pending_submissions = pd.DataFrame(pending_submissions)
@@ -119,6 +119,7 @@ class JobRunner:
             "submission_rows": self.submission_rows,
             "output_path": self.output_path,
             "submission_repo": submission_repo,
+            "time_limit": self.time_limit,
         }
 
         api.add_space_secret(repo_id=space_id, key="PARAMS", value=json.dumps(params))