ci: add HEALTHCHECK step calling the /health_models endpoint

Dockerfile

@@ -177,6 +177,7 @@ ENV IS_AWS_LAMBDA=""
 
 RUN mkdir ${FASTAPI_STATIC}
 
+COPY --chown=python:python ./scripts/healthcheck.py ${WORKDIR_ROOT}/
 COPY --chown=python:python ./app.py ${WORKDIR_ROOT}/
 COPY --chown=python:python ./sam-quantized/machine_learning_models ${WORKDIR_ROOT}/machine_learning_models
 COPY --chown=python:python --from=node_prod_deps /appnode/node_modules* ${FASTAPI_STATIC}/node_modules
@@ -217,8 +218,6 @@ RUN echo "FASTAPI_STATIC:"
 RUN ls -l ${FASTAPI_STATIC}/ || true
 RUN ls -l ${FASTAPI_STATIC}/dist || true
 RUN ls -l ${FASTAPI_STATIC}/node_modules || true
-RUN chown python:python ${WORKDIR_ROOT}/docker_entrypoint.sh ${WORKDIR_ROOT}/entrypoint.sh
-RUN chmod 744 ${WORKDIR_ROOT}/docker_entrypoint.sh ${WORKDIR_ROOT}/entrypoint.sh
 RUN ls -ld ${WORKDIR_ROOT}/
 RUN ls -lA ${WORKDIR_ROOT}/
 RUN ls -l ${WORKDIR_ROOT}/.venv
@@ -230,3 +229,5 @@ RUN ls -l ${WORKDIR_ROOT}/.venv/bin/activate
 #]
 # CMD ["/usr/bin/bash", "-c", "source /var/task/.venv/bin/activate && uvicorn app:app --host 0.0.0.0 --port 7860"]
 CMD ["/var/task/docker_entrypoint.sh"]
+# HEALTHCHECK --interval=30s --timeout=900s --start-period=5s --retries=3 CMD "python -c 'import requests; r1 = requests.get(\"http://localhost:7860/health\"); print(r1.status_code); r2 = requests.get(\"http://localhost:7860/health_models\"); print(f\"status health:{r1.status_code}, health_models:{r2.status_code}!\"); exit(0) if r1.status_code == 200 and r2.status_code == 200 else exit(1)'"
+HEALTHCHECK --interval=10s --timeout=1080s --start-period=10s --start-interval=10s --retries=3 CMD [ "python", "healthcheck.py" ]

app.py

@@ -115,6 +115,31 @@ async def health() -> JSONResponse:
     return JSONResponse(status_code=200, content={"msg": "still alive..."})
 
 
+@app.get("/health_models")
+async def health_models() -> JSONResponse:
+    from samgis_lisa.prediction_api import lisa
+    from samgis_lisa.utilities.constants import LISA_INFERENCE_FN
+    from samgis_web.__version__ import __version__ as version_web
+    from samgis_core.__version__ import __version__ as version_core
+    from lisa_on_cuda.__version__ import __version__ as version_lisa_on_cuda
+    from samgis_lisa.__version__ import __version__ as version_samgis_lisa
+    from samgis_lisa.prediction_api.global_models import models_dict
+
+    app_logger.info(f"still alive, version_web:{version_web}, version_core:{version_core}.")
+    app_logger.info(f"still alive, version_lisa_on_cuda:{version_lisa_on_cuda}, version_samgis_lisa:{version_samgis_lisa}.")
+    app_logger.info(f"try to load inference function for '{LISA_INFERENCE_FN}' model...")
+    if models_dict[LISA_INFERENCE_FN]["inference"] is None:
+        app_logger.info(f"model not found, loading inference function for '{LISA_INFERENCE_FN}' model. This could take some minutes...")
+        lisa.load_model_and_inference_fn(LISA_INFERENCE_FN, inference_decorator=None, device_map="auto", device="cuda")
+    try:
+        model_name = models_dict[LISA_INFERENCE_FN]["inference"]
+        app_logger.info(f"inference function for '{LISA_INFERENCE_FN}' model => '{model_name.__name__}' found and loaded...")
+    except KeyError as ke:
+        app_logger.error(f"model not found, error:{ke}.")
+        raise HTTPException(status_code=500, detail="Internal Server Error")
+    return JSONResponse(status_code=200, content={"msg": f"still alive, inference function for '{LISA_INFERENCE_FN}' model loaded..."})
+
+
 def infer_lisa_gradio(request_input: StringPromptApiRequestBody) -> str:
     from samgis_lisa.io_package.wrappers_helpers import get_parsed_bbox_points_with_string_prompt
     from samgis_lisa.prediction_api import lisa

scripts/healthcheck.py

@@ -0,0 +1,23 @@
+
+def run_healthcheck():
+    import os
+    import requests
+    import sys
+    import structlog
+    from samgis_core.utilities.session_logger import setup_logging
+
+    log_level = os.getenv("LOG_LEVEL", "INFO")
+    url1 = os.getenv("HEALTHCHECK_URL1", "http://localhost:7860/health")
+    url2 = os.getenv("HEALTHCHECK_URL2", "http://localhost:7860/health_models")
+    setup_logging(log_level=log_level)
+    app_logger = structlog.stdlib.get_logger()
+
+    r1 = requests.get(url1)
+    app_logger.info(r1.status_code)
+    r2 = requests.get(url2)
+    app_logger.info(f"status health:{r1.status_code}, health_models:{r2.status_code}!")
+    sys.exit(0) if r1.status_code == 200 and r2.status_code == 200 else sys.exit(1)
+
+
+if __name__ == "__main__":
+    run_healthcheck()