| ARG SDK_VERSION=latest | |
| FROM huggingface/hfendpoints-sdk:${SDK_VERSION} AS sdk | |
| FROM python:3.12-bookworm | |
| RUN apt update && apt install -y libjemalloc2 | |
| RUN --mount=type=bind,from=sdk,source=/opt/hfendpoints/dist,target=/usr/local/endpoints/dist \ | |
| --mount=type=bind,source=requirements.txt,target=/tmp/requirements.txt \ | |
| python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu && \ | |
| python3 -m pip install -r /tmp/requirements.txt && \ | |
| python3 -m pip install /usr/local/endpoints/dist/*.whl | |
| COPY handler.py /usr/local/endpoint/ | |
| # OMP & MALLOC related tuning | |
| ENV KMP_AFFINITY="granularity=fine,compact,1,0" | |
| ENV KMP_BLOCKTIME=1 | |
| ENV MALLOC_CONF=oversize_threshold:1,background_thread:true,metadata_thp:auto,dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000 | |
| ENV LD_PRELOAD=/usr/local/lib/libiomp5.so;/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 | |
| # Network interface | |
| ENV INTERFACE=0.0.0.0 | |
| ENV PORT=80 | |
| EXPOSE 80 | |
| ENTRYPOINT ["python3"] | |
| CMD ["/usr/local/endpoint/handler.py"] | |