Dockerfile

# Hugging Face TGI image
FROM ghcr.io/huggingface/text-generation-inference:3.0.2

# Set working directory
WORKDIR /app

# Create the /data directory inside the container
RUN mkdir -p /data
RUN chmod 777 /data
RUN mkdir -p /.cache
RUN chmod 777 /.cache
RUN mkdir -p /.triton
RUN chmod 777 /.triton

# Expose the model on port 8080
EXPOSE 8080

# Set the Hugging Face token as an environment variable
ARG HF_TOKEN
ENV HF_TOKEN=${HF_TOKEN}

# Run the TGI server directly
CMD ["--model-id", "allenai/OLMo-7B-0724-Instruct-hf", "--port", "8080"]