initial push

.gitignore

@@ -0,0 +1,2 @@
+# Virtual environments
+venv

Dockerfile

@@ -0,0 +1,40 @@
+# Use the official Python image with a specific version
+FROM python:3.12.5-slim
+
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies as root
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    g++ \
+    cmake \
+    git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Switch back to the non-root user
+USER user
+
+# Copy the requirements file into the container
+COPY --chown=user ./requirements.txt requirements.txt
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the application code into the container
+COPY --chown=user ./llm.py /app/llm.py
+COPY --chown=user ./llama-3.2-1b-instruct-q4_k_m.gguf /app/llama-3.2-1b-instruct-q4_k_m.gguf
+
+# Expose the application port
+EXPOSE 7860
+
+# Define the command to run the application
+CMD ["uvicorn", "llm:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,36 @@
+from llama_cpp import Llama
+
+# Initialize the LLM 
+llm = Llama.from_pretrained(
+    repo_id="hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF", 
+    filename="llama-3.2-1b-instruct-q4_k_m.gguf"
+)
+
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from llama_cpp import Llama
+
+# Initialize the LLM once when the application starts
+llm = Llama.from_pretrained(
+    repo_id="hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF", 
+    filename="llama-3.2-1b-instruct-q4_k_m.gguf"
+)
+
+app = FastAPI()
+
+class ChatRequest(BaseModel):
+    message: str
+
+@app.post("/chat")
+async def chat_completion(request: ChatRequest):
+    try:
+        response = llm.create_chat_completion(
+            messages=[
+                {"role": "user", "content": request.message}
+            ]
+        )
+        return {
+            "response": response['choices'][0]['message']['content']
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

requirements.txt

@@ -0,0 +1,27 @@
+annotated-types==0.7.0
+anyio==4.6.2.post1
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
+diskcache==5.6.3
+fastapi==0.115.5
+filelock==3.16.1
+fsspec==2024.10.0
+h11==0.14.0
+huggingface-hub==0.26.2
+idna==3.10
+Jinja2==3.1.4
+llama_cpp_python==0.3.2
+MarkupSafe==3.0.2
+numpy==2.1.3
+packaging==24.2
+pydantic==2.9.2
+pydantic_core==2.23.4
+PyYAML==6.0.2
+requests==2.32.3
+sniffio==1.3.1
+starlette==0.41.2
+tqdm==4.67.0
+typing_extensions==4.12.2
+urllib3==2.2.3
+uvicorn==0.32.0