update docker image

.github/workflows/tests.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install .[dev]
+        python -m pip install .[test]
     - name: Make test
       run: |
         make test

Dockerfile

@@ -1,7 +1,41 @@
-FROM huggingface/autotrain-advanced:latest
+FROM nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04
 
 ENV DEBIAN_FRONTEND=noninteractive \
-    TZ=UTC
+    TZ=UTC \
+    HF_HUB_ENABLE_HF_TRANSFER=1
+
+ENV PATH="${HOME}/miniconda3/bin:${PATH}"
+ARG PATH="${HOME}/miniconda3/bin:${PATH}"
+
+RUN mkdir -p /tmp/model && \
+    chown -R 1000:1000 /tmp/model && \
+    mkdir -p /tmp/data && \
+    chown -R 1000:1000 /tmp/data
+
+RUN apt-get update &&  \
+    apt-get upgrade -y &&  \
+    apt-get install -y \
+    build-essential \
+    cmake \
+    curl \
+    ca-certificates \
+    gcc \
+    git \
+    locales \
+    net-tools \
+    wget \
+    libpq-dev \
+    libsndfile1-dev \
+    git \
+    git-lfs \
+    libgl1 \
+    unzip \
+    && rm -rf /var/lib/apt/lists/* && \
+    apt-get clean
+
+
+RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
+    git lfs install
 
 WORKDIR /app
 RUN mkdir -p /app/.cache
@@ -10,11 +44,27 @@ RUN chown -R 1000:1000 /app
 USER 1000
 ENV HOME=/app
 
+ENV PYTHONPATH=$HOME/app \
+    PYTHONUNBUFFERED=1 \
+    SYSTEM=spaces
+
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
+    && sh Miniconda3-latest-Linux-x86_64.sh -b -p /app/miniconda \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh
+ENV PATH /app/miniconda/bin:$PATH
+
+RUN conda create -p /app/env -y python=3.10
+
 SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
 
+RUN conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia && \
+    conda clean -ya && \
+    conda install -c "nvidia/label/cuda-12.1.0" cuda-nvcc && conda clean -ya
+
 COPY --chown=1000:1000 . /app/
 RUN make socket-kit.so
 
 ENV PATH="/app:${PATH}"
 
-RUN pip install -e .
+RUN pip install -e .

requirements.txt

@@ -1,8 +1,31 @@
+# essentials
+fastapi-0.108.0
 joblib==1.3.2
 loguru==0.7.2
 pandas==2.1.4
-huggingface_hub==0.20.1
+huggingface_hub==0.20.2
 tabulate==0.9.0
 markdown==3.5.1
 psutil==5.9.0
-scikit-learn==1.3.2
+python-multipart==0.0.6
+uvicorn==0.22.0
+py7zr==0.20.6
+pydantic==2.5.3
+hf-transfer
+# machine learning libs
+scikit-learn==1.3.2
+jiwer==3.0.2
+sacremoses==0.0.53
+sentencepiece==0.1.99
+xgboost==1.7.6
+nltk==3.8.1
+optuna==3.3.0
+einops==0.6.1
+peft==0.7.1
+trl==0.7.4
+tiktoken==0.5.1
+transformers==4.36.1
+accelerate==0.25.0
+diffusers==0.21.4
+bitsandbytes==0.41.0
+tqdm==4.65.0

setup.py

@@ -26,7 +26,7 @@ TEST_REQUIRE = ["pytest", "pytest-cov"]
 EXTRAS_REQUIRE = {
     "dev": QUALITY_REQUIRE,
     "quality": QUALITY_REQUIRE,
-    "test": TEST_REQUIRE,
+    "test": QUALITY_REQUIRE + TEST_REQUIRE,
 }
 
 with open("requirements.txt", encoding="utf-8") as f: