Modified download_deps.py (#2747)
Browse files### What problem does this PR solve?
Modified download_deps.py
### Type of change
- [ ] Bug Fix (non-breaking change which fixes an issue)
- [ ] New Feature (non-breaking change which adds functionality)
- [ ] Documentation Update
- [ ] Refactoring
- [ ] Performance Improvement
- [x] Other (please describe): CI
- .github/workflows/tests.yml +1 -1
- Dockerfile +3 -2
- download_deps.py +11 -0
    	
        .github/workflows/tests.yml
    CHANGED
    
    | @@ -48,7 +48,7 @@ jobs: | |
| 48 | 
             
                  - name: Build ragflow:dev-slim
         | 
| 49 | 
             
                    run: |
         | 
| 50 | 
             
                      RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
         | 
| 51 | 
            -
                      cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data .
         | 
| 52 | 
             
                      sudo docker pull ubuntu:24.04
         | 
| 53 | 
             
                      sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
         | 
| 54 |  | 
|  | |
| 48 | 
             
                  - name: Build ragflow:dev-slim
         | 
| 49 | 
             
                    run: |
         | 
| 50 | 
             
                      RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
         | 
| 51 | 
            +
                      cp -r ${RUNNER_WORKSPACE_PREFIX}/huggingface.co ${RUNNER_WORKSPACE_PREFIX}/nltk_data ${RUNNER_WORKSPACE_PREFIX}/libssl*.deb .
         | 
| 52 | 
             
                      sudo docker pull ubuntu:24.04
         | 
| 53 | 
             
                      sudo docker build -f Dockerfile.slim -t infiniflow/ragflow:dev-slim .
         | 
| 54 |  | 
    	
        Dockerfile
    CHANGED
    
    | @@ -24,8 +24,9 @@ RUN --mount=type=cache,id=ragflow_base_apt,target=/var/cache/apt,sharing=locked | |
| 24 |  | 
| 25 | 
             
            # https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
         | 
| 26 | 
             
            # aspose-slides on linux/arm64 is unavailable
         | 
| 27 | 
            -
            RUN  | 
| 28 | 
            -
             | 
|  | |
| 29 | 
             
                fi
         | 
| 30 |  | 
| 31 | 
             
            ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
         | 
|  | |
| 24 |  | 
| 25 | 
             
            # https://forum.aspose.com/t/aspose-slides-for-net-no-usable-version-of-libssl-found-with-linux-server/271344/13
         | 
| 26 | 
             
            # aspose-slides on linux/arm64 is unavailable
         | 
| 27 | 
            +
            RUN --mount=type=bind,source=libssl1.1_1.1.1f-1ubuntu2_amd64.deb,target=/root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb \
         | 
| 28 | 
            +
                if [ "${ARCH}" = "amd64" ]; then \
         | 
| 29 | 
            +
                    dpkg -i /root/libssl1.1_1.1.1f-1ubuntu2_amd64.deb; \
         | 
| 30 | 
             
                fi
         | 
| 31 |  | 
| 32 | 
             
            ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1
         | 
    	
        download_deps.py
    CHANGED
    
    | @@ -3,6 +3,11 @@ | |
| 3 | 
             
            from huggingface_hub import snapshot_download
         | 
| 4 | 
             
            import nltk
         | 
| 5 | 
             
            import os
         | 
|  | |
|  | |
|  | |
|  | |
|  | |
| 6 |  | 
| 7 | 
             
            repos = [
         | 
| 8 | 
             
                "InfiniFlow/text_concat_xgb_v1.0",
         | 
| @@ -20,6 +25,12 @@ def download_model(repo_id): | |
| 20 |  | 
| 21 |  | 
| 22 | 
             
            if __name__ == "__main__":
         | 
|  | |
|  | |
|  | |
|  | |
|  | |
|  | |
| 23 | 
             
                local_dir = os.path.abspath('nltk_data')
         | 
| 24 | 
             
                for data in ['wordnet', 'punkt', 'punkt_tab']:
         | 
| 25 | 
             
                    print(f"Downloading nltk {data}...")
         | 
|  | |
| 3 | 
             
            from huggingface_hub import snapshot_download
         | 
| 4 | 
             
            import nltk
         | 
| 5 | 
             
            import os
         | 
| 6 | 
            +
            import urllib.request
         | 
| 7 | 
            +
             | 
| 8 | 
            +
            urls = [
         | 
| 9 | 
            +
                "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
         | 
| 10 | 
            +
            ]
         | 
| 11 |  | 
| 12 | 
             
            repos = [
         | 
| 13 | 
             
                "InfiniFlow/text_concat_xgb_v1.0",
         | 
|  | |
| 25 |  | 
| 26 |  | 
| 27 | 
             
            if __name__ == "__main__":
         | 
| 28 | 
            +
                for url in urls:
         | 
| 29 | 
            +
                    filename = url.split("/")[-1]
         | 
| 30 | 
            +
                    print(f"Downloading {url}...")
         | 
| 31 | 
            +
                    if not os.path.exists(filename):
         | 
| 32 | 
            +
                        urllib.request.urlretrieve(url, filename)
         | 
| 33 | 
            +
             | 
| 34 | 
             
                local_dir = os.path.abspath('nltk_data')
         | 
| 35 | 
             
                for data in ['wordnet', 'punkt', 'punkt_tab']:
         | 
| 36 | 
             
                    print(f"Downloading nltk {data}...")
         |