diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000000000000000000000000000000000..df7d5525e6823a8792613d9e5bafd18ef011d553 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +{ + "name": "jio-savan-music-downloader", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + "features": { + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12" + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers-contrib/features/poetry:2": {}, + "ghcr.io/devcontainers/features/docker-in-docker:2.12.0": { + "version": "latest", + "moby": true + } + }, + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "./.devcontainer/postCreateCommand.sh", + "forwardPorts": [80, 8000], + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "editorconfig.editorconfig", + "ms-azuretools.vscode-docker", // Docker + "ms-python.isort", // isort + "visualstudioexptteam.vscodeintellicode", // IntelliCode + "codeium.codeium", // Codeium AI + "ms-vscode.makefile-tools", // Makefile tool + "ms-python.python", // Python + "ms-python.black-formatter", // Black + "ms-python.debugpy", // Debugger for Python + "redhat.vscode-yaml" // YAML + ], + "settings": { + "python.testing.pytestArgs": ["tests"], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.defaultInterpreterPath": "/workspaces/jio-savan-music-downloader/.venv/bin/python", + "python.testing.pytestPath": "/workspaces/jio-savan-music-downloader/.venv/bin/pytest", + "python.languageServer": "Pylance", + "editor.formatOnSave": true, + "python.analysis.typeCheckingMode": "basic", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true + } + } + }//, + // "hostRequirements": { + // "cpus": 2, + // "memory": "4gb", + // "storage": "10gb" + // } +} diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100755 index 0000000000000000000000000000000000000000..2aa4fef218cb55e610a9f3411191b42cc34d18f4 --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +# Install fish terminal +sudo apt update -y +sudo apt-get install fish -y + +# Repo Initialization +make init-repo +git config --global --add safe.directory /workspaces/jio-savan-music-downloader + +# Install Dependencies +make reset-env diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000000000000000000000000000000000..6e31f62c567ecf356733fcf5bbd8cccfd2610770 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf +max_line_length = 120 + +[*.bat] +indent_style = tab +end_of_line = crlf + +[LICENSE] +insert_final_newline = false + +[Makefile] +indent_style = tab + +[*.json] +indent_style = space +indent_size = 4 diff --git a/.github/actions/setup-poetry-env/action.yml b/.github/actions/setup-poetry-env/action.yml new file mode 100644 index 0000000000000000000000000000000000000000..150eebbf3e9adef251e033f17da5a4a0f1b85cd8 --- /dev/null +++ b/.github/actions/setup-poetry-env/action.yml @@ -0,0 +1,42 @@ +name: "setup-poetry-env" +description: "Composite action to setup the Python and poetry environment." + +inputs: + python-version: + required: false + description: "The python version to use" + default: "3.12" + +runs: + using: "composite" + steps: + - name: Set up python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install Poetry + env: + POETRY_VERSION: "1.7.1" + run: curl -sSL https://install.python-poetry.org | python - -y + shell: bash + + - name: Add Poetry to Path + run: echo "$HOME/.local/bin" >> $GITHUB_PATH + shell: bash + + - name: Configure Poetry virtual environment in project + run: poetry config virtualenvs.in-project true + shell: bash + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction + shell: bash diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml new file mode 100644 index 0000000000000000000000000000000000000000..7bb887094e887d5bd4f88c7a716c0665559caf4a --- /dev/null +++ b/.github/workflows/build-and-deploy.yml @@ -0,0 +1,55 @@ +name: Build and Deploy Docker Image + +on: + workflow_dispatch: {} + release: + types: [published] + branches: [main] + repository_dispatch: + types: [package-release] + +env: + DOCKER_IMAGE: deepak93p/jio_savan_music_downloader # Replace with your DockerHub image + AZURE_WEBAPP_NAME: jio-savan-music-downloader-app # Replace with your Azure Web App name + AZURE_WEBAPP_RG: jio-savan-music-downloader-rg # Replace with your Azure Web App resource group + +jobs: + build-push-docker-image: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + env: + DOCKER_USER: deepak93p + DOCKER_PWD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }} + run: echo $DOCKER_PWD | docker login -u $DOCKER_USER --password-stdin + + - name: Build and Push Docker Image + run: make bake-container-and-push IMAGE=${{ env.DOCKER_IMAGE }} TAG=${{ github.sha }} + + - name: Clean up Docker system + run: docker system prune -f + + deploy: + runs-on: ubuntu-latest + needs: build-push-docker-image + environment: + name: "production" + + steps: + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: Deploy to Azure Container Apps + run: | + az containerapp update \ + --name ${{ env.AZURE_WEBAPP_NAME }} \ + --resource-group ${{ env.AZURE_WEBAPP_RG }} \ + --image index.docker.io/${{ env.DOCKER_IMAGE }}:${{ github.sha }} \ + --query "properties.configuration.ingress.fqdn" \ + -o tsv diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 0000000000000000000000000000000000000000..ef22b1e5b406914fce7c546b12c875680fc78eb2 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,22 @@ +name: Deploy Documentation + +on: + release: + types: [published] + branches: [main] + repository_dispatch: + types: [package-release] + +jobs: + deploy-docs: + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + + - name: Deploy documentation + run: poetry run mkdocs gh-deploy --force diff --git a/.github/workflows/publish-package.yml b/.github/workflows/publish-package.yml new file mode 100644 index 0000000000000000000000000000000000000000..e716e357f998cdb0511d211e0e73d641a46962d3 --- /dev/null +++ b/.github/workflows/publish-package.yml @@ -0,0 +1,41 @@ +name: Publish Package + +on: + release: + types: [published] + branches: [main] + repository_dispatch: + types: [package-release] + + +jobs: + publish-to-pypi: + runs-on: ubuntu-latest + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + + - name: Export tag + id: vars + run: | + if [ "${{ github.event.client_payload.version }}" != "" ]; then + echo "tag=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT + elif [ "${{ github.event.inputs.version }}" != "" ]; then + echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT + else + echo "No version provided" + exit 1 + fi + + - name: Build and publish + run: | + source .venv/bin/activate + poetry version $RELEASE_VERSION + make bake-and-publish + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + RELEASE_VERSION: ${{ steps.vars.outputs.tag }} diff --git a/.github/workflows/test-check-build.yml b/.github/workflows/test-check-build.yml new file mode 100644 index 0000000000000000000000000000000000000000..cbd89e6dbc8abf6c6484775372aaf62178afd2ad --- /dev/null +++ b/.github/workflows/test-check-build.yml @@ -0,0 +1,91 @@ +name: Test, check and build pipeline + +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +jobs: + quality: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + + - name: Run checks + run: make lint + + tests: + runs-on: ubuntu-latest + needs: quality + strategy: + matrix: + python-version: ["3.12"] + fail-fast: false + defaults: + run: + shell: bash + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + with: + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: poetry run pytest tests --cov --cov-config=pyproject.toml --cov-report=xml + + - name: Check typing + run: poetry run mypy + + - name: Upload coverage reports to Codecov with GitHub Action on Python 3.12 + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + if: ${{ matrix.python-version == '3.12' }} + + build: + runs-on: ubuntu-latest + needs: tests + strategy: + matrix: + python-version: ["3.12"] + fail-fast: false + defaults: + run: + shell: bash + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + with: + python-version: ${{ matrix.python-version }} + + - name: Build + run: make bake + + check-docs: + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + + - name: Check if documentation can be built + run: poetry run mkdocs build -s diff --git a/.github/workflows/version-bump-and-release.yml b/.github/workflows/version-bump-and-release.yml new file mode 100644 index 0000000000000000000000000000000000000000..cf1517b8db8687fc09836b91bb7dc689f56d7977 --- /dev/null +++ b/.github/workflows/version-bump-and-release.yml @@ -0,0 +1,62 @@ +name: Version Bump and Create Release + +on: + workflow_dispatch: + inputs: + bump_type: + description: "Type of version bump" + required: true + type: choice + options: + - patch + - minor + - major + +permissions: + contents: write + + +jobs: + bump-version: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up the environment + uses: ./.github/actions/setup-poetry-env + with: + python-version: "3.12" + + - name: Configure Git + run: | + git config --global user.name 'GitHub Actions' + git config --global user.email 'actions@github.com' + + - name: Bump Version + id: bump + run: | + poetry run bump-my-version bump ${{ github.event.inputs.bump_type }} + echo "NEW_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT + + - name: Push changes + run: | + git push + git push --tags + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + tag_name: ${{ steps.bump.outputs.NEW_TAG }} + + - name: Trigger Package Publish + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + event-type: package-release + client-payload: '{"version": "${{ steps.bump.outputs.NEW_TAG }}"}' + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..bf24ded996dffe6ae08b5bae6ddf7549cb92aded --- /dev/null +++ b/.gitignore @@ -0,0 +1,174 @@ +docs/source + +# From https://raw.githubusercontent.com/github/gitignore/main/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Vscode config files +# .vscode/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# artifect/ +dvc-service-account-key.json + + +# MacOS +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da89ef0c539d767cbddc511052caee6e84645efa --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.4.0" + hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: check-toml + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.5.2" + hooks: + - id: ruff + args: [--exit-non-zero-on-fix, --config=pyproject.toml] + - id: ruff-format + args: [--config=pyproject.toml] + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.0.3" + hooks: + - id: prettier + + - repo: https://github.com/psf/black + rev: 23.9.0 + hooks: + - id: black + + - repo: https://github.com/timothycrosley/isort + rev: 5.12.0 + hooks: + - id: isort + args: [--settings-path=pyproject.toml] + + - repo: https://github.com/PyCQA/pydocstyle + rev: 6.3.0 + hooks: + - id: pydocstyle + args: [--config=pyproject.toml] diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000000000000000000000000000000000000..753117c998fa0d74f5c1e8cf7900d0b537db5596 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,14 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: jio_savan_music_downloader", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/jio_savan_music_downloader/main.py", + "console": "integratedTerminal", + "justMyCode": true, + "jinja": true + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..64faac2fe9505f91b66b060cf9f885f338814114 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "files.exclude": { + "**/.mypy_cache": true, + "**/__pycache__": true, + "**/.pytest_cache": true, + "**/.ruff_cache": true, + "**/.venv": true, + "**/venv": true, + "**/node_modules": true, + "**/site": true, + "**/.coverage": true, + "**/coverage.xml": true, + "**/build": true, + "**/dist": true + } +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8a6d56fae713f2104827f9d42d100a4bbcb4a8f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# Builder Stage +FROM python:3.12-slim AS builder + +# Set environment variables for Poetry +ENV POETRY_VERSION=1.6.1 \ + POETRY_HOME="/opt/poetry" \ + PATH="/opt/poetry/bin:$PATH" + +# Install Poetry and necessary tools +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && curl -sSL https://install.python-poetry.org | python3 - \ + && apt-get remove -y curl && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy only the dependency files first to leverage Docker caching +COPY pyproject.toml poetry.lock /app/ + +# Install dependencies (only for building the wheel) +RUN poetry config virtualenvs.create false \ + && poetry install --no-root --only main + +# Copy the rest of the application code +COPY . /app + +# Build the wheel file +RUN poetry build -f wheel + +# Runtime Stage +FROM python:3.12-slim AS runtime + +# Set environment variables +ENV PYTHONUNBUFFERED=1 + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends libpq-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /app + +# Copy the built wheel file from the builder stage +COPY --from=builder /app/dist/*.whl /app/ + +# Install the wheel file +RUN pip install --no-cache-dir /app/*.whl + +# Expose application port +EXPOSE 80 + +# Command to run the application +CMD ["jio_savan_music_downloader"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6287a8875fc6af9260b5024da771f02224e1179f --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2025, Deepak pant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c804f5d3226cccb20a14ce492a0083359d4233af --- /dev/null +++ b/Makefile @@ -0,0 +1,247 @@ +# ============================= +# Variable Documentation +# ============================= + +# PYPI_TOKEN: Authentication token for PyPI publishing +# Usage: make publish PYPI_TOKEN=your_pypi_token + +# PACKAGE_NAME: Name of the Python package for dependency tree +# Usage: make print-dependency-tree PACKAGE_NAME=your_package_name + + +# ============================= +# Project Configuration +# ============================= +PROJECT_NAME = jio-savan-music-downloader +GITHUB_USERNAME = DeepakPant93 +GITHUB_REPO = $(PROJECT_NAME) +PROJECT_SLUG = jio_savan_music_downloader +CLOUD_REGION = eastus +TAG = latest +IMAGE_NAME = deepak93p/$(PROJECT_SLUG) +RESOURCE_GROUP = $(PROJECT_NAME)-rg +APP_NAME = $(PROJECT_NAME)-app +APP_ENV_NAME = $(APP_NAME)-env +BUMP_TYPE = patch + +# ============================= +# Help (Default Target) +# ============================= +.PHONY: help +help: ## Display this help message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}' + +.DEFAULT_GOAL := help + +# ============================= +# Installation and Setup +# ============================= +.PHONY: bake-env +bake-env: clean-env ## Install the poetry environment and set up pre-commit hooks + @echo "πŸš€ Creating virtual environment using pyenv and poetry" + @poetry install + @poetry run pre-commit install || true + @max_retries=3; count=0; \ + while ! make lint; do \ + count=$$((count + 1)); \ + if [ $$count -ge $$max_retries ]; then \ + echo "Max retries reached. Exiting."; \ + exit 1; \ + fi; \ + echo "Retrying make lint ($$count/$$max_retries)..."; \ + done + @poetry shell + +.PHONY: clean-env +clean-env: ## Remove the poetry environment + @echo "πŸš€ Removing virtual environment" + @rm -rf .venv + +.PHONY: reset-env +reset-env: clean-env bake-env ## Install the poetry environment and set up pre-commit hooks + +.PHONY: init-repo +init-repo: ## Initialize git repository + @echo "πŸš€ Initializing git repository" + @git init + @echo "πŸš€ Creating initial commit" + @git add . + @git commit -m "Initial commit" + @echo "πŸš€ Adding remote repository" + @git branch -M main + @git remote add origin git@github.com:$(GITHUB_USERNAME)/$(GITHUB_REPO).git + @echo "πŸš€ Pushing initial commit" + @git push -u origin main + +.PHONY: setup-cloud-env +setup-cloud-env: ## Create resource group, container app environment, and service principal + @echo "πŸš€ Creating resource group: $(RESOURCE_GROUP)" + @az group create --name $(RESOURCE_GROUP) --location $(CLOUD_REGION) + + @echo "πŸš€ Creating container app environment: $(APP_ENV_NAME)" + @az containerapp env create --name $(APP_ENV_NAME) --resource-group $(RESOURCE_GROUP) --location $(CLOUD_REGION) + + @echo "πŸš€ Fetching subscription ID" + @subscription_id=$$(az account show --query "id" -o tsv) && \ + echo "Subscription ID: $$subscription_id" && \ + echo "πŸš€ Creating service principal for: $(APP_NAME)" && \ + az ad sp create-for-rbac --name "$(APP_NAME)-service-principal" --role contributor --scopes /subscriptions/$$subscription_id --sdk-auth + + @echo "πŸš€ Creating container app: $(APP_NAME)" + @az containerapp create --name $(APP_NAME) --resource-group $(RESOURCE_GROUP) --environment $(APP_ENV_NAME) --image 'nginx:latest' --target-port 80 --ingress 'external' --query "properties.configuration.ingress.fqdn" + +.PHONY: clean-cloud-env +clean-cloud-env: ## Delete resource group, container app environment, and service principal + @echo "πŸš€ Deleting service principal for: $(APP_NAME)-service-principal" + @sp_object_id=$$(az ad sp list --display-name "$(APP_NAME)-service-principal" --query "[0].id" -o tsv) && \ + if [ -n "$$sp_object_id" ]; then \ + az ad sp delete --id $$sp_object_id; \ + echo "Service principal deleted"; \ + else \ + echo "Service principal not found, skipping deletion"; \ + fi + + @echo "πŸš€ Deleting container app: $(APP_NAME)" + @az containerapp delete --name $(APP_NAME) --resource-group $(RESOURCE_GROUP) --yes --no-wait || echo "Container app not found, skipping deletion" + + @echo "πŸš€ Deleting container app environment: $(APP_ENV_NAME)" + @az containerapp env delete --name $(APP_ENV_NAME) --resource-group $(RESOURCE_GROUP) --yes --no-wait || echo "Container app environment not found, skipping deletion" + + @echo "πŸš€ Deleting resource group: $(RESOURCE_GROUP)" + @az group delete --name $(RESOURCE_GROUP) --yes --no-wait || echo "Resource group not found, skipping deletion" + + +# ============================= +# Code Quality and Testing +# ============================= +.PHONY: lint +lint: ## Run code quality tools + @echo "πŸš€ Checking Poetry lock file consistency with 'pyproject.toml'" + @poetry check --lock + @echo "πŸš€ Linting code with pre-commit" + @poetry run pre-commit run -a + @echo "πŸš€ Static type checking with mypy" + @poetry run mypy + @echo "πŸš€ Checking for obsolete dependencies with deptry" + @poetry run deptry . + @echo "πŸš€ Checking for security vulnerabilities with bandit" + @poetry run bandit -c pyproject.toml -r jio_savan_music_downloader/ -ll + + +.PHONY: test +test: ## Run tests with pytest + @echo "πŸš€ Running tests with pytest" + @poetry run pytest --cov --cov-config=pyproject.toml --cov-report=term-missing + + +# ============================= +# Build and Release +# ============================= +.PHONY: bake +bake: clean-bake ## Build wheel file using poetry + @echo "πŸš€ Creating wheel file" + @poetry build + +.PHONY: clean-bake +clean-bake: ## Clean build artifacts + @rm -rf dist + +.PHONY: bump +bump: ## Bump project version + @echo "πŸš€ Bumping version" + @poetry run bump-my-version bump $(BUMP_TYPE) + +.PHONY: publish +publish: ## Publish a release to PyPI + @echo "πŸš€ Publishing: Dry run" + @poetry config pypi-token.pypi $(PYPI_TOKEN) + @poetry publish --dry-run + @echo "πŸš€ Publishing" + @poetry publish + +.PHONY: bake-and-publish +bake-and-publish: bake publish ## Build and publish to PyPI + +.PHONY: update +update: ## Update project dependencies + @echo "πŸš€ Updating project dependencies" + @poetry update + @poetry run pre-commit install --overwrite + @echo "Dependencies updated successfully" + +# ============================= +# Run and Documentation +# ============================= +.PHONY: run +run: ## Run the project's main application + @echo "πŸš€ Running the project" + @poetry run python $(PROJECT_SLUG)/main.py + +.PHONY: docs-test +docs-test: ## Test if documentation can be built without warnings or errors + @poetry run mkdocs build -s + +.PHONY: docs +docs: ## Build and serve the documentation + @poetry run mkdocs serve + +# ============================= +# Docker +# ============================= +.PHONY: bake-container +bake-container: ## Build Docker image + @echo "πŸš€ Building Docker image" + docker build -t $(IMAGE_NAME):$(TAG) -f Dockerfile . + +.PHONY: container-push +container-push: ## Push Docker image to Docker Hub + @echo "πŸš€ Pushing Docker image to Docker Hub" + docker push $(IMAGE_NAME):$(TAG) + +.PHONY: bake-container-and-push +bake-container-and-push: bake-container container-push ## Build and push Docker image to Docker Hub + +.PHONY: clean-container +clean-container: ## Clean up Docker resources related to the app + @echo "πŸš€ Deleting Docker image for app: $(IMAGE_NAME)" + @docker images $(IMAGE_NAME) --format "{{.Repository}}:{{.Tag}}" | xargs -r docker rmi -f || echo "No image to delete" + + @echo "πŸš€ Deleting unused Docker volumes" + @docker volume ls -qf dangling=true | xargs -r docker volume rm || echo "No unused volumes to delete" + + @echo "πŸš€ Deleting unused Docker networks" + @docker network ls -q --filter "dangling=true" | xargs -r docker network rm || echo "No unused networks to delete" + + @echo "πŸš€ Cleaning up stopped containers" + @docker ps -aq --filter "status=exited" | xargs -r docker rm || echo "No stopped containers to clean up" + + +# ============================= +# Debug +# ============================= + +.PHONY: print-dependency-tree +print-dependency-tree: ## Print dependency tree + @echo "Printing dependency tree..." + @poetry run pipdeptree -p $(PACKAGE_NAME) + + +# ============================= +# Cleanup +# ============================= +.PHONY: teardown +teardown: clean-bake clean-container ## Clean up temporary files and directories and destroy the virtual environment, Docker image from your local machine + @echo "πŸš€ Cleaning up temporary files and directories" + @rm -rf .pytest_cache || true + @rm -rf dist || true + @rm -rf build || true + @rm -rf htmlcov || true + @rm -rf .venv || true + @rm -rf .mypy_cache || true + @rm -rf site || true + @find . -type d -name "__pycache__" -exec rm -rf {} + || true + @rm -rf .ruff_cache || true + @echo "πŸš€ Clean up completed." + +.PHONY: teardown-all +teardown-all: teardown clean-cloud-env ## Clean up temporary files and directories and destroy the virtual environment, Docker image, and Cloud resources diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bf017e3aa04367a6452c74d6e174cc7a3816b33d --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# jio-savan-music-downloader + +This app will download Jio-Savan music. + +- **Github repository**: +- **Documentation** + +## Getting started with your project + +First, create a repository on GitHub with the same name as this project, and then run the following commands: + +## Installation + +1. Initialize the repository if it's your first time: + + ```bash + cd jio-savan-music-downloader + make init-repo + ``` + +2. Install dependencies using Poetry: + + ```bash + make bake-env + ``` + +3. Run the FastAPI server: + + ```bash + make run + ``` + +You are now ready to start development on your project! +The CI/CD pipeline will be triggered when you open a pull request, merge to main, or when you create a new release. diff --git a/artifacts/documents/.gitkeep b/artifacts/documents/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/artifacts/processed/.gitkeep b/artifacts/processed/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/codecov.yaml b/codecov.yaml new file mode 100644 index 0000000000000000000000000000000000000000..634e46d987a926644a7fc717e639ab07e1b34e85 --- /dev/null +++ b/codecov.yaml @@ -0,0 +1,9 @@ +coverage: + range: 70..100 + round: down + precision: 1 + status: + project: + default: + target: 90% + threshold: 0.5% diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..f8ca3cc312c9e40ce336351e63884213b1d67259 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,22 @@ +version: "3.9" + +services: + jio_savan_music_downloader: + build: + context: . + dockerfile: Dockerfile + container_name: jio_savan_music_downloader + ports: + - "80:80" + volumes: + - .:/app + environment: + - PYTHONUNBUFFERED=1 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:80/health" ] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + command: > + jio_savan_music_downloader diff --git a/docs/assets/favicon.ico b/docs/assets/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..bea0983a95863de94181df8a00c58f924df9d68c Binary files /dev/null and b/docs/assets/favicon.ico differ diff --git a/docs/assets/logo.png b/docs/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..d437d7304e9ec4a673234ade1879701027f16085 Binary files /dev/null and b/docs/assets/logo.png differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000000000000000000000000000000000..bb0ccca2705e494f965c8ad14300127c92b4fc58 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,87 @@ +# jio-savan-music-downloader + +[![Release](https://img.shields.io/github/v/release/DeepakPant93/jio-savan-music-downloader)](https://img.shields.io/github/v/release/DeepakPant93/jio-savan-music-downloader) +[![Build status](https://img.shields.io/github/actions/workflow/status/DeepakPant93/jio-savan-music-downloader/test-check-build.yml?branch=main)](https://github.com/DeepakPant93/jio-savan-music-downloader/actions/workflows/test-check-build.yml?query=branch%3Amain) +[![Commit activity](https://img.shields.io/github/commit-activity/m/DeepakPant93/jio-savan-music-downloader)](https://img.shields.io/github/commit-activity/m/DeepakPant93/jio-savan-music-downloader) +[![License](https://img.shields.io/github/license/DeepakPant93/jio-savan-music-downloader)](https://img.shields.io/github/license/DeepakPant93/jio-savan-music-downloader) + +This app will download Jio-Savan music. +This repository contains a sample Data Science application built with FastAPI, designed to streamline model training and prediction processes via RESTful APIs. The application leverages **Poetry** for dependency management, ensuring a robust and scalable development environment. + +--- + +## Features + +### FastAPI Endpoints: + +- `/upload-docs`: API endpoint to upload documents for creating embeddings. +- `/ask`: API endpoint for querying the system and receiving context-aware answers. + +### Poetry for Dependency Management: + +- Simplifies package installation and management. +- Ensures compatibility and reproducibility of the project environment. + +### Scalable Architecture: + +- Modular design with clear separation of concerns. +- Easy integration of new features or pipelines. + +--- + +## Prerequisites + +- Python >= 3.12 +- Poetry installed (`pip install poetry`) + +--- + +## Installation + +1. Clone the repository: + + ```bash + git clone https://github.com/DeepakPant93/jio-savan-music-downloader. + cd jio-savan-music-downloader + ``` +1. Initialize the repository if it's your first time: + + ```bash + cd jio-savan-music-downloader + make init-repo + ``` + +2. Install dependencies using Poetry: + + ```bash + make bake-env + ``` + +3. Run the FastAPI server: + + ```bash + make run + ``` + +--- + +## Project Structure + +```plaintext +──jio-savan-music-downloader/ + β”œβ”€β”€ api # API route definitions + β”œβ”€β”€ config # Configuration files and settings + β”œβ”€β”€ constants # Static constants and enumerations + β”œβ”€β”€ core # Core logic for the application + β”œβ”€β”€ entity # Definitions of data models and schemas + β”œβ”€β”€ exception # Custom exception classes for error handling + β”œβ”€β”€ logger # Logging setup for the application + β”œβ”€β”€ models # Request and response models + β”œβ”€β”€ services # Business logic and service layer + β”œβ”€β”€ utils # Utility functions (e.g., file handling, data encoding) + └── main.py # Entry point for the FastAPI application +``` + +--- + +Enjoy building with this RAG FastAPI application! πŸš€ diff --git a/docs/modules.md b/docs/modules.md new file mode 100644 index 0000000000000000000000000000000000000000..ecc31d892a12e2f35445568ccbcb02fa2ee07bc0 --- /dev/null +++ b/docs/modules.md @@ -0,0 +1,41 @@ +::: jio_savan_music_downloader + +## Configuration + +::: jio_savan_music_downloader.config + +## Core + +::: jio_savan_music_downloader.core + +## Constants + +::: jio_savan_music_downloader.constants + +## Logger + +::: jio_savan_music_downloader.logger + +## Utils + +::: jio_savan_music_downloader.utils + +## Exceptions + +::: jio_savan_music_downloader.exception + +## Entities + +::: jio_savan_music_downloader.entity + +## Models + +::: jio_savan_music_downloader.models + +## Services + +::: jio_savan_music_downloader.services + +## APIs + +::: jio_savan_music_downloader.api diff --git a/jio_savan_music_downloader/__init__.py b/jio_savan_music_downloader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9547ce7df4e5f7813163690a09f7d51154be4f23 --- /dev/null +++ b/jio_savan_music_downloader/__init__.py @@ -0,0 +1,28 @@ +""" + +This is the initialization file for the RAG (Retrieval-Augmented Generation) application, +designed to provide context-aware responses by combining document embeddings with large +language model (LLM) capabilities. The application is modular, scalable, and maintains +a clear separation of concerns across its components. + +Modules: + api: Exposes endpoints for document upload and querying the system. + config: Manages application settings and environment variables. + core: Implements embedding generation, LLM integration, and workflow orchestration. + models: Defines schemas for API request validation and response structuring. + service: Provides document management and vector database interaction services. + exception: Contains custom exceptions for handling application-specific errors. + utils: Offers utility functions for common operations and data manipulation. + logger: Implements centralized logging with customizable levels. + constants: Stores application-wide constants for consistency and maintainability. + +Features: + - **Retrieval-Augmented Generation**: Combines document embeddings with LLMs to deliver accurate, context-aware answers. + - **Modular Design**: Ensures scalability, maintainability, and ease of testing. + - **Error Handling and Logging**: Enhances debugging and monitoring with structured logs and custom exceptions. + - **Seamless Integration**: Connects document management, vector database, and LLM workflows efficiently. + - **User-Friendly API**: Simplifies user interaction with the application's core functionalities. + +This package serves as the backbone of the RAG application, ensuring a seamless pipeline +from document ingestion to intelligent query resolution. +""" diff --git a/jio_savan_music_downloader/api/__init__.py b/jio_savan_music_downloader/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dff866e582963bf3f0e2df835f52da03804c2972 --- /dev/null +++ b/jio_savan_music_downloader/api/__init__.py @@ -0,0 +1,15 @@ +""" + +This module contains API endpoints for a Retrieval-Augmented Generation (RAG) application. + +Endpoints: + - POST /upload-docs: This endpoint allows users to upload and manage documents for processing. + - POST /ask: This endpoint is used for querying the system and receiving context-aware answers based on uploaded documents. + +Features: + - **User-Friendly API**: Simplifies user interaction with the application's core functionalities. + - **Seamless Integration**: Connects document management, vector database, and LLM workflows efficiently. + +Purpose: + - Provides the interface for users to interact with the application's core RAG functionalities. +""" diff --git a/jio_savan_music_downloader/api/endpoint.py b/jio_savan_music_downloader/api/endpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..26bb79ec31fe9b6b953b63dfa4df1ccb33af9c1b --- /dev/null +++ b/jio_savan_music_downloader/api/endpoint.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter + +router = APIRouter() + + +@router.post("/upload-docs") +async def upload_docs() -> dict: + """ + Endpoint to upload documentation. This function encapsulates + the logic to upload the documentation and returns a success + message upon completion. + + Returns: + dict: A dictionary containing a success message. + """ + + return {"message": "Training Model Successful"} + + +@router.post("/ask") +async def ask() -> dict: + """ + Endpoint to handle prediction requests. This function processes + incoming requests for predictions and returns a success message + upon completion. + + Returns: + dict: A dictionary containing a success message. + """ + + return {"message": "Prediction Successful"} diff --git a/jio_savan_music_downloader/config/__init__.py b/jio_savan_music_downloader/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..992d6a9989204f4e4d8a30447dce675367e8f02f --- /dev/null +++ b/jio_savan_music_downloader/config/__init__.py @@ -0,0 +1,13 @@ +""" + +This module manages the configuration settings for the application. + +Purpose: + - Centralizes configuration management for environment variables, application settings, and dependencies. + - Ensures consistent and secure access to critical configurations. + +Features: + - Loads environment variables and provides access to them. + - Validates and parses configuration settings using Pydantic models. + - Supports dynamic updates for configuration changes during runtime (if applicable). +""" diff --git a/jio_savan_music_downloader/config/config.py b/jio_savan_music_downloader/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/constants/__init__.py b/jio_savan_music_downloader/constants/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3991c913042e35c35722102aed5cca97e294e2f3 --- /dev/null +++ b/jio_savan_music_downloader/constants/__init__.py @@ -0,0 +1,18 @@ +""" + +This package defines global constants used throughout the project. Constants +help in maintaining consistency and avoiding magic numbers or strings in the codebase. + +Usage: + Import the required constants as needed: + + Example: + ```python + from constants import APP_NAME, ENVIRONMENT + from constants import STATUS_OK, STATUS_BAD_REQUEST + ``` + +Purpose: + - Centralizes constant values for maintainability and reusability. + - Reduces hard-coded values in the project. +""" diff --git a/jio_savan_music_downloader/core/__init__.py b/jio_savan_music_downloader/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..799e3e340e178463f6b9daf1a6781c41aeffe6b3 --- /dev/null +++ b/jio_savan_music_downloader/core/__init__.py @@ -0,0 +1,12 @@ +""" + +This module serves as the core of the application, integrating essential services for the Retrieval-Augmented Generation (RAG) workflow. + +Modules: + EmbeddingService: Handles document embedding generation and retrieval for context-aware processing. + LLMService: Interfaces with large language models (LLMs) to generate answers to user queries. + Processor: Orchestrates workflows between the embedding and LLM services, ensuring seamless data flow and processing. + +Purpose: + - Provides the foundational logic and services required to power the application's RAG functionalities. +""" diff --git a/jio_savan_music_downloader/core/embedding_service.py b/jio_savan_music_downloader/core/embedding_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/core/llm_service.py b/jio_savan_music_downloader/core/llm_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/core/processor.py b/jio_savan_music_downloader/core/processor.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/entity/__init__.py b/jio_savan_music_downloader/entity/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f57621bd97792847a1a20616caeac94f16b75112 --- /dev/null +++ b/jio_savan_music_downloader/entity/__init__.py @@ -0,0 +1,16 @@ +""" + +This module defines the database entities used across the application for data storage and retrieval. + +Purpose: + - Provides structured definitions for database models. + - Facilitates interaction with the database by mapping application data to database schemas. + +Components: + - DB Entities: Defines entities that represent database tables or collections, including fields and relationships. + +Features: + - Centralized entity definitions for consistent database operations. + - Supports validation and serialization of database records. + - Designed to integrate seamlessly with the database layer, ensuring reliable data handling. +""" diff --git a/jio_savan_music_downloader/exception/__init__.py b/jio_savan_music_downloader/exception/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a5149e2e6585119dac7de3c8f90ed7b1a54d92be --- /dev/null +++ b/jio_savan_music_downloader/exception/__init__.py @@ -0,0 +1,61 @@ +""" + +This module defines custom exception classes and error-handling utilities tailored +to the needs of a data science pipeline. It helps standardize error reporting, improve +debugging, and provide meaningful feedback during model training, data preprocessing, +and prediction processes. + +Classes: + DataValidationError: Raised when input data fails validation checks. + ModelTrainingError: Raised during errors in the model training phase, such as convergence issues or invalid configurations. + PredictionError: Raised when the prediction pipeline encounters issues, such as missing features or incompatible input formats. + PipelineExecutionError: Raised for generic errors occurring during pipeline execution. + +Usage: + Import and use the exceptions in various stages of the data science pipeline: + + Example: + ```python + from exception import DataValidationError, ModelTrainingError + + try: + validate_data(input_data) + except DataValidationError as e: + logger.error(f"Data validation failed: {e}") + raise + ``` + +Features: + - Custom exceptions for specific pipeline stages, ensuring meaningful error reporting. + - Enables targeted exception handling, reducing debugging time. + - Provides a consistent structure for error messages across the project. + +Purpose: + - To define project-specific exceptions for common error scenarios in the pipeline. + - To improve the robustness and reliability of the pipeline by enabling clear error handling. + - To make the debugging process more intuitive by raising descriptive errors. + +Examples: + - **Data Validation**: Raise a `DataValidationError` if the input data schema is incorrect or missing required fields. + - **Model Training**: Raise a `ModelTrainingError` if the model fails to converge due to invalid hyperparameters. + - **Prediction**: Raise a `PredictionError` when incompatible input data is passed to the model. + +Additional Notes: + - Use these exceptions in conjunction with logging to provide detailed error information. + - Ensure that custom exceptions are raised with meaningful messages to assist in debugging and error resolution. +""" + +from fastapi import HTTPException + + +class CustomException(HTTPException): + """Custom exception class for handling errors in the data science pipeline.""" + + def __init__(self, status_code: int, detail: str): + """ + Custom exception for handling API errors. + + :param status_code: The HTTP status code to return. + :param detail: A string describing the error in detail. + """ + super().__init__(status_code=status_code, detail=detail) diff --git a/jio_savan_music_downloader/logger/__init__.py b/jio_savan_music_downloader/logger/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3c5e82f300794fafc68bdc6a898c8ea5435311a4 --- /dev/null +++ b/jio_savan_music_downloader/logger/__init__.py @@ -0,0 +1,61 @@ +""" + +This module provides centralized logging utilities for the data science pipeline. +It standardizes logging practices, ensures consistency across components, and facilitates +easy debugging and monitoring of the pipeline's execution, including data preprocessing, +model training, evaluation, and predictions. + +Functions: + setup_logging: Configures the logging system, including log format, level, and output destinations. + get_logger: Returns a logger instance for a specific module or stage of the pipeline. + +Features: + - Centralized logging configuration to maintain consistency. + - Support for different log levels (INFO, DEBUG, WARNING, ERROR, CRITICAL). + - Ability to write logs to files, console, or external monitoring systems. + - Timestamped log entries for accurate tracking of events. + - Integration with custom exception handling for detailed error reporting. + +Usage: + Use this module to log messages in a standardized manner across the project: + + Example: + ```python + from src.logging import logger + + logger.info("Starting the model training process...") + logger.error("An error occurred during data validation.") + ``` + +Purpose: + - To provide a standardized mechanism for logging messages throughout the data science pipeline. + - To assist in debugging by capturing detailed logs of each pipeline stage. + - To enable seamless integration with monitoring and alerting systems. + +Best Practices: + - Use appropriate log levels to categorize messages (e.g., DEBUG for detailed information, ERROR for issues). + - Ensure logs include sufficient context, such as function names or input details, to aid debugging. + - Regularly monitor log files for anomalies or errors in the pipeline. + +Additional Notes: + - The `setup_logging` function can be configured to write logs to multiple destinations, such as files or cloud services. + - The module can be extended to integrate with third-party monitoring tools like Elasticsearch, Splunk, or Datadog. +""" + +import logging +import os +import sys + +logging_str = "[%(asctime)s: %(levelname)s: %(module)s: %(message)s]" +log_dir = "logs" +log_filepath = os.path.join(log_dir,"jio-savan-music-downloader.log") +os.makedirs(log_dir, exist_ok=True) + + +logging.basicConfig( + level=logging.INFO, + format=logging_str, + handlers=[logging.FileHandler(log_filepath), logging.StreamHandler(sys.stdout)], +) + +logger = logging.getLogger("jio-savan-music-downloader-logger") diff --git a/jio_savan_music_downloader/main.py b/jio_savan_music_downloader/main.py new file mode 100644 index 0000000000000000000000000000000000000000..416f947d58ea8a5f8227cf7dba1085ae3a52738a --- /dev/null +++ b/jio_savan_music_downloader/main.py @@ -0,0 +1,92 @@ +from fastapi import FastAPI, responses +from fastapi.openapi.utils import get_openapi +from fastapi_health import health + +from jio_savan_music_downloader.api.endpoint import router + +__version__ = "0.0.1" + +app = FastAPI( + title="jio-savan-music-downloader APIs", + description="This app will download Jio-Savan music.", + version=__version__, + docs_url="/docs", + redoc_url="/redoc", +) + + +app = FastAPI() + +@app.get("/", include_in_schema=False) +async def root() -> responses.RedirectResponse: + """ + Redirects the root URL to the API documentation page. + + Returns: + RedirectResponse: A response object that redirects the client to the "/docs" URL. + """ + + return responses.RedirectResponse("/docs") + + +# Health Check +async def health_check() -> dict: + """ + Checks the health of the API. + + This endpoint checks the health of the API and returns a simple status + message. It is intended to be used by load balancers or other monitoring + systems to determine if the API is functional. + + Returns: + dict: A dictionary containing the status of the API. + """ + return {"status": "healthy"} + + +# Include routers +app.add_api_route( + "/health", + health([health_check]), + tags=["Management"], + description="Management APIs", +) +app.include_router(router, prefix="/api/v1", tags=["Operations"]) + + +def _custom_openapi() -> dict: + if app.openapi_schema: + return app.openapi_schema + openapi_schema = get_openapi( + title="jio-savan-music-downloader APIs", + description="This app will download Jio-Savan music.", + version=__version__, + routes=app.routes, + ) + app.openapi_schema = openapi_schema + return app.openapi_schema + + +app.openapi = _custom_openapi + + +def main() -> None: + """ + The main entry point of the application. + + This function starts the FastAPI server using Uvicorn. It serves the API + on the specified host and port. The function is intended to be run + directly when the script is executed. + + Notes: + - The 'nosec B104' comment is used to suppress a security warning + related to binding to all network interfaces. + """ + + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=80) # nosec B104 + + +if __name__ == "__main__": + main() diff --git a/jio_savan_music_downloader/models/__init__.py b/jio_savan_music_downloader/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..83a395a744121252b358e48952e54ada0d460f07 --- /dev/null +++ b/jio_savan_music_downloader/models/__init__.py @@ -0,0 +1,11 @@ +""" + +This module defines the data models used for API request and response handling. + +Components: + - Request Models: Defines the structure and validation rules for incoming API requests. + - Response Models: Specifies the format and schema for outgoing API responses. + +Purpose: + - Ensures consistent data validation and serialization across the application, adhering to defined schemas. +""" diff --git a/jio_savan_music_downloader/models/request_models.py b/jio_savan_music_downloader/models/request_models.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/models/response_models.py b/jio_savan_music_downloader/models/response_models.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/services/__init__.py b/jio_savan_music_downloader/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5aef3d2cb5b1c41473c47cc47b67b734418cb9db --- /dev/null +++ b/jio_savan_music_downloader/services/__init__.py @@ -0,0 +1,11 @@ +""" + +This module provides the core services for managing documents and vector database operations. + +Components: + - Document Service: Handles document-related operations such as storage, retrieval, and preprocessing. + - Vector DB Service: Manages interactions with the vector database, including storing and querying embeddings. + +Purpose: + - Implements the business logic and service layer to support the application's RAG functionality. +""" diff --git a/jio_savan_music_downloader/services/document_service.py b/jio_savan_music_downloader/services/document_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/services/vector_db_service.py b/jio_savan_music_downloader/services/vector_db_service.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jio_savan_music_downloader/utils/__init__.py b/jio_savan_music_downloader/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d56e993110c3f6d683b8a33fce01cba61d7e4f1 --- /dev/null +++ b/jio_savan_music_downloader/utils/__init__.py @@ -0,0 +1,15 @@ +""" + +The `utils` module provides various utility functions for file I/O, data encoding/decoding, and directory management. + +Functions: + read_yaml: Reads a YAML file and returns its contents as a dictionary. + create_directories: Creates directories if they do not exist. + save_json: Saves data to a JSON file. + load_json: Loads JSON data from a file. + save_bin: Saves binary data to a file. + load_bin: Loads binary data from a file. + get_size: Returns the size of a file or directory in bytes. + decode_image: Decodes an image from a base64 string. + encode_image_into_base64: Encodes an image into a base64 string. +""" diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000000000000000000000000000000000..99b7f71c0f6fc4bb5826310e7155088c7cf25e90 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,89 @@ +site_name: jio-savan-music-downloader +repo_url: https://github.com/DeepakPant93/jio-savan-music-downloader +site_url: https://DeepakPant93.github.io/jio-savan-music-downloader +site_description: This app will download Jio-Savan music. +site_author: Deepak pant +edit_uri: edit/main/docs/ +repo_name: DeepakPant93/jio-savan-music-downloader +copyright: Maintained by Deepak pant. + +nav: + - Home: index.md + - Modules: modules.md + +plugins: + - search + - tags + - mkdocstrings: + handlers: + python: + setup_commands: + show_source: false + show_root_heading: true + show_root_full_path: false + heading_level: 3 + enable_inventory: true + # - import sys + # - sys.path.append('../') + # - git-revision-date-localized: # Shows last updated date + # enable_creation_date: true + - minify: # Minifies HTML and JS + minify_html: true + minify_js: true + +theme: + name: material + logo: assets/logo.png # Optional custom logo + favicon: assets/favicon.ico # Optional favicon + features: + - navigation.tabs # Top-level sections as tabs + - navigation.sections # Sections are expanded + - navigation.top # Back to top button + - search.suggest # Search suggestions + - search.highlight # Highlight search results + - content.tabs.link # Link code tabs + - content.code.annotate # Code block annotations + - content.copy.code # Copy code button + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: deep orange + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: deep orange + toggle: + icon: material/brightness-4 + name: Switch to light mode + icon: + repo: fontawesome/brands/github + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/DeepakPant93/jio-savan-music-downloader + - icon: fontawesome/brands/python + link: https://pypi.org/project/jio-savan-music-downloader + # analytics: # Optional Google Analytics + # provider: google + # property: G-XXXXXXXXXX # Replace with your tracking ID + +markdown_extensions: + - toc: + permalink: true + - pymdownx.arithmatex: + generic: true + - pymdownx.highlight: # Advanced code highlighting + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed + - attr_list + - md_in_html diff --git a/notebooks/trails.ipynb b/notebooks/trails.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 0000000000000000000000000000000000000000..ab1033bd37224ee84b5862fb25f094db73809b74 --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..a84b12f0495f095d2e571f62a6c899fb5b53ecd4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,212 @@ +[tool.poetry] +name = "jio_savan_music_downloader" +version = "0.0.1" +description = "This app will download Jio-Savan music." +authors = ["Deepak pant "] +repository = "https://github.com/DeepakPant93/jio-savan-music-downloader" +documentation = "https://DeepakPant93.github.io/jio-savan-music-downloader/" +readme = "README.md" +packages = [ + {include = "jio_savan_music_downloader"} +] + +[tool.poetry.scripts] +jio_savan_music_downloader = "jio_savan_music_downloader.main:main" + +[tool.poetry.dependencies] +python = ">=3.11,<4.0" +fastapi = "~0.100.0" # Latest compatible version +uvicorn = "^0.23.0" # For running FastAPI apps +fastapi-health = "^0.4.0" # Health check for FastAPI +#pyyaml = "^6.0.2" +#python-box = "^7.2.0" +#ensure = "^1.0.4" +#joblib = "^1.3.2" # Parallel processing and caching +#python-dotenv = "^1.0.0" # Manage environment variables +#PyPDF2 = "^3.0.0" # For PDF manipulation +#pydantic = "^2.0.0" # Data validation and settings management + + + +[tool.poetry.group.dev.dependencies] +deptry = "^0.16.2" # For dependency management +mypy = "^1.5.1" # Static type checking +pre-commit = "^3.4.0" # Pre-commit hooks +tox = "^4.11.1" # Testing in multiple environments +ipykernel = "^6.25.0" # Jupyter kernel +black = "^23.9.0" # Code formatter +build = "^1.0.0" # Build management +bump-my-version = "^0.28.2" # Bump versions automatically +codespell = "^2.2.5" # Spell checking in code +wheel = "^0.41.0" # Build wheels +twine = "^4.0.0" # Publish packages +bandit = "^1.8.0" # Security check +pylint = "^3.0.0" # Powerful linter +pydocstyle = "^6.3.0" # Enforce PEP 257 docstring conventions +isort = "^5.12.0" # Sort imports + +[tool.poetry.group.docs.dependencies] +mkdocs = "^1.5.0" # Documentation site generator +sphinx = "^7.2.0" # Documentation tool +mkdocs-git-revision-date-plugin = "^0.3.2" # Show revision dates +mkdocs-git-revision-date-localized-plugin = "^1.3.0" # Localized dates +mkdocs-jupyter = ">=0.25.1" # For Jupyter Notebook integration +mkdocs-pdf-export-plugin = "^0.5.10" # PDF export +mkdocs-material = ">=9.1.3" # MkDocs Material theme +mkdocstrings-crystal = "^0.3.0" # Mkdocstrings for Crystal +pygments = "^2.16.0" # Syntax highlighting +pymdown-extensions = "^10.0" # Markdown extensions +nbconvert = "^7.7.0" # Convert notebooks to other formats +nbformat = "^5.9.0" # Notebook format support +livereload = "^2.6.3" # Live reload for MkDocs +watchdog = "^3.0.0" # File monitoring +mkdocstrings = {extras = ["python"], version = "^0.27.0"} # Auto-generate documentation from docstrings +mkdocs-minify-plugin = "^0.8.0" # Minify HTML + +[tool.poetry.group.test.dependencies] +pytest-mock = "^3.11.0" # Mocking in tests +factory-boy = "^3.3.1" # Test data generation +pytest-asyncio = "^0.21.0" # Async testing support +pytest-xdist = "^3.3.1" # Parallel test execution +freezegun = "^1.2.0" # Mock datetime +pytest = "^7.2.0" # Testing framework +allure-pytest = "^2.13.0" # Reporting for pytest +pytest-sugar = "^0.9.7" # Better test output +pytest-cov = "^4.0.0" # Test coverage reports +httpx = "^0.24.0" +pytest-runner = "^6.0.0" # Running tests via `python setup.py test` + + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + + +[tool.mypy] +files = ["jio_savan_music_downloader"] +disallow_untyped_defs = true +disallow_any_unimported = true +no_implicit_optional = true +check_untyped_defs = true +warn_return_any = true +warn_unused_ignores = true +show_error_codes = true +pretty = true +show_traceback = true + + +[[tool.mypy.overrides]] +module = [ + "joblib.*", + "yaml.*", + "ensure.*", + "fastapi_health.*", + "jio_savan_music_downloader.main" +] +ignore_missing_imports = true +ignore_errors = true + + +[tool.pytest.ini_options] +testpaths = ["tests"] +norecursedirs = "legacy_tests" +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +filterwarnings = [ + "ignore:.*general_plain_validator_function.*:DeprecationWarning", + "ignore:.*with_info_plain_validator_function.*:DeprecationWarning" +] + + +[tool.ruff] +target-version = "py39" +line-length = 120 +fix = true +select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # isort + "I", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", + # tryceratops + "TRY", +] +ignore = [ + # LineTooLong + "E501", + # DoNotAssignLambda + "E731", + # Possible binding to all interfaces - Require for Docker container + "S104" +] + +[tool.ruff.format] +preview = true + +[tool.coverage.report] +skip_empty = true + + +[tool.coverage.run] +branch = true +source = ["jio_savan_music_downloader"] +# parallel = true +# concurrency = ["thread"] +omit = [ + "**/__init__.py", # Exclude all init files + "jio_savan_music_downloader/main.py", # Exclude main.py file + "jio_savan_music_downloader/constants/*", # Exclude all files in a constants folder + "jio_savan_music_downloader/exception/*", # Exclude all files in a exception folder + "jio_savan_music_downloader/logger/*", # Exclude all files in a logger folder + "jio_savan_music_downloader/entity/*", # Exclude all files in entity folder + "jio_savan_music_downloader/config/*", # Exclude all files in config folder + "jio_savan_music_downloader/models/*", # Exclude all files in model folder +] + +[tool.ruff.per-file-ignores] +"tests/*" = ["S101"] + +[tool.bumpversion] +current_version = "0.0.1" +commit = true +tag = true + +[[tool.bumpversion.files]] +glob = "pyproject.toml" +search = 'version = "{current_version}"' +replace = 'version = "{new_version}"' + +[[tool.bumpversion.files]] +glob = "jio_savan_music_downloader/main.py" +search = '__version__ = "{current_version}"' +replace = '__version__ = "{new_version}"' + +[tool.deptry] +exclude = ["research","artifacts", "notebooks", "tests", "docs", ".venv", "venv", "__pycache__", ".ruff_cache", ".pytest_cache", ".mypy_cache", ".coverage", ".git", "build", "dist", ".github", "site", "config"] + +[tool.pydocstyle] +select = ["D101", "D102"] \ No newline at end of file diff --git a/tests/test_api_endpoint.py b/tests/test_api_endpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..cc54adc1f276404e1518ab3ee5eb71f4ccb2fb1a --- /dev/null +++ b/tests/test_api_endpoint.py @@ -0,0 +1,15 @@ +from starlette.testclient import TestClient +from jio_savan_music_downloader.main import app + +client = TestClient(app=app) + +def test_train_model(): + response = client.post("/api/v1/upload-docs") + assert response.status_code == 200 + assert response.json() is not None + + +def test_predict(): + response = client.post("/api/v1/ask") + assert response.status_code == 200 + assert response.json() is not None diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000000000000000000000000000000000..21e9ef5cb95ac3deb31937ac831e890ca0842cf7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,16 @@ +[tox] +skipsdist = true +envlist = py311, py312 + +[gh-actions] +python = + 3.11: py311 + 3.12: py312 + +[testenv] +passenv = PYTHON_VERSION +allowlist_externals = poetry +commands = + poetry install -v + pytest --doctest-modules tests --cov --cov-config=pyproject.toml --cov-report=xml + mypy