Complete agentic code

.devcontainer/devcontainer.json

@@ -47,7 +47,7 @@
                 "python.linting.pylintEnabled": true
             }
         }
-    }//,
+    } //,
     // "hostRequirements": {
     //     "cpus": 2,
     //     "memory": "4gb",

.dockerignore

@@ -0,0 +1,71 @@
+# Python cache files
+**/__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Poetry-related files
+.poetry/
+
+# Virtual environments
+**/.venv/
+**/venv/
+
+# Node.js dependencies
+**/node_modules/
+
+# Mypy cache
+**/.mypy_cache/
+
+# Pytest cache
+**/.pytest_cache/
+
+# Ruff cache
+**/.ruff_cache/
+
+# Coverage files
+**/.coverage
+**/coverage.xml
+
+# Development tools and IDE-specific files
+*.idea/
+*.vscode/
+**/.vscode/
+.devcontainer/
+.github/
+
+# Exclude notebooks
+notebooks/
+
+# Exclude test-related files and directories
+tests/
+
+# Build and distribution artifacts
+**/build/
+**/dist/
+*.egg-info/
+.eggs/
+
+# Logs and temporary files
+*.log
+*.tmp
+*.swp
+*.swo
+
+# Site-related files (if applicable)
+**/site/
+
+# Sensitive or configuration files
+.env
+*.pem
+*.key
+
+# Markdown and configuration files
+mkdocs.yml
+tox.ini
+pre-commit-config.yml
+codecove.yml
+.gitignore
+
+# Docker-related files (optional if Docker Compose is used locally but not in the image)
+docker-compose.yml

.github/workflows/build-and-deploy.yml

@@ -1,55 +0,0 @@
-name: Build and Deploy Docker Image
-
-on:
-  workflow_dispatch: {}
-  release:
-      types: [published]
-      branches: [main]
-  repository_dispatch:
-      types: [package-release]
-
-env:
-    DOCKER_IMAGE: deepak93p/jio_savan_music_downloader # Replace with your DockerHub image
-    AZURE_WEBAPP_NAME: jio-savan-music-downloader-app # Replace with your Azure Web App name
-    AZURE_WEBAPP_RG: jio-savan-music-downloader-rg # Replace with your Azure Web App resource group
-
-jobs:
-    build-push-docker-image:
-        runs-on: ubuntu-latest
-
-        steps:
-            - name: Checkout repository
-              uses: actions/checkout@v4
-
-            - name: Login to Docker Hub
-              env:
-                  DOCKER_USER: deepak93p
-                  DOCKER_PWD: ${{ secrets.DOCKERHUB_PUSH_TOKEN }}
-              run: echo $DOCKER_PWD | docker login -u $DOCKER_USER --password-stdin
-
-            - name: Build and Push Docker Image
-              run: make bake-container-and-push IMAGE=${{ env.DOCKER_IMAGE }} TAG=${{ github.sha }}
-
-            - name: Clean up Docker system
-              run: docker system prune -f
-
-    deploy:
-        runs-on: ubuntu-latest
-        needs: build-push-docker-image
-        environment:
-            name: "production"
-
-        steps:
-            - name: Azure Login
-              uses: azure/login@v1
-              with:
-                  creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-            - name: Deploy to Azure Container Apps
-              run: |
-                  az containerapp update \
-                    --name ${{ env.AZURE_WEBAPP_NAME }} \
-                    --resource-group ${{ env.AZURE_WEBAPP_RG }} \
-                    --image index.docker.io/${{ env.DOCKER_IMAGE }}:${{ github.sha }} \
-                    --query "properties.configuration.ingress.fqdn" \
-                    -o tsv

.github/workflows/deploy-docs.yml

@@ -1,22 +1,22 @@
-name: Deploy Documentation
+# name: Deploy Documentation
 
-on:
-  release:
-      types: [published]
-      branches: [main]
-  repository_dispatch:
-      types: [package-release]
+# on:
+#     release:
+#         types: [published]
+#         branches: [main]
+#     repository_dispatch:
+#         types: [package-release]
 
-jobs:
-    deploy-docs:
-        runs-on: ubuntu-latest
+# jobs:
+#     deploy-docs:
+#         runs-on: ubuntu-latest
 
-        steps:
-            - name: Check out
-              uses: actions/checkout@v4
+#         steps:
+#             - name: Check out
+#               uses: actions/checkout@v4
 
-            - name: Set up the environment
-              uses: ./.github/actions/setup-poetry-env
+#             - name: Set up the environment
+#               uses: ./.github/actions/setup-poetry-env
 
-            - name: Deploy documentation
-              run: poetry run mkdocs gh-deploy --force
+#             - name: Deploy documentation
+#               run: poetry run mkdocs gh-deploy --force

.github/workflows/hugging_face-deploy.yml

@@ -0,0 +1,27 @@
+name: Deploy to Hugging Face Spaces
+
+on:
+    push:
+        branches:
+            - main # or master, depending on your default branch
+
+jobs:
+    deploy:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+              with:
+                  fetch-depth: 0
+
+            - name: Configure Git
+              run: |
+                  git config --global user.email "github-actions[bot]@users.noreply.github.com"
+                  git config --global user.name "github-actions[bot]"
+
+            - name: Push to Hugging Face Space
+              env:
+                  HF_TOKEN: ${{ secrets.HF_TOKEN }}
+              run: |
+                  git remote add space https://USER:[email protected]/spaces/deepakpant/jio_savan_music_downloader
+                  git push --force space main

.github/workflows/publish-package.yml

@@ -1,41 +0,0 @@
-name: Publish Package
-
-on:
-  release:
-      types: [published]
-      branches: [main]
-  repository_dispatch:
-      types: [package-release]
-
-
-jobs:
-    publish-to-pypi:
-        runs-on: ubuntu-latest
-
-        steps:
-            - name: Check out
-              uses: actions/checkout@v4
-
-            - name: Set up the environment
-              uses: ./.github/actions/setup-poetry-env
-
-            - name: Export tag
-              id: vars
-              run: |
-                  if [ "${{ github.event.client_payload.version }}" != "" ]; then
-                    echo "tag=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT
-                  elif [ "${{ github.event.inputs.version }}" != "" ]; then
-                    echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
-                  else
-                    echo "No version provided"
-                    exit 1
-                  fi
-
-            - name: Build and publish
-              run: |
-                  source .venv/bin/activate
-                  poetry version $RELEASE_VERSION
-                  make bake-and-publish
-              env:
-                  PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-                  RELEASE_VERSION: ${{ steps.vars.outputs.tag }}

.github/workflows/test-check-build.yml

@@ -25,40 +25,9 @@ jobs:
             - name: Run checks
               run: make lint
 
-    tests:
-        runs-on: ubuntu-latest
-        needs: quality
-        strategy:
-            matrix:
-                python-version: ["3.12"]
-            fail-fast: false
-        defaults:
-            run:
-                shell: bash
-        steps:
-            - name: Check out
-              uses: actions/checkout@v4
-
-            - name: Set up the environment
-              uses: ./.github/actions/setup-poetry-env
-              with:
-                  python-version: ${{ matrix.python-version }}
-
-            - name: Run tests
-              run: poetry run pytest tests --cov --cov-config=pyproject.toml --cov-report=xml
-
-            - name: Check typing
-              run: poetry run mypy
-
-            - name: Upload coverage reports to Codecov with GitHub Action on Python 3.12
-              uses: codecov/codecov-action@v5
-              with:
-                  token: ${{ secrets.CODECOV_TOKEN }}
-              if: ${{ matrix.python-version == '3.12' }}
-
     build:
         runs-on: ubuntu-latest
-        needs: tests
+        needs: [quality]
         strategy:
             matrix:
                 python-version: ["3.12"]

.github/workflows/version-bump-and-release.yml

@@ -15,7 +15,6 @@ on:
 permissions:
     contents: write
 
-
 jobs:
     bump-version:
         runs-on: ubuntu-latest
@@ -59,4 +58,3 @@ jobs:
                   token: ${{ secrets.GITHUB_TOKEN }}
                   event-type: package-release
                   client-payload: '{"version": "${{ steps.bump.outputs.NEW_TAG }}"}'
-

.pre-commit-config.yaml

@@ -22,16 +22,16 @@ repos:
       hooks:
           - id: prettier
 
-    - repo: https://github.com/psf/black
-      rev: 23.9.0
-      hooks:
-          - id: black
+    # - repo: https://github.com/psf/black
+    #   rev: 23.9.0
+    #   hooks:
+    #       - id: black
 
-    - repo: https://github.com/timothycrosley/isort
-      rev: 5.12.0
-      hooks:
-          - id: isort
-            args: [--settings-path=pyproject.toml]
+    # - repo: https://github.com/timothycrosley/isort
+    #   rev: 5.12.0
+    #   hooks:
+    #       - id: isort
+    #         args: [--settings-path=pyproject.toml]
 
     - repo: https://github.com/PyCQA/pydocstyle
       rev: 6.3.0

.vscode/launch.json

@@ -5,7 +5,8 @@
             "name": "Python Debugger: jio_savan_music_downloader",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/jio_savan_music_downloader/main.py",
+            "program": "${workspaceFolder}/jio_savan_music_downloader/app.py",
+            "args": ["run"],
             "console": "integratedTerminal",
             "justMyCode": true,
             "jinja": true

Dockerfile

@@ -1,6 +1,8 @@
 # Builder Stage
 FROM python:3.12-slim AS builder
 
+RUN useradd -m -u 1000 user
+
 # Set environment variables for Poetry
 ENV POETRY_VERSION=1.6.1 \
     POETRY_HOME="/opt/poetry" \
@@ -15,14 +17,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl \
 WORKDIR /app
 
 # Copy only the dependency files first to leverage Docker caching
-COPY pyproject.toml poetry.lock /app/
+COPY --chown=user pyproject.toml poetry.lock /app/
 
 # Install dependencies (only for building the wheel)
 RUN poetry config virtualenvs.create false \
     && poetry install --no-root --only main
 
 # Copy the rest of the application code
-COPY . /app
+COPY --chown=user . /app
 
 # Build the wheel file
 RUN poetry build -f wheel
@@ -30,24 +32,38 @@ RUN poetry build -f wheel
 # Runtime Stage
 FROM python:3.12-slim AS runtime
 
+RUN useradd -m -u 1000 user
+
 # Set environment variables
 ENV PYTHONUNBUFFERED=1
 
-# Install runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends libpq-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+# Install runtime dependencies for Chromium and Selenium
+RUN apt-get update && apt-get install -y \
+    chromium \
+    chromium-driver \
+    gnupg \
+    libgconf-2-4 \
+    libnss3 \
+    unzip \
+    wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set environment variables for Selenium to use Chromium
+ENV CHROME_BIN=/usr/bin/chromium
+ENV CHROMEDRIVER_PATH=/usr/bin/chromedriver
 
 # Set working directory
 WORKDIR /app
 
 # Copy the built wheel file from the builder stage
-COPY --from=builder /app/dist/*.whl /app/
+COPY --chown=user --from=builder /app/dist/*.whl /app/
+
 
 # Install the wheel file
 RUN pip install --no-cache-dir /app/*.whl
 
 # Expose application port
-EXPOSE 80
+EXPOSE 7860
 
 # Command to run the application
 CMD ["jio_savan_music_downloader"]

LICENSE

@@ -19,4 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-

Makefile

@@ -121,6 +121,10 @@ lint: ## Run code quality tools
 	@echo "🚀 Linting code with pre-commit"
 	@poetry run pre-commit run -a
 	@echo "🚀 Static type checking with mypy"
+	# @echo "🚀 Sorting imports with isort"
+	# @poetry run isort jio_savan_music_downloader/
+	# @echo "🚀 Linting code with Ruff"
+	# @poetry run ruff format jio_savan_music_downloader/
 	@poetry run mypy
 	@echo "🚀 Checking for obsolete dependencies with deptry"
 	@poetry run deptry .
@@ -175,7 +179,7 @@ update: ## Update project dependencies
 .PHONY: run
 run: ## Run the project's main application
 	@echo "🚀 Running the project"
-	@poetry run python $(PROJECT_SLUG)/main.py
+	@poetry run streamlit run $(PROJECT_SLUG)/app.py
 
 .PHONY: docs-test
 docs-test: ## Test if documentation can be built without warnings or errors

README.md

@@ -1,9 +1,19 @@
+---
+title: Free Music Downloader
+emoji: 🎵
+colorFrom: blue
+colorTo: purple
+sdk: docker
+app_port: 7860
+short_description: AI service for searching and downloading the free music.
+---
+
 # jio-savan-music-downloader
 
 This app will download Jio-Savan music.
 
-- **Github repository**: <https://github.com/DeepakPant93/jio-savan-music-downloader/>
-- **Documentation** <https://DeepakPant93.github.io/jio-savan-music-downloader/>
+-   **Github repository**: <https://github.com/DeepakPant93/jio-savan-music-downloader/>
+-   **Documentation** <https://DeepakPant93.github.io/jio-savan-music-downloader/>
 
 ## Getting started with your project
 

docker-compose.yml

@@ -1,22 +1,16 @@
 version: "3.9"
 
 services:
-  jio_savan_music_downloader:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    container_name: jio_savan_music_downloader
-    ports:
-      - "80:80"
-    volumes:
-      - .:/app
-    environment:
-      - PYTHONUNBUFFERED=1
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:80/health" ]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 10s
-    command: >
-      jio_savan_music_downloader
+    jio_savan_music_downloader:
+        build:
+            context: .
+            dockerfile: Dockerfile
+        container_name: jio_savan_music_downloader
+        ports:
+            - "7860:7860"
+        volumes:
+            - .:/app
+        environment:
+            - PYTHONUNBUFFERED=1
+        command: >
+            jio_savan_music_downloader

docs/index.md

@@ -44,6 +44,7 @@ This repository contains a sample Data Science application built with FastAPI, d
     git clone https://github.com/DeepakPant93/jio-savan-music-downloader.
     cd jio-savan-music-downloader
     ```
+
 1. Initialize the repository if it's your first time:
 
     ```bash
@@ -51,13 +52,13 @@ This repository contains a sample Data Science application built with FastAPI, d
     make init-repo
     ```
 
-2. Install dependencies using Poetry:
+1. Install dependencies using Poetry:
 
     ```bash
     make bake-env
     ```
 
-3. Run the FastAPI server:
+1. Run the FastAPI server:
 
     ```bash
     make run

docs/modules.md

@@ -1,33 +1,5 @@
 ::: jio_savan_music_downloader
 
-## Configuration
-
-::: jio_savan_music_downloader.config
-
-## Core
-
-::: jio_savan_music_downloader.core
-
-## Constants
-
-::: jio_savan_music_downloader.constants
-
-## Logger
-
-::: jio_savan_music_downloader.logger
-
-## Utils
-
-::: jio_savan_music_downloader.utils
-
-## Exceptions
-
-::: jio_savan_music_downloader.exception
-
-## Entities
-
-::: jio_savan_music_downloader.entity
-
 ## Models
 
 ::: jio_savan_music_downloader.models
@@ -35,7 +7,3 @@
 ## Services
 
 ::: jio_savan_music_downloader.services
-
-## APIs
-
-::: jio_savan_music_downloader.api

jio_savan_music_downloader/__main__.py

@@ -0,0 +1,30 @@
+# __main__.py
+import sys
+from pathlib import Path
+
+import streamlit.web.cli as stcli
+
+
+def main() -> None:
+    """
+    Sets up the environment to run a Streamlit application by modifying
+    system arguments and initiating the Streamlit CLI.
+
+    This function determines the current directory of the script, constructs
+    the path to the Streamlit app, updates the system arguments to run the
+    app, and then executes the Streamlit command-line interface to start the
+    application.
+
+    Exits the program when the Streamlit application exits, passing the
+    appropriate exit code.
+    """
+
+    current_dir = Path(__file__).parent
+    streamlit_app_path = current_dir / "app.py"
+
+    sys.argv = ["streamlit", "run", str(streamlit_app_path), "--server.port", "7860"]
+    sys.exit(stcli.main())
+
+
+if __name__ == "__main__":
+    main()

jio_savan_music_downloader/api/__init__.py

@@ -1,15 +0,0 @@
-"""
-
-This module contains API endpoints for a Retrieval-Augmented Generation (RAG) application.
-
-Endpoints:
-    - POST /upload-docs: This endpoint allows users to upload and manage documents for processing.
-    - POST /ask: This endpoint is used for querying the system and receiving context-aware answers based on uploaded documents.
-
-Features:
-    - **User-Friendly API**: Simplifies user interaction with the application's core functionalities.
-    - **Seamless Integration**: Connects document management, vector database, and LLM workflows efficiently.
-
-Purpose:
-    - Provides the interface for users to interact with the application's core RAG functionalities.
-"""

jio_savan_music_downloader/api/endpoint.py

@@ -1,31 +0,0 @@
-from fastapi import APIRouter
-
-router = APIRouter()
-
-
-@router.post("/upload-docs")
-async def upload_docs() -> dict:
-    """
-    Endpoint to upload documentation. This function encapsulates
-    the logic to upload the documentation and returns a success
-    message upon completion.
-
-    Returns:
-        dict: A dictionary containing a success message.
-    """
-
-    return {"message": "Training Model Successful"}
-
-
-@router.post("/ask")
-async def ask() -> dict:
-    """
-    Endpoint to handle prediction requests. This function processes
-    incoming requests for predictions and returns a success message
-    upon completion.
-
-    Returns:
-        dict: A dictionary containing a success message.
-    """
-
-    return {"message": "Prediction Successful"}

jio_savan_music_downloader/app.py

@@ -0,0 +1,53 @@
+import warnings
+
+import streamlit as st
+
+from jio_savan_music_downloader.services.app_service import search_music
+
+warnings.filterwarnings("ignore", category=SyntaxWarning, module="pysbd")
+
+
+# Set page config
+st.set_page_config(page_title="Music Search", page_icon="🎵", layout="wide")
+
+# App title
+st.title("🎵 Music Search Results")
+
+search_query = st.sidebar.text_input("Enter song name or artist")
+
+if search_query:
+    # Show loading spinner
+    with st.spinner("Searching for music..."):
+        music_data = search_music(search_query)
+
+    if music_data is None or len(music_data) == 0:
+        st.warning("No music found. Please try again.")
+
+    for item in music_data:
+        try:
+            song_id = item["song_info"]["song_url"].split("/")[-1]  # Get unique ID from URL
+            song_title = item["song_info"]["title"].split(" - ")[0]
+            musicians = item["song_info"]["musician"]
+            artists = ", ".join(musicians[:2])
+            release_date = item["song_info"]["release_date"]
+
+            # Display song information in a row
+            with st.container():
+                # Create columns
+                col1, col2, col3, col4 = st.columns([1, 2, 2, 2])
+
+                # Column 1: Image
+                with col1:
+                    st.image(item["album_image_url"], width=100)
+
+                # Column 2: Title and Artists
+                with col2:
+                    st.markdown(f"**{song_title}**")
+                    st.markdown(f"*{artists} | {release_date}*")
+
+                # Column 4: Audio Player
+                with col4:
+                    st.audio(item["song_info"]["downloadable_url"])
+        except Exception as e:
+            print(f"An error occurred: {e!s}")
+            continue

jio_savan_music_downloader/config/__init__.py

@@ -1,13 +0,0 @@
-"""
-
-This module manages the configuration settings for the application.
-
-Purpose:
-    - Centralizes configuration management for environment variables, application settings, and dependencies.
-    - Ensures consistent and secure access to critical configurations.
-
-Features:
-    - Loads environment variables and provides access to them.
-    - Validates and parses configuration settings using Pydantic models.
-    - Supports dynamic updates for configuration changes during runtime (if applicable).
-"""

jio_savan_music_downloader/config/agents.yaml

@@ -0,0 +1,9 @@
+music_researcher:
+    role: >
+        Senior Music Researcher at {website}
+    goal: >
+        Discover the latest music from {website} on {topic}
+    backstory: >
+        You're a seasoned music researcher with a passion for discovering new
+        music from {website}. Known for your ability to find the most relevant
+        information and present it in a clear and concise manner.

jio_savan_music_downloader/config/tasks.yaml

@@ -0,0 +1,7 @@
+music_research_task:
+    description: >
+        Find the latest music from  internet for {website} on {topic}. make sure you find the most
+        relevant information and present it in a clear and concise manner.
+    expected_output: >
+        A list of atleast 5 latest music links in a JSON with (title, link) format from internet on {topic}. Search the music in internet and get the most relevent {website} link.
+    agent: music_researcher

jio_savan_music_downloader/constants/__init__.py

@@ -1,18 +0,0 @@
-"""
-
-This package defines global constants used throughout the project. Constants
-help in maintaining consistency and avoiding magic numbers or strings in the codebase.
-
-Usage:
-    Import the required constants as needed:
-
-    Example:
-        ```python
-        from constants import APP_NAME, ENVIRONMENT
-        from constants import STATUS_OK, STATUS_BAD_REQUEST
-        ```
-
-Purpose:
-    - Centralizes constant values for maintainability and reusability.
-    - Reduces hard-coded values in the project.
-"""

jio_savan_music_downloader/core/__init__.py

@@ -1,12 +0,0 @@
-"""
-
-This module serves as the core of the application, integrating essential services for the Retrieval-Augmented Generation (RAG) workflow.
-
-Modules:
-    EmbeddingService: Handles document embedding generation and retrieval for context-aware processing.
-    LLMService: Interfaces with large language models (LLMs) to generate answers to user queries.
-    Processor: Orchestrates workflows between the embedding and LLM services, ensuring seamless data flow and processing.
-
-Purpose:
-    - Provides the foundational logic and services required to power the application's RAG functionalities.
-"""

jio_savan_music_downloader/crew.py

@@ -0,0 +1,53 @@
+from crewai import Agent, Crew, Process, Task
+from crewai.project import CrewBase, agent, crew, task
+
+from jio_savan_music_downloader.models.response_models import MusicDetails
+from jio_savan_music_downloader.tools.custom_tool import search_tool
+
+
+@CrewBase
+class JioSavanMusicDownloaderAgent:
+    """JioSavanMusicDownloaderAgent crew"""
+
+    agents_config = "config/agents.yaml"
+    tasks_config = "config/tasks.yaml"
+
+    @agent
+    def music_researcher(self) -> Agent:
+        """
+        Creates a music researcher agent.
+
+        This agent is responsible for searching for the specified music on JioSaavn
+        and returning the results in a structured format.
+
+        :return: An instance of the Agent class
+        """
+        return Agent(config=self.agents_config["music_researcher"], verbose=True)
+
+    @task
+    def music_research_task(self) -> Task:
+        """
+        Creates the music research task.
+
+        This task is responsible for searching for the specified music on JioSaavn
+        and returning the results in a structured format.
+
+        :return: An instance of the Task class
+        """
+
+        return Task(
+            config=self.tasks_config["music_research_task"],
+            tools=[search_tool],
+            output_json=MusicDetails,
+        )
+
+    @crew
+    def crew(self) -> Crew:
+        """Creates the JioSavanMusicDownloaderAgent crew"""
+
+        return Crew(
+            agents=self.agents,  # Automatically created by the @agent decorator
+            tasks=self.tasks,  # Automatically created by the @task decorator
+            process=Process.sequential,
+            verbose=False,
+        )

jio_savan_music_downloader/entity/__init__.py

@@ -1,16 +0,0 @@
-"""
-
-This module defines the database entities used across the application for data storage and retrieval.
-
-Purpose:
-    - Provides structured definitions for database models.
-    - Facilitates interaction with the database by mapping application data to database schemas.
-
-Components:
-    - DB Entities: Defines entities that represent database tables or collections, including fields and relationships.
-
-Features:
-    - Centralized entity definitions for consistent database operations.
-    - Supports validation and serialization of database records.
-    - Designed to integrate seamlessly with the database layer, ensuring reliable data handling.
-"""

jio_savan_music_downloader/exception/__init__.py

@@ -1,61 +0,0 @@
-"""
-
-This module defines custom exception classes and error-handling utilities tailored
-to the needs of a data science pipeline. It helps standardize error reporting, improve
-debugging, and provide meaningful feedback during model training, data preprocessing,
-and prediction processes.
-
-Classes:
-    DataValidationError: Raised when input data fails validation checks.
-    ModelTrainingError: Raised during errors in the model training phase, such as convergence issues or invalid configurations.
-    PredictionError: Raised when the prediction pipeline encounters issues, such as missing features or incompatible input formats.
-    PipelineExecutionError: Raised for generic errors occurring during pipeline execution.
-
-Usage:
-    Import and use the exceptions in various stages of the data science pipeline:
-
-    Example:
-        ```python
-        from exception import DataValidationError, ModelTrainingError
-
-        try:
-            validate_data(input_data)
-        except DataValidationError as e:
-            logger.error(f"Data validation failed: {e}")
-            raise
-        ```
-
-Features:
-    - Custom exceptions for specific pipeline stages, ensuring meaningful error reporting.
-    - Enables targeted exception handling, reducing debugging time.
-    - Provides a consistent structure for error messages across the project.
-
-Purpose:
-    - To define project-specific exceptions for common error scenarios in the pipeline.
-    - To improve the robustness and reliability of the pipeline by enabling clear error handling.
-    - To make the debugging process more intuitive by raising descriptive errors.
-
-Examples:
-    - **Data Validation**: Raise a `DataValidationError` if the input data schema is incorrect or missing required fields.
-    - **Model Training**: Raise a `ModelTrainingError` if the model fails to converge due to invalid hyperparameters.
-    - **Prediction**: Raise a `PredictionError` when incompatible input data is passed to the model.
-
-Additional Notes:
-    - Use these exceptions in conjunction with logging to provide detailed error information.
-    - Ensure that custom exceptions are raised with meaningful messages to assist in debugging and error resolution.
-"""
-
-from fastapi import HTTPException
-
-
-class CustomException(HTTPException):
-    """Custom exception class for handling errors in the data science pipeline."""
-
-    def __init__(self, status_code: int, detail: str):
-        """
-        Custom exception for handling API errors.
-
-        :param status_code: The HTTP status code to return.
-        :param detail: A string describing the error in detail.
-        """
-        super().__init__(status_code=status_code, detail=detail)

jio_savan_music_downloader/logger/__init__.py

@@ -1,61 +0,0 @@
-"""
-
-This module provides centralized logging utilities for the data science pipeline.
-It standardizes logging practices, ensures consistency across components, and facilitates
-easy debugging and monitoring of the pipeline's execution, including data preprocessing,
-model training, evaluation, and predictions.
-
-Functions:
-    setup_logging: Configures the logging system, including log format, level, and output destinations.
-    get_logger: Returns a logger instance for a specific module or stage of the pipeline.
-
-Features:
-    - Centralized logging configuration to maintain consistency.
-    - Support for different log levels (INFO, DEBUG, WARNING, ERROR, CRITICAL).
-    - Ability to write logs to files, console, or external monitoring systems.
-    - Timestamped log entries for accurate tracking of events.
-    - Integration with custom exception handling for detailed error reporting.
-
-Usage:
-    Use this module to log messages in a standardized manner across the project:
-
-    Example:
-        ```python
-        from src.logging import logger
-
-        logger.info("Starting the model training process...")
-        logger.error("An error occurred during data validation.")
-        ```
-
-Purpose:
-    - To provide a standardized mechanism for logging messages throughout the data science pipeline.
-    - To assist in debugging by capturing detailed logs of each pipeline stage.
-    - To enable seamless integration with monitoring and alerting systems.
-
-Best Practices:
-    - Use appropriate log levels to categorize messages (e.g., DEBUG for detailed information, ERROR for issues).
-    - Ensure logs include sufficient context, such as function names or input details, to aid debugging.
-    - Regularly monitor log files for anomalies or errors in the pipeline.
-
-Additional Notes:
-    - The `setup_logging` function can be configured to write logs to multiple destinations, such as files or cloud services.
-    - The module can be extended to integrate with third-party monitoring tools like Elasticsearch, Splunk, or Datadog.
-"""
-
-import logging
-import os
-import sys
-
-logging_str = "[%(asctime)s: %(levelname)s: %(module)s: %(message)s]"
-log_dir = "logs"
-log_filepath = os.path.join(log_dir,"jio-savan-music-downloader.log")
-os.makedirs(log_dir, exist_ok=True)
-
-
-logging.basicConfig(
-    level=logging.INFO,
-    format=logging_str,
-    handlers=[logging.FileHandler(log_filepath), logging.StreamHandler(sys.stdout)],
-)
-
-logger = logging.getLogger("jio-savan-music-downloader-logger")

jio_savan_music_downloader/main.py

@@ -1,92 +0,0 @@
-from fastapi import FastAPI, responses
-from fastapi.openapi.utils import get_openapi
-from fastapi_health import health
-
-from jio_savan_music_downloader.api.endpoint import router
-
-__version__ = "0.0.1"
-
-app = FastAPI(
-    title="jio-savan-music-downloader APIs",
-    description="This app will download Jio-Savan music.",
-    version=__version__,
-    docs_url="/docs",
-    redoc_url="/redoc",
-)
-
-
-app = FastAPI()
-
-@app.get("/", include_in_schema=False)
-async def root() -> responses.RedirectResponse:
-    """
-    Redirects the root URL to the API documentation page.
-
-    Returns:
-        RedirectResponse: A response object that redirects the client to the "/docs" URL.
-    """
-
-    return responses.RedirectResponse("/docs")
-
-
-# Health Check
-async def health_check() -> dict:
-    """
-    Checks the health of the API.
-
-    This endpoint checks the health of the API and returns a simple status
-    message. It is intended to be used by load balancers or other monitoring
-    systems to determine if the API is functional.
-
-    Returns:
-        dict: A dictionary containing the status of the API.
-    """
-    return {"status": "healthy"}
-
-
-# Include routers
-app.add_api_route(
-    "/health",
-    health([health_check]),
-    tags=["Management"],
-    description="Management APIs",
-)
-app.include_router(router, prefix="/api/v1", tags=["Operations"])
-
-
-def _custom_openapi() -> dict:
-    if app.openapi_schema:
-        return app.openapi_schema
-    openapi_schema = get_openapi(
-        title="jio-savan-music-downloader APIs",
-        description="This app will download Jio-Savan music.",
-        version=__version__,
-        routes=app.routes,
-    )
-    app.openapi_schema = openapi_schema
-    return app.openapi_schema
-
-
-app.openapi = _custom_openapi
-
-
-def main() -> None:
-    """
-    The main entry point of the application.
-
-    This function starts the FastAPI server using Uvicorn. It serves the API
-    on the specified host and port. The function is intended to be run
-    directly when the script is executed.
-
-    Notes:
-        - The 'nosec B104' comment is used to suppress a security warning
-          related to binding to all network interfaces.
-    """
-
-    import uvicorn
-
-    uvicorn.run(app, host="0.0.0.0", port=80)  # nosec B104
-
-
-if __name__ == "__main__":
-    main()

jio_savan_music_downloader/models/response_models.py

@@ -0,0 +1,16 @@
+from pydantic import BaseModel
+
+
+class MusicLink(BaseModel):
+    """
+    A class representing a music link."""
+
+    title: str
+    link: str
+
+
+class MusicDetails(BaseModel):
+    """
+    A class representing music details."""
+
+    links: list[MusicLink]

jio_savan_music_downloader/services/app_service.py

@@ -0,0 +1,41 @@
+import warnings
+
+from jio_savan_music_downloader.crew import JioSavanMusicDownloaderAgent
+from jio_savan_music_downloader.services.web_scarapper_service import scrape_pages
+
+warnings.filterwarnings("ignore", category=SyntaxWarning, module="pysbd")
+
+
+def search_music(query: str) -> list[dict]:
+    music_details = []
+    try:
+        # Search the internet for music
+        print(f"Searching for music: {query}")
+        search_results = search_internet(query)
+        print(f"Found {len(search_results)} results")
+
+        # Get music details
+        print("Getting music details")
+        music_details = get_music_details(search_results)
+        print(f"Got details for {len(music_details)} songs")
+        print(f"Music details: {music_details}")
+        print("Done")
+    except Exception as e:
+        print(f"An error occurred: {e!s}")
+
+    return music_details
+
+
+def search_internet(query: str) -> list[dict]:
+    # Run the crew
+    inputs = {"website": "https://www.jiosaavn.com", "topic": query}
+    result = JioSavanMusicDownloaderAgent().crew().kickoff(inputs=inputs)
+    links = result.to_dict().get("links", [])
+    return links if isinstance(links, list) else []
+
+
+def get_music_details(songs: list[dict]) -> list[dict]:
+    # Get music details by scrapping the pages
+
+    links: list[str] = [album["link"] for album in songs]
+    return scrape_pages(links)

jio_savan_music_downloader/services/web_scarapper_service.py

@@ -0,0 +1,231 @@
+import time
+from datetime import datetime
+from typing import Any
+from urllib.parse import urlparse
+
+import requests
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+def _setup_driver() -> webdriver.Chrome:
+    # Setup Chrome WebDriver (or other driver)
+    options = webdriver.ChromeOptions()
+
+    # Essential container arguments
+    options.add_argument("--headless=new")
+    options.add_argument("--disable-gpu")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+
+    # JavaScript-specific configurations
+    options.add_argument("--enable-javascript")
+    options.add_argument("--disable-web-security")
+    options.add_argument("--allow-running-insecure-content")
+
+    # Performance optimizations
+    options.add_argument("--window-size=1920,1080")
+    options.add_argument("--disable-extensions")
+    options.add_argument("--disable-setuid-sandbox")
+
+    # Memory management
+    options.add_argument("--disable-dev-tools")
+    options.add_argument("--no-zygote")
+    options.add_argument("--single-process")
+
+    # Handle Chrome Driver installation
+    try:
+        # For container environments, specify the Chrome version
+        print("Setting up Chrome WebDriver")
+        chrome_service = Service("/usr/bin/chromedriver")
+        driver = webdriver.Chrome(service=chrome_service, options=options)
+        driver = webdriver.Chrome(options=options)
+    except Exception as e:
+        # Fallback to direct path if ChromeDriverManager fails
+        print(f"An error occurred: {e!s}")
+        print("Falling back to direct path")
+
+        driver = webdriver.Chrome(options=options)
+    return driver
+
+
+def _get_downloadable_audio_link(url: str) -> str:
+    if not url:
+        return ""
+
+    # Extract the album ID and file ID from the URL
+    parsed_url = urlparse(url)
+    path_parts = parsed_url.path.split("/")
+    album_id = path_parts[-2]
+    file_id = path_parts[-1].split(".")[0]
+
+    # Construct the downloadable audio link
+    return f"https://aac.saavncdn.com/{album_id}/{file_id}.mp4"
+
+
+def _extract_musician_name(url: str) -> str:
+    return url.split("/")[-2].replace("-songs", "").replace("-", " ").title()
+
+
+def scrape_dynamic_page(url: str, wait_time: int = 5) -> dict[str, Any]:
+    """
+    Scrape a webpage including content loaded by JavaScript
+
+    Parameters:
+    url (str): The URL to scrape
+    wait_time (int): Maximum time to wait for dynamic content to load
+
+    Returns:
+    dict: Dictionary containing various elements from the page
+    """
+    driver = _setup_driver()
+
+    try:
+        # Load the page
+        driver.get(url)
+
+        # Wait for the button to be present
+        button = WebDriverWait(driver, wait_time).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR, 'a.c-btn.c-btn--primary[data-btn-icon="q"]'))
+        )
+
+        # Check visibility and enablement
+        is_displayed = button.is_displayed()
+        is_enabled = button.is_enabled()
+        print(f"Button displayed: {is_displayed}, Button enabled: {is_enabled}")
+
+        if is_displayed and is_enabled:
+            # Click the button
+            driver.execute_script("arguments[0].scrollIntoView(true);", button)
+            # driver.execute_script("arguments[0].click();", button)
+            try:
+                button.click()
+            except Exception as e:
+                print(f"An error occurred: {e!s}")
+                driver.execute_script("arguments[0].click();", button)
+        else:
+            print("Button is not interactable!")
+
+        # Wait a moment for any JavaScript updates
+        time.sleep(5)
+
+        # Get the updated HTML
+        html_content = driver.page_source
+        soup = BeautifulSoup(html_content, "html.parser")
+
+        # Extract elements
+        details = {
+            "album_title": soup.title.text if soup.title else "",
+            "description": soup.find("meta", {"name": "description"})["content"]
+            if soup.find("meta", {"name": "description"})
+            else "",
+            "album_description": soup.find("meta", {"property": "og:description"})["content"]
+            if soup.find("meta", {"property": "og:description"})
+            else "",
+            "album_url": soup.find("meta", {"property": "music:album"})["content"]
+            if soup.find("meta", {"property": "music:album"})
+            else "",
+            "album_image_url": soup.find("meta", {"property": "twitter:image"})["content"]
+            if soup.find("meta", {"property": "twitter:image"})
+            else "",
+            "song_info": {
+                "name": soup.title.text if soup.title else "",
+                "title": soup.find("meta", {"property": "twitter:title"})["content"]
+                if soup.find("meta", {"property": "twitter:title"})
+                else "",
+                "musician": [
+                    _extract_musician_name(musician["content"])
+                    for musician in soup.find_all("meta", {"property": "music:musician"})
+                ],
+                "release_date": datetime.strptime(
+                    soup.find("meta", {"property": "music:release_date"})["content"],
+                    "%Y-%m-%d",
+                ).strftime("%B %d, %Y")
+                if soup.find("meta", {"property": "music:release_date"})
+                else "",
+                "song_url": soup.find("meta", {"property": "twitter:url"})["content"]
+                if soup.find("meta", {"property": "twitter:url"})
+                else "",
+                "description": soup.find("meta", {"property": "twitter:description"})["content"]
+                if soup.find("meta", {"property": "twitter:description"})
+                else "",
+                "downloadable_url": _get_downloadable_audio_link(
+                    soup.find("audio").find("source")["src"] if soup.find("audio").find("source") else ""
+                ),
+                "song_lyrics_url": "https://www.jiosaavn.com" + soup.find("a", title="Song Lyrics")["href"]
+                if soup.find("a", title="Song Lyrics")
+                else "",
+            },
+        }
+    except TimeoutException:
+        print(f"Timeout waiting for page to load: {url}")
+        return {}
+    except Exception as e:
+        print(f"An error occurred: {e!s}")
+        return {}
+    else:
+        return details
+    finally:
+        driver.quit()
+
+
+def scrape_pages(urls: list[str]) -> list[dict]:
+    """
+    Scrape multiple webpages and return a list of elements
+
+    Parameters:
+    urls (list of str): List of URLs to scrape
+    wait_time (int): Maximum time to wait for dynamic content to load
+
+    Returns:
+    list of dict: List of dictionaries containing various elements from each page
+    """
+    results = []
+    for url in urls:
+        details = scrape_dynamic_page(url)
+        if details:
+            results.append(details)
+    return results
+
+
+def download_file(url: str) -> None:
+    """
+    Download a file from a URL and save it to a local file
+
+    Parameters:
+    url (str): URL of the file to be downloaded
+
+    Returns:
+    None
+    """
+    try:
+        response = requests.get(url, stream=True, timeout=10)
+
+        # Check if the request was successful
+        if response.status_code == 200:
+            # Open a local file with the specified filename in binary write mode
+            filename = _get_filename_name(url)
+            filename = f"downloads/{filename}.mp4"
+
+            with open(filename, "wb") as file:
+                # Write the content of the response to the file in chunks
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+            print(f"File downloaded successfully as '{filename}'")
+        else:
+            print(f"Failed to download file. HTTP Status Code: {response.status_code}")
+    except requests.exceptions.Timeout:
+        print(f"Request to {url} timed out.")
+    except requests.exceptions.RequestException as e:
+        print(f"Request to {url} failed: {e}")
+
+
+def _get_filename_name(url: str) -> str:
+    parsed_url = urlparse(url)
+    path_parts = parsed_url.path.split("/")
+    return path_parts[2]

jio_savan_music_downloader/tools/custom_tool.py

@@ -0,0 +1,9 @@
+from crewai_tools import FileWriterTool, ScrapeWebsiteTool, SerperDevTool
+
+jio_savan_scapper_tool = ScrapeWebsiteTool(website_url="https://www.jiosaavn.com")
+file_writer_tool = FileWriterTool()
+search_tool = SerperDevTool(
+    country="in",  # Set to 'in' for India
+    locale="en",  # Set locale to English
+    n_results=5,  # You can adjust the number of results as needed
+)

jio_savan_music_downloader/utils/__init__.py

@@ -1,15 +0,0 @@
-"""
-
-The `utils` module provides various utility functions for file I/O, data encoding/decoding, and directory management.
-
-Functions:
-    read_yaml: Reads a YAML file and returns its contents as a dictionary.
-    create_directories: Creates directories if they do not exist.
-    save_json: Saves data to a JSON file.
-    load_json: Loads JSON data from a file.
-    save_bin: Saves binary data to a file.
-    load_bin: Loads binary data from a file.
-    get_size: Returns the size of a file or directory in bytes.
-    decode_image: Decodes an image from a base64 string.
-    encode_image_into_base64: Encodes an image into a base64 string.
-"""

pyproject.toml

@@ -11,21 +11,16 @@ packages = [
 ]
 
 [tool.poetry.scripts]
-jio_savan_music_downloader = "jio_savan_music_downloader.main:main"
+jio_savan_music_downloader = "jio_savan_music_downloader.__main__:main"
 
 [tool.poetry.dependencies]
-python = ">=3.11,<4.0"
-fastapi = "~0.100.0"  # Latest compatible version
-uvicorn = "^0.23.0"  # For running FastAPI apps
-fastapi-health = "^0.4.0"  # Health check for FastAPI
-#pyyaml = "^6.0.2"
-#python-box = "^7.2.0"
-#ensure = "^1.0.4"
-#joblib = "^1.3.2"  # Parallel processing and caching
-#python-dotenv = "^1.0.0"  # Manage environment variables
-#PyPDF2 = "^3.0.0"  # For PDF manipulation
-#pydantic = "^2.0.0"  # Data validation and settings management
-
+python = ">=3.11,<3.13"
+crewai = {version = ">=0.86.0,<1.0.0", extras = ["tools"]}
+streamlit = "^1.41.1"
+selenium = "^4.27.1"
+# chromedriver_autoinstaller = "^0.6.4"
+# webdriver-manager = "*"
+bs4 = "*"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -43,7 +38,8 @@ twine = "^4.0.0"  # Publish packages
 bandit = "^1.8.0"  # Security check
 pylint = "^3.0.0"  # Powerful linter
 pydocstyle = "^6.3.0"  # Enforce PEP 257 docstring conventions
-isort = "^5.12.0"  # Sort imports
+# isort = "^5.12.0"  # Sort imports
+# ruff = "^0.8.6"  # Linting tool
 
 [tool.poetry.group.docs.dependencies]
 mkdocs = "^1.5.0"  # Documentation site generator
@@ -63,20 +59,6 @@ watchdog = "^3.0.0"  # File monitoring
 mkdocstrings = {extras = ["python"], version = "^0.27.0"}  # Auto-generate documentation from docstrings
 mkdocs-minify-plugin = "^0.8.0"  # Minify HTML
 
-[tool.poetry.group.test.dependencies]
-pytest-mock = "^3.11.0"  # Mocking in tests
-factory-boy = "^3.3.1"  # Test data generation
-pytest-asyncio = "^0.21.0"  # Async testing support
-pytest-xdist = "^3.3.1"  # Parallel test execution
-freezegun = "^1.2.0"  # Mock datetime
-pytest = "^7.2.0"  # Testing framework
-allure-pytest = "^2.13.0"  # Reporting for pytest
-pytest-sugar = "^0.9.7"  # Better test output
-pytest-cov = "^4.0.0"  # Test coverage reports
-httpx = "^0.24.0"
-pytest-runner = "^6.0.0"  # Running tests via `python setup.py test`
-
-
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
@@ -101,7 +83,10 @@ module = [
     "yaml.*",
     "ensure.*",
     "fastapi_health.*",
-    "jio_savan_music_downloader.main"
+    "crewai.*",
+    "crewai_tools.*",
+    "bs4.*",
+    "jio_savan_music_downloader.crew"
 ]
 ignore_missing_imports = true
 ignore_errors = true
@@ -178,11 +163,7 @@ source = ["jio_savan_music_downloader"]
 # concurrency = ["thread"]
 omit = [
     "**/__init__.py", # Exclude all init files
-    "jio_savan_music_downloader/main.py",  # Exclude main.py file
-    "jio_savan_music_downloader/constants/*",   # Exclude all files in a constants folder
-    "jio_savan_music_downloader/exception/*",   # Exclude all files in a exception folder
-    "jio_savan_music_downloader/logger/*",  # Exclude all files in a logger folder
-    "jio_savan_music_downloader/entity/*",  # Exclude all files in entity folder
+    "jio_savan_music_downloader/crew.py",  # Exclude crew.py file
     "jio_savan_music_downloader/config/*",  # Exclude all files in config folder
     "jio_savan_music_downloader/models/*",  # Exclude all files in model folder
 ]
@@ -200,13 +181,16 @@ glob = "pyproject.toml"
 search = 'version = "{current_version}"'
 replace = 'version = "{new_version}"'
 
-[[tool.bumpversion.files]]
-glob = "jio_savan_music_downloader/main.py"
-search = '__version__ = "{current_version}"'
-replace = '__version__ = "{new_version}"'
-
 [tool.deptry]
 exclude = ["research","artifacts", "notebooks", "tests", "docs", ".venv", "venv", "__pycache__", ".ruff_cache", ".pytest_cache", ".mypy_cache", ".coverage", ".git", "build", "dist", ".github", "site", "config"]
+ignore = ["DEP003"]
 
 [tool.pydocstyle]
-select = ["D101", "D102"]
+select = ["D101", "D102"]
+
+[tool.isort]
+profile = "black"
+known_third_party = ["requests", "bs4", "pydantic", "crewai_tools"]
+default_section = "THIRDPARTY"
+force_sort_within_sections = true
+line_length = 120

tests/test_api_endpoint.py

@@ -1,15 +0,0 @@
-from starlette.testclient import TestClient
-from jio_savan_music_downloader.main import app
-
-client = TestClient(app=app)
-
-def test_train_model():
-    response = client.post("/api/v1/upload-docs")
-    assert response.status_code == 200
-    assert response.json() is not None
-
-
-def test_predict():
-    response = client.post("/api/v1/ask")
-    assert response.status_code == 200
-    assert response.json() is not None