Enterprise: Clean up directory structure and add proper CI/CD, monitoring, and deployment configurations

.github/workflows/ci-cd.yml

@@ -0,0 +1,109 @@
+name: CI/CD Pipeline
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9, 3.10, 3.11]
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pre-commit
+    
+    - name: Run pre-commit hooks
+      run: pre-commit run --all-files
+    
+    - name: Run tests
+      run: |
+        pytest tests/ -v --cov=src --cov-report=xml
+    
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    
+    - name: Install dependencies
+      run: |
+        pip install flake8 black isort mypy
+    
+    - name: Run linting
+      run: |
+        flake8 src/ tests/
+        black --check src/ tests/
+        isort --check-only src/ tests/
+        mypy src/
+
+  security:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Run security scan
+      uses: snyk/actions/python@master
+      env:
+        SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      with:
+        args: --severity-threshold=high
+
+  build:
+    needs: [test, lint, security]
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: true
+        tags: |
+          ghcr.io/${{ github.repository }}:latest
+          ghcr.io/${{ github.repository }}:${{ github.sha }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+    environment: production
+    
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Deploy to Kubernetes
+      run: |
+        echo "Deploying to production..."
+        # Add your deployment commands here 

Makefile

@@ -7,11 +7,12 @@ help: ## Show this help message
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
 
 install: ## Install dependencies
-	pip install -r requirements.txt
+	pip install -e .
+	pip install -e ".[dev]"
 	pre-commit install
 
 test: ## Run tests
-	pytest tests/ -v --cov=src --cov-report=html
+	pytest tests/ -v --cov=src --cov-report=html --cov-report=xml
 
 lint: ## Run linting
 	flake8 src/ tests/
@@ -26,6 +27,9 @@ clean: ## Clean build artifacts
 	find . -type d -name "__pycache__" -delete
 	rm -rf .pytest_cache/
 	rm -rf htmlcov/
+	rm -rf build/
+	rm -rf dist/
+	rm -rf *.egg-info/
 
 build: ## Build Docker image
 	docker build -t fred-ml .
@@ -33,20 +37,33 @@ build: ## Build Docker image
 run: ## Run application locally
 	uvicorn src.main:app --reload --host 0.0.0.0 --port 8000
 
-run-docker: ## Run with Docker Compose
-	docker-compose up --build
+run-docker: ## Run with Docker Compose (development)
+	docker-compose -f deploy/docker/docker-compose.dev.yml up --build
+
+run-prod: ## Run with Docker Compose (production)
+	docker-compose -f deploy/docker/docker-compose.prod.yml up --build
 
 deploy: ## Deploy to Kubernetes
-	kubectl apply -f kubernetes/
-	helm install fred-ml helm/
+	kubectl apply -f deploy/kubernetes/
+
+deploy-helm: ## Deploy with Helm
+	helm install fred-ml deploy/helm/
 
 logs: ## View application logs
-	docker-compose logs -f fred-ml
+	docker-compose -f deploy/docker/docker-compose.dev.yml logs -f fred-ml
 
 shell: ## Open shell in container
-	docker-compose exec fred-ml bash
+	docker-compose -f deploy/docker/docker-compose.dev.yml exec fred-ml bash
 
 migrate: ## Run database migrations
 	alembic upgrade head
 
-setup-dev: install format lint test ## Setup development environment 
+setup-dev: install format lint test ## Setup development environment
+
+ci: test lint format ## Run CI checks locally
+
+package: clean build ## Build package for distribution
+	python -m build
+
+publish: package ## Publish to PyPI
+	twine upload dist/* 

alerts/alertmanager.yml

@@ -1,21 +0,0 @@
-global:
-  resolve_timeout: 5m
-
-route:
-  group_by: ['alertname']
-  group_wait: 10s
-  group_interval: 10s
-  repeat_interval: 1h
-  receiver: 'web.hook'
-
-receivers:
-  - name: 'web.hook'
-    webhook_configs:
-      - url: 'http://127.0.0.1:5001/'
-
-inhibit_rules:
-  - source_match:
-      severity: 'critical'
-    target_match:
-      severity: 'warning'
-    equal: ['alertname', 'dev', 'instance'] 

deploy/docker/docker-compose.prod.yml

@@ -0,0 +1,65 @@
+version: '3.8'
+
+services:
+  fred-ml:
+    image: ghcr.io/eaname/fredml:latest
+    ports:
+      - "8000:8000"
+    environment:
+      - FRED_API_KEY=${FRED_API_KEY}
+      - ENVIRONMENT=production
+      - LOG_LEVEL=INFO
+    volumes:
+      - ./data:/app/data
+      - ./logs:/app/logs
+    depends_on:
+      - redis
+      - postgres
+    networks:
+      - fred-ml-network
+    restart: unless-stopped
+
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    volumes:
+      - redis_data:/data
+    networks:
+      - fred-ml-network
+    restart: unless-stopped
+
+  postgres:
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_DB: fred_ml
+      POSTGRES_USER: fred_user
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    networks:
+      - fred-ml-network
+    restart: unless-stopped
+
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+    depends_on:
+      - fred-ml
+    networks:
+      - fred-ml-network
+    restart: unless-stopped
+
+volumes:
+  redis_data:
+  postgres_data:
+
+networks:
+  fred-ml-network:
+    driver: bridge 

{kubernetes → deploy/kubernetes}/deployment.yaml

@@ -4,6 +4,7 @@ metadata:
   name: fred-ml
   labels:
     app: fred-ml
+    version: v1.0.0
 spec:
   replicas: 3
   selector:
@@ -13,12 +14,14 @@ spec:
     metadata:
       labels:
         app: fred-ml
+        version: v1.0.0
     spec:
       containers:
       - name: fred-ml
-        image: fred-ml:latest
+        image: ghcr.io/eaname/fredml:latest
         ports:
         - containerPort: 8000
+          name: http
         env:
         - name: FRED_API_KEY
           valueFrom:
@@ -27,6 +30,8 @@ spec:
               key: fred-api-key
         - name: ENVIRONMENT
           value: "production"
+        - name: LOG_LEVEL
+          value: "INFO"
         resources:
           requests:
             memory: "256Mi"
@@ -40,17 +45,35 @@ spec:
             port: 8000
           initialDelaySeconds: 30
           periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
         readinessProbe:
           httpGet:
             path: /ready
             port: 8000
           initialDelaySeconds: 5
           periodSeconds: 5
+          timeoutSeconds: 3
+          failureThreshold: 3
+        volumeMounts:
+        - name: data-volume
+          mountPath: /app/data
+        - name: logs-volume
+          mountPath: /app/logs
+      volumes:
+      - name: data-volume
+        persistentVolumeClaim:
+          claimName: fred-ml-data-pvc
+      - name: logs-volume
+        persistentVolumeClaim:
+          claimName: fred-ml-logs-pvc
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: fred-ml-service
+  labels:
+    app: fred-ml
 spec:
   selector:
     app: fred-ml
@@ -58,4 +81,27 @@ spec:
     - protocol: TCP
       port: 80
       targetPort: 8000
-  type: LoadBalancer 
+      name: http
+  type: LoadBalancer
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fred-ml-data-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: fred-ml-logs-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi 

helm/Chart.yaml

@@ -1,17 +0,0 @@
-apiVersion: v2
-name: fred-ml
-description: A Helm chart for FRED ML Economic Data Analysis
-type: application
-version: 1.0.0
-appVersion: "1.0.0"
-keywords:
-  - economics
-  - data-analysis
-  - machine-learning
-  - fred
-home: https://github.com/EAName/FREDML
-sources:
-  - https://github.com/EAName/FREDML
-maintainers:
-  - name: Edwin Salguero
-    email: [email protected] 

infrastructure/alerts/alerts.yml

@@ -0,0 +1,56 @@
+groups:
+  - name: fred-ml
+    rules:
+      - alert: FredMLHighErrorRate
+        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "High error rate detected"
+          description: "Error rate is {{ $value }} errors per second"
+
+      - alert: FredMLHighResponseTime
+        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High response time detected"
+          description: "95th percentile response time is {{ $value }} seconds"
+
+      - alert: FredMLDown
+        expr: up{job="fred-ml"} == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "FredML service is down"
+          description: "FredML service has been down for more than 1 minute"
+
+      - alert: FredMLHighMemoryUsage
+        expr: (container_memory_usage_bytes{container="fred-ml"} / container_spec_memory_limit_bytes{container="fred-ml"}) > 0.8
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High memory usage detected"
+          description: "Memory usage is {{ $value | humanizePercentage }}"
+
+      - alert: FredMLHighCPUUsage
+        expr: rate(container_cpu_usage_seconds_total{container="fred-ml"}[5m]) > 0.8
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage detected"
+          description: "CPU usage is {{ $value | humanizePercentage }}"
+
+      - alert: FredMLAPIRateLimit
+        expr: increase(fred_api_rate_limit_exceeded_total[5m]) > 0
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "FRED API rate limit exceeded"
+          description: "API rate limit has been exceeded in the last 5 minutes" 

infrastructure/monitoring/prometheus.yml

@@ -0,0 +1,62 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    cluster: fred-ml-cluster
+    job: fred-ml
+
+rule_files:
+  - "alerts.yml"
+
+scrape_configs:
+  - job_name: 'fred-ml'
+    static_configs:
+      - targets: ['fred-ml-service:8000']
+    metrics_path: '/metrics'
+    scrape_interval: 5s
+    scrape_timeout: 3s
+    honor_labels: true
+
+  - job_name: 'kubernetes-pods'
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+  - job_name: 'kubernetes-service-endpoints'
+    kubernetes_sd_configs:
+      - role: endpoints
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          - alertmanager:9093 

monitoring/prometheus.yml

@@ -1,18 +0,0 @@
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-
-rule_files:
-  # - "first_rules.yml"
-  # - "second_rules.yml"
-
-scrape_configs:
-  - job_name: 'fred-ml'
-    static_configs:
-      - targets: ['localhost:8000']
-    metrics_path: '/metrics'
-    scrape_interval: 5s
-
-  - job_name: 'prometheus'
-    static_configs:
-      - targets: ['localhost:9090'] 

pyproject.toml

@@ -0,0 +1,127 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "fred-ml"
+version = "1.0.0"
+description = "Enterprise-grade economic data analysis platform using FRED API"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "Edwin Salguero", email = "[email protected]"}
+]
+maintainers = [
+    {name = "Edwin Salguero", email = "[email protected]"}
+]
+keywords = ["economics", "data-analysis", "machine-learning", "fred", "api"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Financial and Insurance Industry",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+requires-python = ">=3.9"
+dependencies = [
+    "fredapi==0.4.2",
+    "pandas==2.1.4",
+    "numpy==1.24.3",
+    "matplotlib==3.7.2",
+    "seaborn==0.12.2",
+    "jupyter==1.0.0",
+    "python-dotenv==1.0.0",
+    "requests==2.31.0",
+    "PyYAML==6.0.2",
+    "APScheduler==3.10.4",
+    "scikit-learn==1.3.0",
+    "scipy==1.11.1",
+    "statsmodels==0.14.0",
+    "fastapi==0.104.1",
+    "uvicorn[standard]==0.24.0",
+    "pydantic==1.10.13",
+    "redis==5.0.1",
+    "psycopg2-binary==2.9.9",
+    "sqlalchemy==2.0.23",
+    "alembic==1.13.0",
+    "prometheus-client==0.19.0",
+    "structlog==23.2.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest==7.4.0",
+    "pytest-asyncio==0.21.1",
+    "httpx==0.25.2",
+    "black==23.11.0",
+    "flake8==6.1.0",
+    "mypy==1.7.1",
+    "pre-commit==3.6.0",
+    "isort==5.12.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/EAName/FREDML"
+Documentation = "https://github.com/EAName/FREDML#readme"
+Repository = "https://github.com/EAName/FREDML.git"
+"Bug Tracker" = "https://github.com/EAName/FREDML/issues"
+
+[project.scripts]
+fred-ml = "src.main:main"
+
+[tool.black]
+line-length = 88
+target-version = ['py39']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+'''
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 88
+known_first_party = ["src"]
+
+[tool.mypy]
+python_version = "3.9"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+disallow_untyped_decorators = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-ra -q --strict-markers --strict-config"
+testpaths = ["tests"]
+python_files = ["test_*.py", "*_test.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+]