Upload 6 files

app.py

@@ -1,16 +1,22 @@
+```python
 import gradio as gr
 import logging
 import requests
+import os
 from datetime import datetime
 from typing import Dict, List, Optional, Tuple
 from enhanced_legal_scraper import EnhancedLegalScraper, LegalDocument, IRANIAN_LEGAL_SOURCES
 
+# Create log directory
+log_dir = '/app/logs'
+os.makedirs(log_dir, exist_ok=True)
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s',
     handlers=[
-        logging.FileHandler('/app/logs/legal_scraper.log'),
+        logging.FileHandler(os.path.join(log_dir, 'legal_scraper.log')),
         logging.StreamHandler()
     ]
 )
@@ -203,7 +209,7 @@ class LegalScraperInterface:
     def export_data(self, export_format: str) -> Tuple[str, Optional[gr.File]]:
         try:
             timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-            filename = f"legal_documents_{timestamp}.{export_format.lower()}"
+            filename = f"/app/data/legal_documents_{timestamp}.{export_format.lower()}"
             if export_format == "CSV":
                 result = self.scraper.export_to_csv(filename)
                 if result:
@@ -584,4 +590,5 @@ if __name__ == "__main__":
         show_error=True,
         show_tips=True,
         enable_queue=True
-    )
+    )
+```

docker-compose.yaml

@@ -1,3 +1,4 @@
+```yaml
 version: "3.8"
 
 services:
@@ -15,57 +16,4 @@ services:
       - ./data:/app/data:rw
       - ./cache:/app/cache:rw
       - ./logs:/app/logs:rw
-      - ./uploads:/app/uploads:rw
-    environment:
-      - DATABASE_DIR=/app/data
-      - DATABASE_PATH=/app/data/legal_scraper.db
-      - TRANSFORMERS_CACHE=/app/cache
-      - HF_HOME=/app/cache
-      - LOG_LEVEL=INFO
-      - ENVIRONMENT=production
-      - PYTHONPATH=/app
-      - PYTHONUNBUFFERED=1
-      - APP_MODE=gradio
-      - API_BASE_URL=http://fastapi:8000
-    depends_on:
-      fastapi:
-        condition: service_healthy
-
-  fastapi:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    container_name: legal_dashboard_fastapi
-    restart: unless-stopped
-    ports:
-      - "8000:8000"
-    networks:
-      - app_network
-    volumes:
-      - ./data:/app/data:rw
-      - ./cache:/app/cache:rw
-      - ./logs:/app/logs:rw
-      - ./uploads:/app/uploads:rw
-    environment:
-      - DATABASE_DIR=/app/data
-      - DATABASE_PATH=/app/data/legal_scraper.db
-      - TRANSFORMERS_CACHE=/app/cache
-      - HF_HOME=/app/cache
-      - LOG_LEVEL=INFO
-      - ENVIRONMENT=production
-      - PYTHONPATH=/app
-      - PYTHONUNBUFFERED=1
-      - APP_MODE=fastapi
-    healthcheck:
-      test: ["CMD-SHELL", "curl -fs http://localhost:8000/health || exit 1"]
-      interval: 45s
-      timeout: 30s
-      retries: 10
-      start_period: 180s
-
-  redis:
-    image: redis:7-alpine
-    container_name: legal_dashboard_redis
-    restart: unless-stopped
-    networks:
-      - app_network
+      -

dockerfile

@@ -1,67 +1,82 @@
-```
-   # Stage 1: Builder
-   FROM python:3.10-slim AS builder
-
-   # Install build dependencies
-   RUN apt-get update && apt-get install -y \
-       build-essential \
-       gcc \
-       g++ \
-       libffi-dev \
-       libssl-dev \
-       && rm -rf /var/lib/apt/lists/*
-
-   # Upgrade pip and install wheel
-   RUN pip install --upgrade pip setuptools wheel
-
-   # Create virtual environment
-   RUN python -m venv /opt/venv
-   ENV PATH="/opt/venv/bin:$PATH"
-
-   # Copy requirements and install dependencies
-   WORKDIR /build
-   COPY requirements.txt .
-   RUN pip install --no-cache-dir -r requirements.txt
-
-   # Stage 2: Production
-   FROM python:3.10-slim
-
-   # Install runtime dependencies
-   RUN apt-get update && apt-get install -y \
-       sqlite3 \
-       && rm -rf /var/lib/apt/lists/* \
-       && apt-get clean
-
-   # Create non-root user
-   RUN groupadd -g 1000 appuser && useradd -r -u 1000 -g appuser appuser
-
-   # Copy virtual environment from builder
-   COPY --from=builder /opt/venv /opt/venv
-   ENV PATH="/opt/venv/bin:$PATH"
-
-   # Set working directory
-   WORKDIR /app
-
-   # Copy all files
-   COPY --chown=appuser:appuser . .
-
-   # Environment variables
-   ENV PYTHONPATH=/app
-   ENV TRANSFORMERS_CACHE=/app/cache
-   ENV HF_HOME=/app/cache
-   ENV LOG_LEVEL=INFO
-   ENV ENVIRONMENT=production
-   ENV PYTHONUNBUFFERED=1
-   ENV API_BASE_URL=http://localhost:8000
-   ENV APP_MODE=gradio
-
-   # Switch to non-root user
-   USER appuser
-
-   # Expose ports for Gradio and FastAPI
-   EXPOSE 7860
-   EXPOSE 8000
-
-   # Start application via run.py
-   CMD ["python", "run.py"]
-   ```
+```dockerfile
+# Stage 1: Builder
+FROM python:3.10-slim AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    gcc \
+    g++ \
+    libffi-dev \
+    libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Upgrade pip and install wheel
+RUN pip install --upgrade pip setuptools wheel
+
+# Create virtual environment
+RUN python -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy requirements and install dependencies
+WORKDIR /build
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Pre-download transformer models
+RUN mkdir -p /app/cache && \
+    python -c "from transformers import AutoModel, AutoTokenizer; \
+               AutoModel.from_pretrained('HooshvareLab/bert-fa-base-uncased', cache_dir='/app/cache'); \
+               AutoTokenizer.from_pretrained('HooshvareLab/bert-fa-base-uncased', cache_dir='/app/cache')" || true
+RUN python -c "from transformers import TrOCRProcessor, VisionEncoderDecoderModel; \
+               TrOCRProcessor.from_pretrained('microsoft/trocr-base-printed', cache_dir='/app/cache'); \
+               VisionEncoderDecoderModel.from_pretrained('microsoft/trocr-base-printed', cache_dir='/app/cache')" || true
+
+# Stage 2: Production
+FROM python:3.10-slim
+
+# Install runtime dependencies
+RUN apt-get update && apt-get install -y \
+    sqlite3 \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create non-root user
+RUN groupadd -g 1000 appuser && useradd -r -u 1000 -g appuser appuser
+
+# Copy virtual environment from builder
+COPY --from=builder /opt/venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy cached models
+COPY --from=builder /app/cache /app/cache
+
+# Create required directories
+RUN mkdir -p /app/data /app/logs /app/uploads && \
+    chown -R appuser:appuser /app/data /app/logs /app/uploads /app/cache
+
+# Set working directory
+WORKDIR /app
+
+# Copy all files
+COPY --chown=appuser:appuser . .
+
+# Environment variables
+ENV PYTHONPATH=/app
+ENV HF_HOME=/app/cache
+ENV LOG_LEVEL=INFO
+ENV ENVIRONMENT=production
+ENV PYTHONUNBUFFERED=1
+ENV API_BASE_URL=http://localhost:8000
+ENV APP_MODE=gradio
+
+# Switch to non-root user
+USER appuser
+
+# Expose ports for Gradio and FastAPI
+EXPOSE 7860
+EXPOSE 8000
+
+# Start application via run.py
+CMD ["python", "run.py"]
+```

enhanced_legal_scraper.py

@@ -1,9 +1,11 @@
+```python
 import requests
 import time
 import json
 import csv
 import sqlite3
 import logging
+import os
 from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple
 from urllib.parse import urljoin, urlparse
@@ -24,12 +26,16 @@ except ImportError as e:
     NLP_AVAILABLE = False
     logging.warning(f"⚠️ NLP libraries not available: {e}")
 
+# Create log directory
+log_dir = '/app/logs'
+os.makedirs(log_dir, exist_ok=True)
+
 # Configure logging
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s',
     handlers=[
-        logging.FileHandler('/app/logs/legal_scraper.log'),
+        logging.FileHandler(os.path.join(log_dir, 'legal_scraper.log')),
         logging.StreamHandler()
     ]
 )
@@ -85,14 +91,18 @@ class PersianNLPProcessor:
         self.model_tokenizer = None
         if NLP_AVAILABLE:
             try:
+                logger.info("Initializing Persian NLP components...")
                 self.normalizer = Normalizer()
                 self.tokenizer = WordTokenizer()
                 self.sentence_tokenizer = SentenceTokenizer()
-                self.model = AutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased")
-                self.model_tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased")
+                if os.getenv("ENVIRONMENT") != "huggingface_free":
+                    self.model = AutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased", cache_dir="/app/cache")
+                    self.model_tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased", cache_dir="/app/cache")
+                logger.info("Persian NLP components initialized")
             except Exception as e:
-                logger.error(f"Failed to initialize NLP components: {e}")
+                logger.warning(f"Failed to initialize NLP components: {e}. Falling back to basic text processing.")
                 self.model = None
+                self.model_tokenizer = None
     
     def normalize_text(self, text: str) -> str:
         if self.normalizer:
@@ -185,6 +195,8 @@ class PersianNLPProcessor:
 
 class EnhancedLegalScraper:
     def __init__(self, delay: float = 2.0, db_path: str = "/app/data/legal_scraper.db"):
+        # Create data directory
+        os.makedirs('/app/data', exist_ok=True)
         self.nlp = PersianNLPProcessor() if NLP_AVAILABLE else None
         self.session = requests.Session()
         self.delay = delay
@@ -212,7 +224,7 @@ class EnhancedLegalScraper:
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     title TEXT NOT NULL,
                     content TEXT NOT NULL,
-                    source_url TEXT UNIQUE NOT NULL,
+                    source_url TEXT UNIQUE NOT NOT NULL,
                     document_type TEXT NOT NULL,
                     date_published TEXT,
                     date_scraped TEXT NOT NULL,
@@ -615,4 +627,5 @@ class EnhancedLegalScraper:
             return stats
         except Exception as e:
             logger.error(f"Statistics failed: {e}")
-            return {}
+            return {}
+```

main.py

@@ -1,3 +1,4 @@
+```python
 import os
 import tempfile
 import logging
@@ -29,12 +30,16 @@ except ImportError as e:
     ML_AVAILABLE = False
     logger.warning(f"⚠️ ML libraries not available: {e}")
 
+# Create log directory
+log_dir = '/app/logs'
+os.makedirs(log_dir, exist_ok=True)
+
 # Configure logging
 logging.basicConfig(
     level=os.getenv("LOG_LEVEL", "INFO").upper(),
     format='%(asctime)s - %(levelname)s - %(message)s',
     handlers=[
-        logging.FileHandler('/app/logs/legal_dashboard.log'),
+        logging.FileHandler(os.path.join(log_dir, 'legal_dashboard.log')),
         logging.StreamHandler()
     ]
 )
@@ -66,19 +71,19 @@ class OCRService:
         self.model = None
         self.processor = None
         self.model_loaded = False
+        if ML_AVAILABLE and os.getenv("ENVIRONMENT") != "huggingface_free":
+            self._load_model()
     
-    async def _load_model_async(self):
-        if not ML_AVAILABLE:
-            return
+    def _load_model(self):
         try:
             logger.info("Loading TrOCR model...")
             model_name = "microsoft/trocr-base-printed"
-            self.processor = TrOCRProcessor.from_pretrained(model_name)
-            self.model = VisionEncoderDecoderModel.from_pretrained(model_name)
+            self.processor = TrOCRProcessor.from_pretrained(model_name, cache_dir="/app/cache")
+            self.model = VisionEncoderDecoderModel.from_pretrained(model_name, cache_dir="/app/cache")
             self.model_loaded = True
             logger.info("✅ TrOCR model loaded successfully")
         except Exception as e:
-            logger.error(f"❌ Failed to load TrOCR model: {e}")
+            logger.warning(f"❌ Failed to load TrOCR model: {e}. OCR will use basic processing.")
             self.model_loaded = False
     
     async def extract_text_from_pdf(self, file_path: str) -> OCRResponse:
@@ -163,8 +168,8 @@ legal_api = LegalDashboardAPI()
 
 @app.on_event("startup")
 async def startup_event():
-    if ML_AVAILABLE:
-        await legal_api.ocr_service._load_model_async()
+    if ML_AVAILABLE and os.getenv("ENVIRONMENT") != "huggingface_free":
+        legal_api.ocr_service._load_model()
 
 @app.get("/health")
 async def health_check():
@@ -299,4 +304,5 @@ async def global_exception_handler(request: Request, exc: Exception):
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=False, log_level="info")
+    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=False, log_level="info")
+```

requirements.txt

@@ -1,16 +1,18 @@
-   requests>=2.28.0
-   beautifulsoup4>=4.11.0
-   pandas>=2.0.0,<3.0.0
-   scikit-learn>=1.4.0
-   transformers>=4.39.0
-   torch>=2.0.0
-   hazm==0.10.0
-   numpy==1.24.3
-   gradio>=4.0.0
-   fastapi>=0.95.0
-   uvicorn>=0.20.0
-   python-multipart>=0.0.6
-   pillow>=9.0.0
-   pymupdf>=1.21.0
-   python-dotenv>=0.21.0
-   plotly>=5.0.0
+```
+requests>=2.28.0
+beautifulsoup4>=4.11.0
+pandas>=2.0.0,<3.0.0
+scikit-learn>=1.4.0
+transformers>=4.39.0
+torch>=2.0.0
+hazm==0.10.0
+numpy==1.24.3
+gradio>=4.0.0
+fastapi>=0.95.0
+uvicorn>=0.20.0
+python-multipart>=0.0.6
+pillow>=9.0.0
+pymupdf>=1.21.0
+python-dotenv>=0.21.0
+plotly>=5.0.0
+```