Coverage for hf_space_main.py: 0.00%
308 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-11-25 15:37 +0330
1"""
2HuggingFace Space Main Application
3Complete backend for index.html UI
4"""
6import os
7import logging
8from contextlib import asynccontextmanager
9from fastapi import FastAPI, Request
10from fastapi.middleware.cors import CORSMiddleware
11from fastapi.staticfiles import StaticFiles
12from fastapi.responses import FileResponse, Response
13from starlette.middleware.base import BaseHTTPMiddleware
14from pathlib import Path
16# Helper function to get debug log path dynamically
17def _get_debug_log_path():
18 """Find workspace root and return path to debug.log"""
19 workspace_root = Path("/app" if Path("/app").exists() else (Path("/workspace") if Path("/workspace").exists() else Path("."))).resolve()
20 debug_log_dir = workspace_root / ".cursor"
21 debug_log_dir.mkdir(parents=True, exist_ok=True)
22 return debug_log_dir / "debug.log"
24# Import routers
25from backend.routers.hf_service_api import router as service_router
26from backend.routers.hf_models_api import router as models_router
27from backend.routers.hf_providers_api import router as providers_router
28from backend.routers.market_router import router as market_router
29from backend.routers.sentiment_router import router as sentiment_router
30from backend.routers.news_router import router as news_router
31from backend.routers.whales_router import router as whales_router
32from backend.routers.onchain_router import router as onchain_router
33from backend.routers.system_router import router as system_router
35# Import additional API routers
36try:
37 from api.api_hub_endpoints import router as hub_router
38 HUB_ROUTER_AVAILABLE = True
39except ImportError as e:
40 logging.warning(f"Hub router not available: {e}")
41 HUB_ROUTER_AVAILABLE = False
42 hub_router = None
44try:
45 from api.blockchain_endpoints import router as blockchain_router
46 BLOCKCHAIN_ROUTER_AVAILABLE = True
47except ImportError as e:
48 logging.warning(f"Blockchain router not available: {e}")
49 BLOCKCHAIN_ROUTER_AVAILABLE = False
50 blockchain_router = None
52try:
53 from api.data_endpoints import router as data_router
54 DATA_ROUTER_AVAILABLE = True
55except ImportError as e:
56 logging.warning(f"Data router not available: {e}")
57 DATA_ROUTER_AVAILABLE = False
58 data_router = None
60try:
61 from api.integration_proxy_endpoints import router as integration_proxy_router
62 INTEGRATION_PROXY_AVAILABLE = True
63except ImportError as e:
64 logging.warning(f"Integration proxy router not available: {e}")
65 INTEGRATION_PROXY_AVAILABLE = False
66 integration_proxy_router = None
68# Import WebSocket routers
69try:
70 from api.websocket import router as websocket_router
71 WEBSOCKET_AVAILABLE = True
72except ImportError as e:
73 logging.warning(f"WebSocket router not available: {e}")
74 WEBSOCKET_AVAILABLE = False
75 websocket_router = None
77try:
78 from api.ws_unified_router import router as ws_unified_router
79 WS_UNIFIED_AVAILABLE = True
80except ImportError as e:
81 logging.warning(f"Unified WebSocket router not available: {e}")
82 WS_UNIFIED_AVAILABLE = False
83 ws_unified_router = None
85# Import services
86from backend.services.provider_registry_service import get_registry_service
87from backend.services.provider_fallback_manager import fallback_manager
88from backend.services.data_resolver import get_data_resolver
89from database.db_manager import db_manager
90from database.models import Base
92# Setup logging
93logging.basicConfig(
94 level=logging.INFO,
95 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
96)
97logger = logging.getLogger(__name__)
100@asynccontextmanager
101async def lifespan(app: FastAPI):
102 """Application lifespan - startup and shutdown"""
103 logger.info("=" * 70)
104 logger.info("🚀 Starting HuggingFace Space Backend")
105 logger.info("=" * 70)
107 # 1. Initialize database
108 logger.info("📊 Initializing database...")
109 try:
110 db_manager.init_database()
111 Base.metadata.create_all(bind=db_manager.engine)
112 logger.info("✅ Database initialized")
113 except Exception as e:
114 logger.error(f"❌ Database initialization failed: {e}")
115 raise
117 # 2. Build provider registry
118 logger.info("📋 Building provider registry...")
119 try:
120 registry_service = get_registry_service()
121 registry = registry_service.build_registry()
122 registry_service.save_to_config_file(registry)
123 registry_service.save_registry_json(registry)
124 logger.info(f"✅ Registry built: {registry['total_count']} providers ({registry['free_count']} free)")
125 except Exception as e:
126 logger.warning(f"⚠️ Registry build warning: {e}")
128 # 3. Initialize fallback manager
129 logger.info("🔄 Initializing fallback manager...")
130 try:
131 # Fallback manager is already initialized as singleton
132 status = fallback_manager.get_provider_status()
133 logger.info(f"✅ Fallback manager ready: {status['available_providers']}/{status['total_providers']} providers available")
134 except Exception as e:
135 logger.warning(f"⚠️ Fallback manager warning: {e}")
137 # 4. Initialize data resolver
138 logger.info("🔧 Initializing data resolver...")
139 try:
140 resolver = await get_data_resolver()
141 logger.info("✅ Data resolver initialized")
142 except Exception as e:
143 logger.warning(f"⚠️ Data resolver warning: {e}")
145 # 5. Initialize AI models (Hugging Face)
146 logger.info("🤖 Initializing AI models (Hugging Face)...")
147 try:
148 from ai_models import initialize_models, get_model_info
149 model_status = initialize_models()
150 model_info = get_model_info()
151 logger.info(f"✅ AI models initialized: {model_status.get('status', 'unknown')}")
152 logger.info(f" Models loaded: {model_status.get('models_loaded', 0)}")
153 logger.info(f" Models failed: {model_status.get('models_failed', 0)}")
154 logger.info(f" Total models: {model_info.get('total_models', 0)}")
155 if model_status.get('models_failed', 0) > 0:
156 logger.warning(f"⚠️ {model_status.get('models_failed', 0)} models failed to load")
157 except Exception as e:
158 logger.warning(f"⚠️ AI models initialization warning: {e}")
159 # Continue even if models fail - can use fallback
161 # 6. Start background workers
162 logger.info("⚙️ Starting background workers...")
163 try:
164 # Start market data worker (fetches from CoinGecko)
165 try:
166 from workers.market_data_worker import start_market_data_worker
167 await start_market_data_worker()
168 logger.info("✅ Market data worker started")
169 except Exception as e:
170 logger.warning(f"⚠️ Market data worker startup warning: {e}")
172 # Start OHLC data worker (fetches from Binance)
173 try:
174 from workers.ohlc_data_worker import start_ohlc_data_worker
175 await start_ohlc_data_worker()
176 logger.info("✅ OHLC data worker started")
177 except Exception as e:
178 logger.warning(f"⚠️ OHLC data worker startup warning: {e}")
180 # Start comprehensive data worker (fetches from ALL sources)
181 try:
182 from workers.comprehensive_data_worker import start_comprehensive_worker
183 await start_comprehensive_worker()
184 logger.info("✅ Comprehensive data worker started")
185 logger.info(" 📊 Collecting from 148+ data sources:")
186 logger.info(" - 23 Market Data APIs")
187 logger.info(" - 15 News APIs")
188 logger.info(" - 12 Sentiment APIs")
189 logger.info(" - 13 On-chain Analytics APIs")
190 logger.info(" - 9 Whale Tracking APIs")
191 logger.info(" - 18 Block Explorers")
192 except Exception as e:
193 logger.warning(f"⚠️ Comprehensive worker startup warning: {e}")
194 except Exception as e:
195 logger.error(f"❌ Worker startup error: {e}", exc_info=True)
196 # Don't fail on worker errors - they will retry
198 # 7. Start WebSocket streaming services (if available)
199 if WS_UNIFIED_AVAILABLE:
200 try:
201 from api.ws_unified_router import start_all_websocket_streams
202 await start_all_websocket_streams()
203 logger.info("✅ WebSocket streaming services started")
204 except Exception as e:
205 logger.warning(f"⚠️ WebSocket streaming services warning: {e}")
207 logger.info("=" * 70)
208 logger.info("✅ HuggingFace Space Backend is ready!")
209 logger.info("📍 API Documentation: /docs")
210 logger.info("📍 OpenAPI Schema: /openapi.json")
211 if WEBSOCKET_AVAILABLE or WS_UNIFIED_AVAILABLE:
212 logger.info("📍 WebSocket Endpoints: /ws, /ws/live, /ws/master, /ws/all")
213 logger.info("=" * 70)
215 yield
217 # Shutdown
218 logger.info("🛑 Shutting down...")
219 await fallback_manager.close()
221 # Close WebSocket connections
222 if WEBSOCKET_AVAILABLE:
223 try:
224 from api.websocket import manager
225 await manager.close_all_connections()
226 await manager.stop_background_tasks()
227 logger.info("✅ WebSocket connections closed")
228 except Exception as e:
229 logger.warning(f"⚠️ WebSocket shutdown warning: {e}")
232# Create FastAPI app
233app = FastAPI(
234 title="Crypto Intelligence Hub API",
235 description="Complete backend for Crypto Intelligence Hub - AI-Powered Data Center",
236 version="1.0.0",
237 lifespan=lifespan,
238 docs_url="/docs",
239 openapi_url="/openapi.json"
240)
242# Add exception handler for WebSocket connections
243from starlette.websockets import WebSocketState
244from fastapi.exceptions import RequestValidationError
245from starlette.exceptions import HTTPException as StarletteHTTPException
246import json
247import time
249@app.exception_handler(StarletteHTTPException)
250async def http_exception_handler(request: Request, exc: StarletteHTTPException):
251 # #region agent log
252 if request.url.path.startswith("/ws"):
253 try:
254 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
255 f.write(json.dumps({"timestamp": int(time.time() * 1000), "sessionId": "debug-session", "runId": "run1", "hypothesisId": "G", "location": "hf_space_main.py:224", "message": "HTTPException for WebSocket", "data": {"path": request.url.path, "status_code": exc.status_code, "detail": str(exc.detail)}}) + "\n")
256 except: pass
257 # #endregion
258 from fastapi.responses import JSONResponse
259 return JSONResponse(
260 status_code=exc.status_code,
261 content={"detail": exc.detail}
262 )
264# CORS middleware - Aggressively configured for Hugging Face Spaces
265# Wildcard is necessary for HF Spaces dynamic subdomains and reverse proxy
266app.add_middleware(
267 CORSMiddleware,
268 allow_origins=["*"], # Wildcard required for HF Spaces
269 allow_credentials=True,
270 allow_methods=["*"],
271 allow_headers=["*"],
272 expose_headers=["*"],
273)
275# WebSocket debugging middleware - Enhanced with header inspection and CORS fixes
276class WebSocketDebugMiddleware(BaseHTTPMiddleware):
277 """Debug middleware to log all WebSocket connection attempts and fix headers"""
278 async def dispatch(self, request: Request, call_next):
279 # Log WebSocket connection attempts with full header details
280 if request.url.path.startswith("/ws"):
281 import json
282 import time
283 origin = request.headers.get("origin")
284 upgrade = request.headers.get("upgrade", "").lower()
285 connection = request.headers.get("connection", "").lower()
287 logger.info(f"WebSocket request: {request.url.path} | Origin: {origin} | Upgrade: {upgrade} | Connection: {connection}")
289 # Log to debug file
290 try:
291 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
292 f.write(json.dumps({
293 "timestamp": int(time.time() * 1000),
294 "sessionId": "debug-session",
295 "runId": "run1",
296 "hypothesisId": "H",
297 "location": "hf_space_main.py:WebSocketDebugMiddleware",
298 "message": "WebSocket request received",
299 "data": {
300 "path": request.url.path,
301 "method": request.method,
302 "origin": origin,
303 "upgrade": upgrade,
304 "connection": connection,
305 "headers": dict(request.headers),
306 "client": str(request.client)
307 }
308 }) + "\n")
309 except: pass
310 # #endregion
311 try:
312 response = await call_next(request)
314 # Add WebSocket-friendly headers for successful connections
315 if request.url.path.startswith("/ws"):
316 # Ensure CORS headers are set for WebSocket handshake
317 origin = request.headers.get("origin")
318 if origin:
319 response.headers["Access-Control-Allow-Origin"] = origin
320 response.headers["Access-Control-Allow-Credentials"] = "true"
322 # Log successful processing
323 try:
324 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
325 f.write(json.dumps({
326 "timestamp": int(time.time() * 1000),
327 "sessionId": "debug-session",
328 "runId": "run1",
329 "hypothesisId": "H",
330 "location": "hf_space_main.py:WebSocketDebugMiddleware",
331 "message": "WebSocket request processed",
332 "data": {
333 "path": request.url.path,
334 "status_code": response.status_code if hasattr(response, 'status_code') else None,
335 "origin": origin
336 }
337 }) + "\n")
338 except: pass
340 return response
341 except Exception as e:
342 # Log exceptions with full context
343 if request.url.path.startswith("/ws"):
344 origin = request.headers.get("origin")
345 logger.error(f"WebSocket error on {request.url.path} from origin {origin}: {e}", exc_info=True)
346 try:
347 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
348 f.write(json.dumps({
349 "timestamp": int(time.time() * 1000),
350 "sessionId": "debug-session",
351 "runId": "run1",
352 "hypothesisId": "H",
353 "location": "hf_space_main.py:WebSocketDebugMiddleware",
354 "message": "WebSocket request exception",
355 "data": {
356 "path": request.url.path,
357 "error": str(e),
358 "error_type": type(e).__name__,
359 "origin": origin,
360 "headers": dict(request.headers)
361 }
362 }) + "\n")
363 except: pass
364 raise
366app.add_middleware(WebSocketDebugMiddleware)
368# Permissions-Policy middleware - fixes browser warnings
369class PermissionsPolicyMiddleware(BaseHTTPMiddleware):
370 """
371 Middleware to sanitize and set a safe Permissions-Policy header.
373 Removes:
374 - Legacy Feature-Policy header (deprecated)
375 - Unsupported/removed features: battery, document-domain, legacy-image-formats,
376 oversized-images, vr, wake-lock, ambient-light-sensor, layout-animations
378 Sets a minimal, widely-supported policy with only standard features that
379 are recognized by all modern browsers.
380 """
381 # List of unsupported/problematic features to remove
382 UNSUPPORTED_FEATURES = {
383 'battery', # Deprecated Battery Status API
384 'document-domain', # Very old, not recognized
385 'legacy-image-formats', # Not a valid permission token
386 'oversized-images', # Not a valid permission token (Lighthouse hint)
387 'vr', # Replaced by WebXR (xr-spatial-tracking)
388 'wake-lock', # Should be screen-wake-lock if needed
389 'ambient-light-sensor', # May not be supported in all builds
390 'layout-animations', # Experimental, older browsers don't recognize
391 }
393 async def dispatch(self, request: Request, call_next):
394 # #region agent log
395 import json
396 import time
397 is_websocket = request.url.path.startswith("/ws")
398 try:
399 with open(".cursor/debug.log", "a", encoding="utf-8") as f:
400 f.write(json.dumps({"timestamp": int(time.time() * 1000), "sessionId": "debug-session", "runId": "run1", "hypothesisId": "C", "location": "hf_space_main.py:265", "message": "PermissionsPolicyMiddleware before call_next", "data": {"path": request.url.path, "method": request.method, "is_websocket": is_websocket}}) + "\n")
401 except: pass
402 # #endregion
403 try:
404 response = await call_next(request)
405 # #region agent log
406 try:
407 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
408 f.write(json.dumps({"timestamp": int(time.time() * 1000), "sessionId": "debug-session", "runId": "run1", "hypothesisId": "C", "location": "hf_space_main.py:266", "message": "PermissionsPolicyMiddleware after call_next", "data": {"path": request.url.path, "status_code": response.status_code if hasattr(response, 'status_code') else None, "is_websocket": is_websocket}}) + "\n")
409 except: pass
410 # #endregion
411 except Exception as e:
412 # #region agent log
413 try:
414 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
415 f.write(json.dumps({"timestamp": int(time.time() * 1000), "sessionId": "debug-session", "runId": "run1", "hypothesisId": "C", "location": "hf_space_main.py:266", "message": "PermissionsPolicyMiddleware exception", "data": {"path": request.url.path, "error": str(e), "error_type": type(e).__name__, "is_websocket": is_websocket}}) + "\n")
416 except: pass
417 # #endregion
418 raise
420 # Remove legacy Feature-Policy header (deprecated, replaced by Permissions-Policy)
421 if 'Feature-Policy' in response.headers:
422 del response.headers['Feature-Policy']
424 # Remove any existing Permissions-Policy that might contain unsupported features
425 if 'Permissions-Policy' in response.headers:
426 existing_policy = response.headers['Permissions-Policy']
427 # Check if it contains any unsupported features
428 if any(feature in existing_policy for feature in self.UNSUPPORTED_FEATURES):
429 # Remove the problematic header - we'll set a clean one below
430 del response.headers['Permissions-Policy']
432 # Set a minimal, safe Permissions-Policy with only widely-supported features
433 # Features included:
434 # - geolocation: Standard, widely supported
435 # - microphone: Standard, widely supported
436 # - camera: Standard, widely supported
437 # All set to () (deny) since this app doesn't need these permissions
438 # If you need to allow specific origins, use: geolocation=(self "https://example.com")
439 safe_policy = "geolocation=(), microphone=(), camera=()"
440 response.headers['Permissions-Policy'] = safe_policy
442 return response
444app.add_middleware(PermissionsPolicyMiddleware)
446# Include routers
447app.include_router(service_router)
448app.include_router(models_router)
449app.include_router(providers_router)
450app.include_router(market_router)
451app.include_router(sentiment_router)
452app.include_router(news_router)
453app.include_router(whales_router)
454app.include_router(onchain_router)
455app.include_router(system_router)
457# Include additional API routers
458if HUB_ROUTER_AVAILABLE and hub_router:
459 app.include_router(hub_router)
460 logger.info("✅ Hub API router loaded")
462if BLOCKCHAIN_ROUTER_AVAILABLE and blockchain_router:
463 app.include_router(blockchain_router)
464 logger.info("✅ Blockchain endpoints router loaded")
466if DATA_ROUTER_AVAILABLE and data_router:
467 app.include_router(data_router)
468 logger.info("✅ Data endpoints router loaded")
470if INTEGRATION_PROXY_AVAILABLE and integration_proxy_router:
471 app.include_router(integration_proxy_router)
472 logger.info("✅ Integration proxy router loaded")
474# Include WebSocket routers
475if WEBSOCKET_AVAILABLE and websocket_router:
476 app.include_router(websocket_router)
477 logger.info("✅ WebSocket router loaded")
479if WS_UNIFIED_AVAILABLE and ws_unified_router:
480 app.include_router(ws_unified_router)
481 logger.info("✅ Unified WebSocket router loaded")
482 # #region agent log
483 import json
484 import time
485 try:
486 with open(_get_debug_log_path(), "a", encoding="utf-8") as f:
487 f.write(json.dumps({"timestamp": int(time.time() * 1000), "sessionId": "debug-session", "runId": "run1", "hypothesisId": "E", "location": "hf_space_main.py:348", "message": "WebSocket routers included", "data": {"websocket_available": WEBSOCKET_AVAILABLE, "ws_unified_available": WS_UNIFIED_AVAILABLE}}) + "\n")
488 except: pass
489 # #endregion
491# Serve static files if they exist
492static_dir = Path("static")
493if static_dir.exists():
494 app.mount("/static", StaticFiles(directory="static"), name="static")
496# Serve index.html at root
497index_path = Path("index.html")
498if not index_path.exists():
499 # Try /mnt/data/index.html
500 index_path = Path("/mnt/data/index.html")
502if index_path.exists():
503 @app.get("/")
504 async def read_root():
505 return FileResponse(index_path)
507 @app.get("/index.html")
508 async def read_index():
509 return FileResponse(index_path)
512# Health and status endpoints
513@app.get("/health")
514async def health():
515 """Health check endpoint"""
516 return {
517 "status": "healthy",
518 "service": "Crypto Intelligence Hub API",
519 "version": "1.0.0"
520 }
523@app.get("/api/health")
524async def api_health():
525 """API health check"""
526 return {
527 "status": "healthy",
528 "database": "connected" if db_manager else "disconnected",
529 "providers": fallback_manager.get_provider_status()["available_providers"] if fallback_manager else 0
530 }
533# /api/status is handled by system_router
536if __name__ == "__main__":
537 import uvicorn
539 port = int(os.getenv("PORT", "7860"))
540 host = os.getenv("HOST", "0.0.0.0")
542 logger.info(f"Starting server on {host}:{port}")
544 uvicorn.run(
545 "hf_space_main:app",
546 host=host,
547 port=port,
548 log_level="info",
549 reload=False
550 )