#!/usr/bin/env python3 """ Health Monitoring System Continuous health monitoring for all API endpoints """ import schedule import time import requests import json import logging from datetime import datetime from typing import Dict, List, Optional from pathlib import Path logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class HealthMonitor: """Continuous health monitoring for all endpoints""" def __init__(self, base_url: str = "http://localhost:7860"): self.base_url = base_url self.endpoints = self.load_endpoints() self.health_history = [] self.alert_threshold = 3 # Number of consecutive failures before alert self.failure_counts = {} # Track consecutive failures per endpoint def load_endpoints(self) -> List[Dict]: """Load endpoints from service registry""" registry_file = Path("config/service_registry.json") if not registry_file.exists(): logger.warning("⚠ Service registry not found, using default endpoints") return self._get_default_endpoints() try: with open(registry_file, 'r') as f: registry = json.load(f) endpoints = [] for service in registry.get("services", []): for endpoint in service.get("endpoints", []): endpoints.append({ "path": endpoint.get("path", ""), "method": endpoint.get("method", "GET"), "category": service.get("category", "unknown"), "service_id": service.get("id", "unknown"), "base_url": self.base_url }) return endpoints except Exception as e: logger.error(f"❌ Failed to load endpoints: {e}") return self._get_default_endpoints() def _get_default_endpoints(self) -> List[Dict]: """Get default endpoints for monitoring""" return [ {"path": "/api/health", "method": "GET", "category": "system", "base_url": self.base_url}, {"path": "/api/ohlcv/BTC", "method": "GET", "category": "market_data", "base_url": self.base_url}, {"path": "/api/v1/ohlcv/BTC", "method": "GET", "category": "market_data", "base_url": self.base_url}, {"path": "/api/market/ohlcv", "method": "GET", "category": "market_data", "base_url": self.base_url, "params": {"symbol": "BTC", "interval": "1d", "limit": 30}}, ] def check_endpoint_health(self, endpoint: Dict) -> Dict: """Check health of single endpoint""" path = endpoint["path"] method = endpoint.get("method", "GET").upper() params = endpoint.get("params", {}) try: start_time = time.time() url = f"{endpoint['base_url']}{path}" if method == "GET": response = requests.get(url, params=params, timeout=10) elif method == "POST": response = requests.post(url, json=params, timeout=10) else: response = requests.request(method, url, json=params, timeout=10) response_time = (time.time() - start_time) * 1000 is_healthy = response.status_code in [200, 201] result = { "endpoint": path, "status": "healthy" if is_healthy else "degraded", "status_code": response.status_code, "response_time_ms": round(response_time, 2), "timestamp": datetime.now().isoformat(), "method": method } # Update failure count if is_healthy: self.failure_counts[path] = 0 else: self.failure_counts[path] = self.failure_counts.get(path, 0) + 1 result["consecutive_failures"] = self.failure_counts[path] return result except requests.exceptions.Timeout: self.failure_counts[path] = self.failure_counts.get(path, 0) + 1 return { "endpoint": path, "status": "down", "error": "timeout", "timestamp": datetime.now().isoformat(), "method": method, "consecutive_failures": self.failure_counts[path] } except Exception as e: self.failure_counts[path] = self.failure_counts.get(path, 0) + 1 return { "endpoint": path, "status": "down", "error": str(e), "timestamp": datetime.now().isoformat(), "method": method, "consecutive_failures": self.failure_counts[path] } def check_all_endpoints(self): """Check health of all registered endpoints""" results = [] logger.info(f"🔍 Checking {len(self.endpoints)} endpoints...") for endpoint in self.endpoints: health = self.check_endpoint_health(endpoint) results.append(health) # Check if alert needed if health['status'] != "healthy": self.handle_unhealthy_endpoint(health) # Store in history self.health_history.append({ "check_time": datetime.now().isoformat(), "results": results, "summary": { "total": len(results), "healthy": sum(1 for r in results if r['status'] == "healthy"), "degraded": sum(1 for r in results if r['status'] == "degraded"), "down": sum(1 for r in results if r['status'] == "down") } }) # Keep only last 100 checks if len(self.health_history) > 100: self.health_history = self.health_history[-100:] # Save to file self.save_health_report(results) return results def handle_unhealthy_endpoint(self, health: Dict): """Handle unhealthy endpoint detection""" path = health["endpoint"] consecutive_failures = health.get("consecutive_failures", 0) if consecutive_failures >= self.alert_threshold: self.send_alert(health) def send_alert(self, health: Dict): """Send alert about failing endpoint""" alert_message = f""" ⚠️ ALERT: Endpoint Health Issue Endpoint: {health['endpoint']} Status: {health['status']} Error: {health.get('error', 'N/A')} Time: {health['timestamp']} Consecutive Failures: {health.get('consecutive_failures', 0)} """ logger.error(alert_message) # Save alert to file alerts_file = Path("monitoring/alerts.json") alerts_file.parent.mkdir(parents=True, exist_ok=True) try: if alerts_file.exists(): with open(alerts_file, 'r') as f: alerts = json.load(f) else: alerts = [] alerts.append({ "timestamp": datetime.now().isoformat(), "endpoint": health["endpoint"], "status": health["status"], "error": health.get("error"), "consecutive_failures": health.get("consecutive_failures", 0) }) # Keep only last 50 alerts alerts = alerts[-50:] with open(alerts_file, 'w') as f: json.dump(alerts, f, indent=2) except Exception as e: logger.error(f"Failed to save alert: {e}") def save_health_report(self, results: List[Dict]): """Save health check results to file""" reports_dir = Path("monitoring/reports") reports_dir.mkdir(parents=True, exist_ok=True) report_file = reports_dir / f"health_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" report = { "timestamp": datetime.now().isoformat(), "total_endpoints": len(results), "healthy": sum(1 for r in results if r['status'] == "healthy"), "degraded": sum(1 for r in results if r['status'] == "degraded"), "down": sum(1 for r in results if r['status'] == "down"), "results": results } try: with open(report_file, 'w') as f: json.dump(report, f, indent=2) # Also update latest report latest_file = reports_dir / "health_report_latest.json" with open(latest_file, 'w') as f: json.dump(report, f, indent=2) except Exception as e: logger.error(f"Failed to save health report: {e}") def get_health_summary(self) -> Dict: """Get summary of health status""" if not self.health_history: return { "status": "unknown", "message": "No health checks performed yet" } latest = self.health_history[-1] summary = latest["summary"] total = summary["total"] healthy = summary["healthy"] health_percentage = (healthy / total * 100) if total > 0 else 0 return { "status": "healthy" if health_percentage >= 95 else "degraded" if health_percentage >= 80 else "unhealthy", "health_percentage": round(health_percentage, 2), "total_endpoints": total, "healthy": healthy, "degraded": summary["degraded"], "down": summary["down"], "last_check": latest["check_time"] } def start_monitoring(self, interval_minutes: int = 5): """Start continuous monitoring""" logger.info(f"🔍 Health monitoring started (checking every {interval_minutes} minutes)") logger.info(f"📊 Monitoring {len(self.endpoints)} endpoints") # Run initial check self.check_all_endpoints() # Schedule periodic checks schedule.every(interval_minutes).minutes.do(self.check_all_endpoints) try: while True: schedule.run_pending() time.sleep(1) except KeyboardInterrupt: logger.info("🛑 Health monitoring stopped") if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="Health Monitoring System") parser.add_argument("--base-url", default="http://localhost:7860", help="Base URL for API") parser.add_argument("--interval", type=int, default=5, help="Check interval in minutes") parser.add_argument("--once", action="store_true", help="Run once and exit") args = parser.parse_args() monitor = HealthMonitor(base_url=args.base_url) if args.once: results = monitor.check_all_endpoints() summary = monitor.get_health_summary() print("\n" + "="*50) print("HEALTH SUMMARY") print("="*50) print(json.dumps(summary, indent=2)) print("="*50) else: monitor.start_monitoring(interval_minutes=args.interval)