utils/free_providers_loader.py

#!/usr/bin/env python3
"""
Free Providers Loader
=====================
بارگذاری همه منابع رایگان از all_apis_merged_2025.json و تبدیل به فرمت config.py
"""

import json
import os
import re
import logging
from typing import Dict, List, Any, Optional
from pathlib import Path

logger = logging.getLogger(__name__)

# مسیر فایل registry
REGISTRY_FILE_PATHS = [
    "all_apis_merged_2025.json",
    "/mnt/data/all_apis_merged_2025.json",
    os.path.join(os.getcwd(), "all_apis_merged_2025.json")
]


def load_registry_file() -> Dict[str, Any]:
    """بارگذاری فایل registry"""
    for path in REGISTRY_FILE_PATHS:
        if os.path.exists(path):
            try:
                with open(path, 'r', encoding='utf-8') as f:
                    data = json.load(f)
                logger.info(f"✅ Loaded registry from {path}")
                return data
            except Exception as e:
                logger.error(f"❌ Failed to load {path}: {e}")
                continue
    
    logger.warning("⚠️ Registry file not found")
    return {}


def extract_free_providers_from_content(content: str) -> List[Dict[str, Any]]:
    """استخراج provider های رایگان از محتوای متنی"""
    providers = []
    
    # Market Data APIs (FREE)
    free_market_apis = [
        {
            "id": "coingecko",
            "name": "CoinGecko",
            "base_url": "https://api.coingecko.com/api/v3",
            "category": "market_data",
            "free": True,
            "rate_limit": {"requests_per_minute": 50}
        },
        {
            "id": "coincap",
            "name": "CoinCap",
            "base_url": "https://api.coincap.io/v2",
            "category": "market_data",
            "free": True,
            "rate_limit": {"requests_per_minute": 200}
        },
        {
            "id": "coinpaprika",
            "name": "CoinPaprika",
            "base_url": "https://api.coinpaprika.com/v1",
            "category": "market_data",
            "free": True,
            "rate_limit": {"requests_per_month": 20000}
        },
        {
            "id": "binance_public",
            "name": "Binance Public API",
            "base_url": "https://api.binance.com/api/v3",
            "category": "market_data",
            "free": True
        },
        {
            "id": "nomics",
            "name": "Nomics",
            "base_url": "https://api.nomics.com/v1",
            "category": "market_data",
            "free": True
        },
        {
            "id": "messari",
            "name": "Messari",
            "base_url": "https://data.messari.io/api/v1",
            "category": "market_data",
            "free": True
        },
        {
            "id": "coinlore",
            "name": "CoinLore",
            "base_url": "https://api.coinlore.net/api",
            "category": "market_data",
            "free": True
        },
        {
            "id": "coindesk",
            "name": "CoinDesk",
            "base_url": "https://api.coindesk.com/v1",
            "category": "market_data",
            "free": True
        },
        {
            "id": "mobula",
            "name": "Mobula API",
            "base_url": "https://api.mobula.io/api/1",
            "category": "market_data",
            "free": True
        },
        {
            "id": "diadata",
            "name": "DIA Data",
            "base_url": "https://api.diadata.org/v1",
            "category": "market_data",
            "free": True
        },
        {
            "id": "coinstats",
            "name": "CoinStats",
            "base_url": "https://api.coinstats.app/public/v1",
            "category": "market_data",
            "free": True
        }
    ]
    
    # News APIs (FREE)
    free_news_apis = [
        {
            "id": "cryptopanic",
            "name": "CryptoPanic",
            "base_url": "https://cryptopanic.com/api/v1",
            "category": "news",
            "free": True
        },
        {
            "id": "reddit_crypto",
            "name": "Reddit Crypto",
            "base_url": "https://www.reddit.com/r/CryptoCurrency",
            "category": "news",
            "free": True,
            "rate_limit": {"requests_per_minute": 60}
        },
        {
            "id": "cryptocontrol",
            "name": "CryptoControl",
            "base_url": "https://cryptocontrol.io/api/v1/public",
            "category": "news",
            "free": True
        },
        {
            "id": "coindesk_rss",
            "name": "CoinDesk RSS",
            "base_url": "https://www.coindesk.com/arc/outboundfeeds/rss",
            "category": "news",
            "free": True
        },
        {
            "id": "cointelegraph_rss",
            "name": "CoinTelegraph RSS",
            "base_url": "https://cointelegraph.com/rss",
            "category": "news",
            "free": True
        },
        {
            "id": "decrypt_rss",
            "name": "Decrypt RSS",
            "base_url": "https://decrypt.co/feed",
            "category": "news",
            "free": True
        },
        {
            "id": "bitcoin_magazine_rss",
            "name": "Bitcoin Magazine RSS",
            "base_url": "https://bitcoinmagazine.com/.rss/full",
            "category": "news",
            "free": True
        }
    ]
    
    # Sentiment APIs (FREE)
    free_sentiment_apis = [
        {
            "id": "alternative_me_fng",
            "name": "Alternative.me Fear & Greed",
            "base_url": "https://api.alternative.me/fng",
            "category": "sentiment",
            "free": True
        },
        {
            "id": "cfgi",
            "name": "CFGI API",
            "base_url": "https://api.cfgi.io/v1",
            "category": "sentiment",
            "free": True
        },
        {
            "id": "coingecko_community",
            "name": "CoinGecko Community Data",
            "base_url": "https://api.coingecko.com/api/v3",
            "category": "sentiment",
            "free": True
        },
        {
            "id": "messari_social",
            "name": "Messari Social",
            "base_url": "https://data.messari.io/api/v1",
            "category": "sentiment",
            "free": True
        }
    ]
    
    # Block Explorer APIs (FREE)
    free_explorer_apis = [
        {
            "id": "blockscout_eth",
            "name": "BlockScout (Ethereum)",
            "base_url": "https://eth.blockscout.com/api",
            "category": "blockchain_explorer",
            "free": True
        },
        {
            "id": "blockchair_eth",
            "name": "Blockchair (Ethereum)",
            "base_url": "https://api.blockchair.com/ethereum",
            "category": "blockchain_explorer",
            "free": True,
            "rate_limit": {"requests_per_day": 1440}
        },
        {
            "id": "blockchair_bsc",
            "name": "Blockchair (BSC)",
            "base_url": "https://api.blockchair.com/binance-smart-chain",
            "category": "blockchain_explorer",
            "free": True,
            "rate_limit": {"requests_per_day": 1440}
        },
        {
            "id": "blockchair_tron",
            "name": "Blockchair (TRON)",
            "base_url": "https://api.blockchair.com/tron",
            "category": "blockchain_explorer",
            "free": True,
            "rate_limit": {"requests_per_day": 1440}
        },
        {
            "id": "ethplorer",
            "name": "Ethplorer",
            "base_url": "https://api.ethplorer.io",
            "category": "blockchain_explorer",
            "free": True,
            "api_key": "freekey"
        },
        {
            "id": "trongrid",
            "name": "TronGrid",
            "base_url": "https://api.trongrid.io",
            "category": "blockchain_explorer",
            "free": True
        },
        {
            "id": "tronstack",
            "name": "TronStack",
            "base_url": "https://api.tronstack.io",
            "category": "blockchain_explorer",
            "free": True
        }
    ]
    
    # Whale Tracking APIs (FREE)
    free_whale_apis = [
        {
            "id": "clankapp",
            "name": "ClankApp",
            "base_url": "https://clankapp.com/api",
            "category": "whale_tracking",
            "free": True
        },
        {
            "id": "bitquery",
            "name": "BitQuery",
            "base_url": "https://graphql.bitquery.io",
            "category": "whale_tracking",
            "free": True,
            "rate_limit": {"queries_per_month": 10000}
        }
    ]
    
    # RPC Nodes (FREE)
    free_rpc_nodes = [
        {
            "id": "publicnode_eth",
            "name": "PublicNode (Ethereum)",
            "base_url": "https://ethereum.publicnode.com",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "ankr_eth",
            "name": "Ankr (Ethereum)",
            "base_url": "https://rpc.ankr.com/eth",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "llamanodes_eth",
            "name": "LlamaNodes (Ethereum)",
            "base_url": "https://eth.llamarpc.com",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "cloudflare_eth",
            "name": "Cloudflare (Ethereum)",
            "base_url": "https://cloudflare-eth.com",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "drpc_eth",
            "name": "dRPC (Ethereum)",
            "base_url": "https://eth.drpc.org",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "bsc_official",
            "name": "BSC Official RPC",
            "base_url": "https://bsc-dataseed.binance.org",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "ankr_bsc",
            "name": "Ankr (BSC)",
            "base_url": "https://rpc.ankr.com/bsc",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "publicnode_bsc",
            "name": "PublicNode (BSC)",
            "base_url": "https://bsc-rpc.publicnode.com",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "polygon_official",
            "name": "Polygon Official",
            "base_url": "https://polygon-rpc.com",
            "category": "rpc_node",
            "free": True
        },
        {
            "id": "ankr_polygon",
            "name": "Ankr (Polygon)",
            "base_url": "https://rpc.ankr.com/polygon",
            "category": "rpc_node",
            "free": True
        }
    ]
    
    # On-Chain Analytics (FREE)
    free_onchain_apis = [
        {
            "id": "thegraph",
            "name": "The Graph",
            "base_url": "https://api.thegraph.com/subgraphs/name",
            "category": "onchain_analytics",
            "free": True
        },
        {
            "id": "intotheblock",
            "name": "IntoTheBlock",
            "base_url": "https://api.intotheblock.com/v1",
            "category": "onchain_analytics",
            "free": True
        }
    ]
    
    all_free = (
        free_market_apis +
        free_news_apis +
        free_sentiment_apis +
        free_explorer_apis +
        free_whale_apis +
        free_rpc_nodes +
        free_onchain_apis
    )
    
    return all_free


def get_all_free_providers() -> Dict[str, Any]:
    """دریافت همه provider های رایگان"""
    registry_data = load_registry_file()
    
    # استخراج از محتوای فایل
    providers = []
    raw_files = registry_data.get('raw_files', [])
    for file_data in raw_files:
        content = file_data.get('content', '')
        providers.extend(extract_free_providers_from_content(content))
    
    # استخراج کلیدها
    discovered_keys = registry_data.get('discovered_keys', {})
    
    return {
        "providers": providers,
        "discovered_keys": discovered_keys,
        "total_count": len(providers),
        "free_count": len(providers)  # همه رایگان هستند
    }


def convert_to_config_format(providers_data: Dict[str, Any]) -> Dict[str, Any]:
    """تبدیل به فرمت EXTERNAL_PROVIDERS در config.py"""
    providers = providers_data.get("providers", [])
    discovered_keys = providers_data.get("discovered_keys", {})
    
    config_providers = {}
    
    for provider in providers:
        provider_id = provider.get("id", "").lower()
        if not provider_id:
            continue
        
        # استخراج API key از discovered_keys
        api_key = None
        if provider_id in discovered_keys:
            keys = discovered_keys[provider_id]
            if keys:
                api_key = keys[0]
        elif "etherscan" in provider_id:
            keys = discovered_keys.get("etherscan", [])
            if keys:
                api_key = keys[0]
        elif "bscscan" in provider_id or "bsc" in provider_id:
            keys = discovered_keys.get("bscscan", [])
            if keys:
                api_key = keys[0]
        elif "tronscan" in provider_id or "tron" in provider_id:
            keys = discovered_keys.get("tronscan", [])
            if keys:
                api_key = keys[0]
        
        # تنظیمات rate limit
        rate_limit = provider.get("rate_limit", {})
        if isinstance(rate_limit, dict):
            rate_limit_config = {}
            if "requests_per_minute" in rate_limit:
                rate_limit_config["requests_per_minute"] = rate_limit["requests_per_minute"]
            if "requests_per_day" in rate_limit:
                rate_limit_config["requests_per_day"] = rate_limit["requests_per_day"]
            if "requests_per_month" in rate_limit:
                rate_limit_config["requests_per_month"] = rate_limit["requests_per_month"]
            if "queries_per_month" in rate_limit:
                rate_limit_config["queries_per_month"] = rate_limit["queries_per_month"]
        else:
            rate_limit_config = {}
        
        config_providers[provider_id] = {
            "enabled": True,
            "api_key": os.getenv(f"{provider_id.upper()}_API_KEY") or api_key or provider.get("api_key"),
            "base_url": provider.get("base_url", ""),
            "timeout": 10.0,
            "priority": 3,  # Free providers have lower priority
            "category": provider.get("category", "generic"),
            "rate_limit": rate_limit_config or {
                "requests_per_minute": 10,
                "requests_per_day": 1000
            }
        }
    
    return config_providers