feat: Add delay between API requests for block retrieval

app.py

@@ -5,248 +5,305 @@ import plotly.express as px
 from datetime import datetime, timedelta
 import json
 from web3 import Web3
-import time
-# RPC URLs
+
 OPTIMISM_RPC_URL = 'https://opt-mainnet.g.alchemy.com/v2/U5gnXPYxeyH43MJ9tP8ONBQHEDRav7H0'
-BASE_RPC_URL = 'https://base-mainnet.g.alchemy.com/v2/U5gnXPYxeyH43MJ9tP8ONBQHEDRav7H0'
-ETH_RPC_URL = 'https://eth-mainnet.g.alchemy.com/v2/U5gnXPYxeyH43MJ9tP8ONBQHEDRav7H0'
 
-# Initialize Web3 instances
-print("Initializing Web3 instances...")
-web3_optimism = Web3(Web3.HTTPProvider(OPTIMISM_RPC_URL))
-web3_base = Web3(Web3.HTTPProvider(BASE_RPC_URL))
-web3_eth = Web3(Web3.HTTPProvider(ETH_RPC_URL))
+# Initialize a Web3 instance
+web3 = Web3(Web3.HTTPProvider(OPTIMISM_RPC_URL))
+
+# Check if connection is successful
+if not web3.is_connected():
+    raise Exception("Failed to connect to the Optimism network.")
 
-# Contract addresses for service registries
-contract_address_optimism = '0x3d77596beb0f130a4415df3D2D8232B3d3D31e44'
-contract_address_base = '0x3C1fF68f5aa342D296d4DEe4Bb1cACCA912D95fE'
-contract_address_eth = '0x48b6af7B12C71f09e2fC8aF4855De4Ff54e775cA'
+# Contract address
+contract_address = '0x3d77596beb0f130a4415df3D2D8232B3d3D31e44'
 
-# Load the ABI from a local JSON file
+# Load the ABI from the provided JSON file
 with open('service_registry_abi.json', 'r') as abi_file:
     contract_abi = json.load(abi_file)
 
-# Create the contract instances
-service_registry_optimism = web3_optimism.eth.contract(address=contract_address_optimism, abi=contract_abi)
-service_registry_base = web3_base.eth.contract(address=contract_address_base, abi=contract_abi)
-service_registry_eth = web3_eth.eth.contract(address=contract_address_eth, abi=contract_abi)
-print("Service registry contracts loaded.")
+# Now you can create the contract
+service_registry = web3.eth.contract(address=contract_address, abi=contract_abi)
+
+def get_transfers(integrator: str, wallet: str) -> str:
+    url = f"https://li.quest/v1/analytics/transfers?integrator={integrator}&wallet={wallet}"
+    headers = {"accept": "application/json"}
+    response = requests.get(url, headers=headers)
+    return response.json()
+
+def load_activity_checker_contract(w3, staking_token_address):
+    """
+    Loads the Staking Token and Activity Checker contracts.
+
+    :param w3: Web3 instance
+    :param staking_token_address: Address of the staking token contract
+    :return: Tuple of (Staking Token contract instance, Activity Checker contract instance)
+    """
+    try:
+        # Load the ABI file for the Staking Token contract
+        with open('./contracts/StakingToken.json', "r", encoding="utf-8") as file:
+            staking_token_data = json.load(file)
+        
+        staking_token_abi = staking_token_data.get("abi", [])
+
+        # Create the Staking Token contract instance
+        staking_token_contract = w3.eth.contract(address=staking_token_address, abi=staking_token_abi)
+        
+        # Get the activity checker contract address from staking_token_contract
+        activity_checker_address = staking_token_contract.functions.activityChecker().call()
+
+        # Load the ABI file for the Activity Checker contract
+        with open('./contracts/StakingActivityChecker.json', "r", encoding="utf-8") as file:
+            activity_checker_data = json.load(file)
+        
+        activity_checker_abi = activity_checker_data.get("abi", [])
+
+        # Create the Activity Checker contract instance
+        activity_checker_contract = w3.eth.contract(address=activity_checker_address, abi=activity_checker_abi)
+        
+        return staking_token_contract, activity_checker_contract
+    
+    except Exception as e:
+        print(f"An error occurred while loading the contracts: {e}")
+        raise
 
-# Check if connection is successful
-if not web3_optimism.is_connected():
-    raise Exception("Failed to connect to the Optimism network.")
-if not web3_base.is_connected():
-    raise Exception("Failed to connect to the Base network.")
-if not web3_eth.is_connected():
-    raise Exception("Failed to connect to the ETH network.")
-print("Successfully connected to Ethereum, Optimism, and Base networks.")
 
+def fetch_and_aggregate_transactions():
+    total_services = service_registry.functions.totalSupply().call()
+    aggregated_transactions = []
+    daily_agent_counts = {}
+    daily_agents_with_transactions = {}
 
-def fetch_service_safes(web3, registry_contract):
-    print("\nFetching service safes...")
-    total_services = registry_contract.functions.totalSupply().call()
-    print(f"Total services: {total_services}")
-    service_safes = set()
+    _staking_token_contract, activity_checker_contract = load_activity_checker_contract(web3, '0x88996bbdE7f982D93214881756840cE2c77C4992')
 
     for service_id in range(1, total_services + 1):
-        print(f"Processing service ID: {service_id}")
-        service = registry_contract.functions.getService(service_id).call()
+        service = service_registry.functions.getService(service_id).call()
+        
+        # Extract the list of agent IDs from the service data
         agent_ids = service[-1]  # Assuming the last element is the list of agent IDs
-        print(f"Agent IDs: {agent_ids}")
 
+        # Check if 25 is in the list of agent IDs
         if 25 in agent_ids:
-            agent_address = registry_contract.functions.getAgentInstances(service_id).call()
-            service_safe = service[1]
-            print(f"Found agent_address: {agent_address}")
-            print(f"Found service safe: {service_safe}")
-            service_safes.add(service_safe)
-
-    print(f"Total service safes found: {len(service_safes)}")
-    return service_safes
-
-# Fetch service safes for each network
-service_safes_optimism = fetch_service_safes(web3_optimism, service_registry_optimism)
-service_safes_base = fetch_service_safes(web3_base, service_registry_base)
-service_safes_eth = fetch_service_safes(web3_eth, service_registry_eth)
-service_safes_eth = {safe for safe in service_safes_eth if safe.lower() != '0x0000000000000000000000000000000000000000'}
-
-def get_block_range_for_date(chain_id, date_str, api_key, base_url):
-    """Get the block range for a specific date."""
-    target_date = datetime.strptime(date_str, "%Y-%m-%d")
-    start_of_day = datetime.combine(target_date, datetime.min.time())
-    end_of_day = datetime.combine(target_date, datetime.max.time())
-
-    start_timestamp = int(start_of_day.timestamp())
-    end_timestamp = int(end_of_day.timestamp())
-
-    # Get start block
-
-    start_response = requests.get(
-        f"{base_url}?module=block&action=getblocknobytime&timestamp={start_timestamp}&closest=before&apikey={api_key}"
-    )
-    if start_response.status_code == 200:
-        start_data = start_response.json()
-        start_block = start_data.get('result')
-    else:
-        print(f"Error fetching start block for {date_str} on chain {chain_id}")
-        return None, None
-
-    if start_block is None:
-        print(f"No start block found for chain {chain_id} on {date_str}")
-        return None, None
-    print(f"Start block for chain {chain_id} on {date_str}: {start_block}")
-
-    # Get end block
-    time.sleep(1)
-    end_response = requests.get(
-        f"{base_url}?module=block&action=getblocknobytime&timestamp={end_timestamp}&closest=before&apikey={api_key}"
-    )
-    if end_response.status_code == 200:
-        end_data = end_response.json()
-        end_block = end_data.get('result')
-    else:
-        print(f"Error fetching end block for {date_str} on chain {chain_id}")
-        return None, None
-
-    if end_block is None:
-        print(f"No end block found for chain {chain_id} on {date_str}")
-        return None, None
-    print(f"End block for chain {chain_id} on {date_str}: {end_block}")
-
-    return start_block, end_block
-
-def get_transactions(api_keys, wallet_address, chain_name, start_block, end_block):
-    """Retrieve transactions for the given wallet address, chain, and block range using the Etherscan or similar API."""
-    base_url = {
-        'optimism': "https://api-optimistic.etherscan.io/api",
-        'base': "https://api.basescan.org/api",
-        'ethereum': "https://api.etherscan.io/api"
-    }.get(chain_name)
-
-    if not base_url:
-        print(f"Invalid chain name: {chain_name}")
-        return []
-
-    params = {
-        'module': 'account',
-        'action': 'txlist',
-        'address': wallet_address,
-        'startblock': start_block,
-        'endblock': end_block,
-        'sort': 'asc',
-        'apikey': api_keys.get(chain_name)
-    }
-
-    response = requests.get(base_url, params=params)
-    data = response.json()
-
-    time.sleep(1)
+            agent_address = service_registry.functions.getAgentInstances(service_id).call()[1][0]
+            response_transfers = get_transfers("valory", agent_address)
+            transfers = response_transfers.get("transfers", [])
+            if isinstance(transfers, list):
+                aggregated_transactions.extend(transfers)
+        
+            # Track the daily number of agents
+            creation_event = service_registry.events.CreateService.create_filter(
+                from_block=0, argument_filters={'serviceId': service_id, 'configHash': service[2]}
+            ).get_all_entries()
+            
+            if creation_event:
+                block_number = creation_event[0]['blockNumber']
+                block = web3.eth.get_block(block_number)
+                creation_timestamp = datetime.fromtimestamp(block['timestamp'])
+                date_str = creation_timestamp.strftime('%Y-%m-%d')
+                
+                if date_str not in daily_agent_counts:
+                    daily_agent_counts[date_str] = set()
+                if date_str not in daily_agents_with_transactions:
+                    daily_agents_with_transactions[date_str] = set()
+                
+                for agent_id in agent_ids:
+                    service_safe = service[1]
+                    print("agent_address",agent_address,"service_safe",service_safe)
+                    multisig_nonces = activity_checker_contract.functions.getMultisigNonces(service_safe).call()[0]
+                    if multisig_nonces > 0:
+                        daily_agents_with_transactions[date_str].add(agent_id)
+                    daily_agent_counts[date_str].add(agent_id)
     
-    if data['status'] != '1':
-        print(f"Error: {data['message']}")
-        return []
-
-    valid_transactions = [tx for tx in data['result'] if tx['isError'] == '0']
+    # Convert set to count
+    daily_agent_counts = {date: len(agents) for date, agents in daily_agent_counts.items()}
+    daily_agents_with_transactions = {date: len(agents) for date, agents in daily_agents_with_transactions.items()}
     
-    return valid_transactions
-
-def date_range(start_date, end_date):
-    """Generates a range of dates from start_date to end_date inclusive."""
-    start_dt = datetime.strptime(start_date, "%Y-%m-%d")
-    end_dt = datetime.strptime(end_date, "%Y-%m-%d")
-    delta = timedelta(days=1)
-    current_dt = start_dt
-    while current_dt <= end_dt:
-        yield current_dt.strftime("%Y-%m-%d")
-        current_dt += delta
-
-def fetch_transactions():
-    # User inputs
-    api_keys = {
-        'optimism': 'XQ72JA5XZ51QC7TG1W295AAIF4KTV92K1K',
-        'base': '4BFQMVW1QUKEPVDA4VW711CF4462682CY8',
-        'ethereum': '3GRYJGX55W3QWCEKGREF4H53AFHCAIVVR7'
-    }
-
-    base_urls = {
-        10: "https://api-optimistic.etherscan.io/api",
-        8453: "https://api.basescan.org/api",
-        1: "https://api.etherscan.io/api"
-    }
-
-    start_date = '2024-09-19'  # Starting date
-    current_date = datetime.now().strftime('%Y-%m-%d')  # Till present date
+    return aggregated_transactions, daily_agent_counts, daily_agents_with_transactions
+
+# Function to parse the transaction data and prepare it for visualization
+def process_transactions_and_agents(data):
+    transactions, daily_agent_counts, daily_agents_with_transactions = data
+
+    # Convert the data into a pandas DataFrame for easy manipulation
+    rows = []
+    for tx in transactions:
+        # Normalize amounts
+        sending_amount = float(tx["sending"]["amount"]) / (10 ** tx["sending"]["token"]["decimals"])
+        receiving_amount = float(tx["receiving"]["amount"]) / (10 ** tx["receiving"]["token"]["decimals"])
+
+        # Convert timestamps to datetime objects
+        sending_timestamp = datetime.utcfromtimestamp(tx["sending"]["timestamp"])
+        receiving_timestamp = datetime.utcfromtimestamp(tx["receiving"]["timestamp"])
+
+        # Prepare row data
+        rows.append({
+            "transactionId": tx["transactionId"],
+            "from_address": tx["fromAddress"],
+            "to_address": tx["toAddress"],
+            "sending_chain": tx["sending"]["chainId"],
+            "receiving_chain": tx["receiving"]["chainId"],
+            "sending_token_symbol": tx["sending"]["token"]["symbol"],
+            "receiving_token_symbol": tx["receiving"]["token"]["symbol"],
+            "sending_amount": sending_amount,
+            "receiving_amount": receiving_amount,
+            "sending_amount_usd": float(tx["sending"]["amountUSD"]),
+            "receiving_amount_usd": float(tx["receiving"]["amountUSD"]),
+            "sending_gas_used": int(tx["sending"]["gasUsed"]),
+            "receiving_gas_used": int(tx["receiving"]["gasUsed"]),
+            "sending_timestamp": sending_timestamp,
+            "receiving_timestamp": receiving_timestamp,
+            "date": sending_timestamp.date(),  # Group by day
+            "week": sending_timestamp.strftime('%Y-%m-%d')  # Group by week
+        })
+    
+    df_transactions = pd.DataFrame(rows)
+    df_agents = pd.DataFrame(list(daily_agent_counts.items()), columns=['date', 'agent_count'])
+    df_agents_with_transactions = pd.DataFrame(list(daily_agents_with_transactions.items()), columns=['date', 'agent_count_with_transactions'])
 
-    chains = {
-        10: ('optimism', service_safes_optimism),      # Optimism chain ID and service safes
-        8453: ('base', service_safes_base),           # Base chain ID and service safes
-        1: ('ethereum', service_safes_eth)            # Ethereum mainnet chain ID and service safes
+    # Convert the date column to datetime
+    df_agents['date'] = pd.to_datetime(df_agents['date'])
+    df_agents_with_transactions['date'] = pd.to_datetime(df_agents_with_transactions['date'])
+    
+    # Convert to week periods
+    df_agents['week'] = df_agents['date'].dt.to_period('W').apply(lambda r: r.start_time)
+    df_agents_with_transactions['week'] = df_agents_with_transactions['date'].dt.to_period('W').apply(lambda r: r.start_time)
+    
+    # Group by week
+    df_agents_weekly = df_agents[['week', 'agent_count']].groupby('week').sum().reset_index()
+    df_agents_with_transactions_weekly = df_agents_with_transactions[['week', 'agent_count_with_transactions']].groupby('week').sum().reset_index()
+
+    return df_transactions, df_agents_weekly, df_agents_with_transactions_weekly
+
+# Function to create visualizations based on the metrics
+def create_visualizations():
+    transactions_data = fetch_and_aggregate_transactions()
+    df_transactions, df_agents_weekly, df_agents_with_transactions_weekly = process_transactions_and_agents(transactions_data)
+
+    # Map chain IDs to chain names
+    chain_name_map = {
+        10: "Optimism",
+        8453: "Base",
+        1: "Ethereum"
     }
-
-    all_transactions = []  # List to hold all transactions
-
-    for chain_id, (chain_name, service_safes) in chains.items():
-        base_url = base_urls[chain_id]
-        api_key = api_keys[chain_name]
-
-        for safe_address in service_safes:
-            print(f"\nProcessing {chain_name.capitalize()} for safe address {safe_address}...")
-            for single_date in date_range(start_date, current_date):
-                start_block, end_block = get_block_range_for_date(chain_id, single_date, api_key, base_url)
-                if start_block is None or end_block is None:
-                    print(f"Skipping date {single_date} for chain {chain_name} due to missing block data.")
-                    continue
-
-                print(f"Start Block: {start_block}, End Block: {end_block} for date {single_date}")
-
-                transactions = get_transactions(api_keys, safe_address, chain_name, start_block, end_block)
-
-                if transactions:
-                    print(f"Found {len(transactions)} transactions on {single_date} for {chain_name.capitalize()} safe address {safe_address}:")
-                    for tx in transactions:
-                        tx_time = datetime.fromtimestamp(int(tx['timeStamp']))
-                        all_transactions.append({
-                            'chain': chain_name,
-                            'safe_address': safe_address,
-                            'date': single_date,
-                            'transaction_hash': tx['hash'],
-                            'timestamp': tx_time,
-                            'from': tx['from'],
-                            'to': tx['to'],
-                            'value_eth': int(tx['value']) / 1e18  # Convert value to ETH
-                        })
-                else:
-                    print(f"No transactions found for safe address {safe_address} on {single_date} on {chain_name.capitalize()}.")
-
-    # Convert the collected transactions into a DataFrame
-    df_transactions_new = pd.DataFrame(all_transactions)
-    return df_transactions_new
-
-def create_transcation_visualizations():
-    df_transactions_new = fetch_transactions()
-    df_transactions_new.to_csv('daily_transactions_new.csv', index=False)
-
-    df_transactions_new['timestamp'] = pd.to_datetime(df_transactions_new['timestamp'])
-
-    # Group by date and chain, count transactions
-    daily_counts = df_transactions_new.groupby([df_transactions_new['timestamp'].dt.date, 'chain']).size().unstack(fill_value=0)
-
-    # Set up the plot
-    fig_tx_chain = px.bar(
-        daily_counts,
+    df_transactions["sending_chain"] = df_transactions["sending_chain"].map(chain_name_map)
+    df_transactions["receiving_chain"] = df_transactions["receiving_chain"].map(chain_name_map)
+
+    # Ensure that chain IDs are strings for consistent grouping
+    df_transactions["sending_chain"] = df_transactions["sending_chain"].astype(str)
+    df_transactions["receiving_chain"] = df_transactions["receiving_chain"].astype(str)
+    df_transactions['date'] = pd.to_datetime(df_transactions['date'])
+
+    # Identify swap transactions
+    df_transactions["is_swap"] = df_transactions.apply(lambda x: x["sending_token_symbol"] != x["receiving_token_symbol"], axis=1)
+
+    # Total swaps per chain per day
+    swaps_per_chain = df_transactions[df_transactions["is_swap"]].groupby(["date", "sending_chain"]).size().reset_index(name="swap_count")
+    fig_swaps_chain = px.bar(
+        swaps_per_chain,
+        x="date",
+        y="swap_count",
+        color="sending_chain",
+        title="Chain Daily Activity: Swaps",
+        labels={"sending_chain": "Transaction Chain", "swap_count": "Daily Swap Nr"},
+        barmode="stack",
+        color_discrete_map={
+            "Optimism": "blue",
+            "Ethereum": "darkgreen",
+            "Base": "purple"
+        }
+    )
+    fig_swaps_chain.update_layout(
+        xaxis_title=None,
+        yaxis=dict(tickmode='linear', tick0=0, dtick=1),
+        xaxis=dict(
+            tickmode='array',
+            tickvals=swaps_per_chain['date'],
+            ticktext=swaps_per_chain['date'].dt.strftime('%y-%m-%d'),
+            tickangle=0,
+        ),
+        bargap=0.8,
+        height=700,
+    )
+    fig_swaps_chain.update_xaxes(tickformat="%Y-%m-%d")
+
+    # Identify bridge transactions
+    df_transactions["is_bridge"] = df_transactions.apply(lambda x: x["sending_chain"] != x["receiving_chain"], axis=1)
+
+    # Total bridges per chain per day
+    bridges_per_chain = df_transactions[df_transactions["is_bridge"]].groupby(["date", "sending_chain"]).size().reset_index(name="bridge_count")
+    fig_bridges_chain = px.bar(
+        bridges_per_chain,
+        x="date",
+        y="bridge_count",
+        color="sending_chain",
+        title="Chain Daily Activity: Bridges",
+        labels={"sending_chain": "Transaction Chain", "bridge_count": "Daily Bridge Nr"},
         barmode="stack",
-        title="Chain Daily Activity: Transactions"
+        color_discrete_map={
+            "Optimism": "blue",
+            "Ethereum": "darkgreen",
+            "Base": "purple"
+        }
+    )
+    fig_bridges_chain.update_layout(
+        xaxis_title=None,
+        yaxis=dict(tickmode='linear', tick0=0, dtick=1),
+        xaxis=dict(
+            tickmode='array',
+            tickvals=bridges_per_chain['date'],
+            ticktext=bridges_per_chain['date'].dt.strftime('%y-%m-%d'),
+            tickangle=0,
+        ),
+        bargap=0.8,
+        height=700,
     )
-    fig_tx_chain.update_layout(
-        xaxis_title="Date",
-        yaxis_title="Daily Transaction Nr",
-        legend_title="Transaction Chain",
-        xaxis_tickformat="%Y-%m-%d",
+    fig_bridges_chain.update_xaxes(tickformat="%Y-%m-%d")
+
+    # Number of agents per week
+    fig_agents_daily = px.bar(
+        df_agents_weekly,
+        x="week",
+        y="agent_count",
+        title="Number of Agents Registered WoW",
+        labels={"week": "Week", "agent_count": "Number of Agents"},
+        color_discrete_sequence=["purple"]
+    )
+    fig_agents_daily.update_layout(
+        xaxis_title=None,
+        yaxis=dict(tickmode='linear', tick0=0, dtick=1),
+        xaxis=dict(
+            tickmode='array',
+            tickvals=df_agents_weekly['week'],
+            ticktext=df_agents_weekly['week'].dt.strftime('%y-%m-%d'),
+            tickangle=0
+        ),
+        bargap=0.8,
         height=700,
     )
 
-    return fig_tx_chain
+    # Number of agents with transactions per week
+    fig_agents_with_transactions_daily = px.bar(
+        df_agents_with_transactions_weekly,
+        x="week",
+        y="agent_count_with_transactions",
+        title="Agents With at Least 1 Transaction WoW",
+        labels={"week": "Week", "agent_count_with_transactions": "Number of Agents with Transactions"},
+        color_discrete_sequence=["darkgreen"]
+    )
+    fig_agents_with_transactions_daily.update_layout(
+        xaxis_title=None,
+        yaxis=dict(tickmode='linear', tick0=0, dtick=1),
+        xaxis=dict(
+            tickmode='array',
+            tickvals=df_agents_with_transactions_weekly['week'],
+            ticktext=df_agents_with_transactions_weekly['week'].dt.strftime('%y-%m-%d'),
+            tickangle=0
+        ),
+        bargap=0.8,
+        height=700,
+    )
 
+    return fig_swaps_chain, fig_bridges_chain, fig_agents_daily, fig_agents_with_transactions_daily
 
 # Gradio interface
 def dashboard():
@@ -254,12 +311,22 @@ def dashboard():
         gr.Markdown("# Valory Transactions Dashboard")
 
         # Fetch and display visualizations
-        with gr.Tab("Transactions"):
-            fig_tx_chain = create_transcation_visualizations()
-            gr.Plot(fig_tx_chain)
-
-        # Add more tabs as needed...
+        with gr.Tab("Swaps"):
+            fig_swaps_chain, fig_bridges_chain, fig_agents_daily, fig_agents_with_transactions_daily = create_visualizations()
+            gr.Plot(fig_swaps_chain)
+        
+        with gr.Tab("Bridges"):
+            fig_swaps_chain, fig_bridges_chain, fig_agents_daily, fig_agents_with_transactions_daily = create_visualizations()
+            gr.Plot(fig_bridges_chain)
         
+        with gr.Tab("Agents Week Over Week"):
+            fig_swaps_chain, fig_bridges_chain, fig_agents_daily, fig_agents_with_transactions_daily = create_visualizations()
+            gr.Plot(fig_agents_daily)
+        
+        with gr.Tab("Agents with Transactions Week Over Week"):
+            fig_swaps_chain, fig_bridges_chain, fig_agents_daily, fig_agents_with_transactions_daily = create_visualizations()
+            gr.Plot(fig_agents_with_transactions_daily)
+
     return demo
 
 # Launch the dashboard