Added data loading and reloading utility

* Add basic run instruction for api

* Add reload scheduler

* remove comment

* Disable test mode by default and add explicit host

---------

Co-authored-by: schampoux <[email protected]>

README.md

@@ -11,3 +11,6 @@ license: apache-2.0
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+
+To run the API `python api.py`

api.py

@@ -1,46 +1,52 @@
-import utils
-import time
+
+import atexit
 import datetime
 
-import pandas as pd
-import bittensor as bt
-from typing import Dict, List, Any, Optional, Tuple
 from flask import Flask, request, jsonify
+from apscheduler.schedulers.background import BackgroundScheduler
 
+import utils
 
 app = Flask(__name__)
 
 # Global variables (saves time on loading data)
+state_vars = None
+reload_timestamp = datetime.datetime.now().strftime('%D %T')
+
+
+def load_data(test=False):
+    """
+    Reload the state variables
+    """
+    global state_vars, reload_timestamp
+    if test:
+        state_vars = utils.test_load_state_vars()    
+    else:  
+        state_vars = utils.load_state_vars()
+        
+    reload_timestamp = datetime.datetime.now().strftime('%D %T')
+
+    print(f'Reloaded data at {reload_timestamp}')
+
+
+def start_scheduler():
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(func=load_data, trigger="interval", seconds=60*30)
+    scheduler.start()
+
+    # Shut down the scheduler when exiting the app
+    atexit.register(lambda: scheduler.shutdown())
 
-state_vars = utils.test_load_state_vars()
-metagraph = state_vars["metagraph"]
-model_data = state_vars["model_data"]
-vali_runs = state_vars["vali_runs"]
-scores = state_vars["scores"]
-validator_df = state_vars["validator_df"]
-benchmarks = state_vars.get("benchmarks", None)
-benchmark_timestamp = state_vars.get("benchmark_timestamp", None)
 
 @app.route('/', methods=['GET'])
 def home():
     return "Welcome to the Bittensor Pretraining Leaderboard API!"
 
-@app.route('/reload', methods=['GET'])
-def reload():
-    """
-    Reload the state variables
-    """
-    global metagraph, model_data, vali_runs, scores, validator_df, benchmarks, benchmark_timestamp
-    state_vars = utils.load_state_vars()
-    metagraph = state_vars["metagraph"]
-    model_data = state_vars["model_data"]
-    vali_runs = state_vars["vali_runs"]
-    scores = state_vars["scores"]
-    validator_df = state_vars["validator_df"]
-    benchmarks = state_vars.get("benchmarks", None)
-    benchmark_timestamp = state_vars.get("benchmark_timestamp", None)
 
-    return jsonify({"message": "State variables reloaded"})
+@app.route('/updated', methods=['GET'])
+def updated():
+    return reload_timestamp
+
 
 @app.route('/benchmark', methods=['GET'])
 def benchmark():
@@ -52,6 +58,9 @@ def benchmark():
     - benchmark_timestamp: String
     """
 
+    benchmarks = state_vars.get("benchmarks", None)
+    benchmark_timestamp = state_vars.get("benchmark_timestamp", None)
+
     return jsonify(
         {
             "benchmarks": benchmarks.to_dict(orient='records'),
@@ -66,6 +75,9 @@ def metagraph():
     Returns:
     - metagraph_data: List of dicts (from pandas DataFrame)
     """
+    
+    metagraph = state_vars["metagraph"]
+     
     return jsonify(
         utils.make_metagraph_dataframe(metagraph).to_dict(orient='records')
     )
@@ -77,6 +89,9 @@ def leaderboard():
     Returns:
     - leaderboard_data: List of dicts (from pandas DataFrame)
     """
+
+    model_data = state_vars["model_data"]
+    scores = state_vars["scores"]
     show_stale = request.args.get('show_stale')
     return jsonify(
         utils.leaderboard_data(model_data, scores, show_stale=show_stale)
@@ -90,6 +105,8 @@ def loss():
     Returns:
     - losses_over_time: List of dicts (from pandas DataFrame)
     """
+    vali_runs = state_vars["vali_runs"]
+
     return jsonify(
         utils.get_losses_over_time(vali_runs).to_dict(orient='records')
         )
@@ -102,11 +119,17 @@ def validator():
     Returns:
     - validator_data: List of dicts (from pandas DataFrame)
     """
+    model_data = state_vars["model_data"]
+    validator_df = state_vars["validator_df"]       
+
     return jsonify(
         utils.make_validator_dataframe(validator_df, model_data).to_dict(orient='records')
         )
 
 
 if __name__ == '__main__':
+    
+    load_data()
+    start_scheduler()
 
-    app.run(port=5000, debug=True)
+    app.run(host='0.0.0.0', port=5000, debug=True)

utils.py

@@ -367,7 +367,6 @@ def load_state_vars() -> dict[Any]:
             for uid, stats in validator_df.items():
                 weight_keys.update(stats[-1].keys())
 
-            # TODO: re-enable benchmarks
             # Enable benchmark if the flag is set
             if BENCHMARK_FLAG:
                 benchmarks, benchmark_timestamp = get_benchmarks()