Update app.py

app.py

@@ -85,7 +85,7 @@ def parse_dms_coordinates(text):
 def fetch_firms_data():
     """
     Fetch NASA FIRMS VIIRS active fire data for the last 24 hours
-    Filters for USA only and returns relevant fire hotspot data
+    Filters for USA only and returns relevant fire hotspot data with cleaned numeric fields
     """
     firms_url = "https://firms.modaps.eosdis.nasa.gov/data/active_fire/viirs/csv/J1_VIIRS_C2_Global_24h.csv"
     
@@ -118,12 +118,36 @@ def fetch_firms_data():
         
         print(f"Filtered to {len(usa_firms)} USA fire hotspots")
         
-        # Add datetime column for easier processing
-        usa_firms['datetime'] = pd.to_datetime(usa_firms['acq_date'] + ' ' + usa_firms['acq_time'].astype(str).str.zfill(4), 
-                                             format='%Y-%m-%d %H%M')
+        # Clean numeric columns to handle "nominal" values
+        if 'frp' in usa_firms.columns:
+            # Clean FRP column
+            usa_firms['frp'] = usa_firms['frp'].astype(str).str.replace('nominal', '', regex=False)
+            usa_firms['frp'] = usa_firms['frp'].str.replace(r'[^\d\.]', '', regex=True)  # Keep only digits and decimals
+            usa_firms['frp'] = usa_firms['frp'].replace('', '0')  # Replace empty strings with 0
+            usa_firms['frp'] = pd.to_numeric(usa_firms['frp'], errors='coerce').fillna(0)
+            print(f"Cleaned FRP column, mean FRP: {usa_firms['frp'].mean():.2f}")
+        
+        if 'confidence' in usa_firms.columns:
+            # Clean confidence column
+            usa_firms['confidence'] = usa_firms['confidence'].astype(str).str.replace('nominal', '', regex=False)
+            usa_firms['confidence'] = usa_firms['confidence'].str.replace(r'[^\d\.]', '', regex=True)
+            usa_firms['confidence'] = usa_firms['confidence'].replace('', '50')  # Default confidence
+            usa_firms['confidence'] = pd.to_numeric(usa_firms['confidence'], errors='coerce').fillna(50)
+            print(f"Cleaned confidence column, mean confidence: {usa_firms['confidence'].mean():.2f}")
         
-        # Sort by acquisition time (most recent first)
-        usa_firms = usa_firms.sort_values('datetime', ascending=False)
+        # Add datetime column for easier processing
+        if 'acq_date' in usa_firms.columns and 'acq_time' in usa_firms.columns:
+            try:
+                usa_firms['datetime'] = pd.to_datetime(
+                    usa_firms['acq_date'] + ' ' + usa_firms['acq_time'].astype(str).str.zfill(4), 
+                    format='%Y-%m-%d %H%M',
+                    errors='coerce'
+                )
+                # Sort by acquisition time (most recent first)
+                usa_firms = usa_firms.sort_values('datetime', ascending=False)
+                print(f"Added datetime column, latest detection: {usa_firms['datetime'].max()}")
+            except Exception as e:
+                print(f"Warning: Could not create datetime column: {e}")
         
         return usa_firms
         
@@ -135,7 +159,7 @@ def fetch_firms_data():
 def match_firms_to_inciweb(inciweb_df, firms_df, max_distance_km=50):
     """
     Match FIRMS hotspots to InciWeb incidents based on geographic proximity
-    Enhanced with better error handling
+    Enhanced with better error handling and data cleaning
     """
     if firms_df.empty or inciweb_df.empty:
         print("Warning: Empty dataframes passed to matching function")
@@ -163,31 +187,46 @@ def match_firms_to_inciweb(inciweb_df, firms_df, max_distance_km=50):
         
         for idx, incident in incidents_with_coords.iterrows():
             try:
-                incident_coords = (incident['latitude'], incident['longitude'])
+                incident_coords = (float(incident['latitude']), float(incident['longitude']))
                 
                 # Find FIRMS hotspots within the specified distance
-                hotspot_distances = []
                 matched_hotspots = []
                 
                 for _, hotspot in firms_df.iterrows():
                     try:
-                        hotspot_coords = (hotspot['latitude'], hotspot['longitude'])
+                        hotspot_lat = float(hotspot['latitude'])
+                        hotspot_lon = float(hotspot['longitude'])
+                        hotspot_coords = (hotspot_lat, hotspot_lon)
+                        
                         distance = geodesic(incident_coords, hotspot_coords).kilometers
                         
                         if distance <= max_distance_km:
-                            hotspot_distances.append(distance)
-                            matched_hotspots.append(hotspot)
-                    except Exception as e:
-                        continue  # Skip invalid coordinates
+                            # Create a clean hotspot record with safe conversions
+                            clean_hotspot = {
+                                'latitude': hotspot_lat,
+                                'longitude': hotspot_lon,
+                                'frp': float(hotspot.get('frp', 0)) if pd.notna(hotspot.get('frp')) else 0.0,
+                                'confidence': float(hotspot.get('confidence', 50)) if pd.notna(hotspot.get('confidence')) else 50.0,
+                                'datetime': hotspot.get('datetime', None),
+                                'distance': distance
+                            }
+                            matched_hotspots.append(clean_hotspot)
+                            
+                    except (ValueError, TypeError, KeyError) as e:
+                        # Skip invalid hotspot data
+                        continue
                 
                 if matched_hotspots:
-                    matched_df = pd.DataFrame(matched_hotspots)
-                    
                     # Calculate aggregated metrics safely
                     num_hotspots = len(matched_hotspots)
-                    total_frp = float(matched_df['frp'].sum()) if 'frp' in matched_df.columns else 0.0
-                    avg_confidence = float(matched_df['confidence'].mean()) if 'confidence' in matched_df.columns else 0.0
-                    latest_hotspot = matched_df['datetime'].max() if 'datetime' in matched_df.columns else None
+                    total_frp = sum(hs['frp'] for hs in matched_hotspots)
+                    avg_confidence = sum(hs['confidence'] for hs in matched_hotspots) / num_hotspots if num_hotspots > 0 else 0.0
+                    
+                    # Get latest hotspot time
+                    latest_hotspot = None
+                    hotspot_times = [hs['datetime'] for hs in matched_hotspots if hs['datetime'] is not None]
+                    if hotspot_times:
+                        latest_hotspot = max(hotspot_times)
                     
                     # Determine activity level based on hotspot count and FRP
                     if num_hotspots >= 20 and total_frp >= 100:
@@ -209,10 +248,10 @@ def match_firms_to_inciweb(inciweb_df, firms_df, max_distance_km=50):
                     inciweb_df.at[idx, 'is_active'] = True
                     inciweb_df.at[idx, 'activity_level'] = activity_level
                     
-                    # Store hotspot coordinates for visualization (simplified)
-                    hotspot_coords = [(float(hs['latitude']), float(hs['longitude']), float(hs.get('frp', 1))) 
-                                    for hs in matched_hotspots[:10]]  # Limit to 10 for performance
-                    inciweb_df.at[idx, 'hotspot_coords'] = str(hotspot_coords)  # Store as string for safety
+                    # Store simplified hotspot coordinates for visualization
+                    hotspot_coords_str = str([(hs['latitude'], hs['longitude'], hs['frp']) 
+                                            for hs in matched_hotspots[:10]])  # Limit to 10 for performance
+                    inciweb_df.at[idx, 'hotspot_coords'] = hotspot_coords_str
                     
                     print(f"  {incident['name']}: {num_hotspots} hotspots, {total_frp:.1f} FRP, {activity_level} activity")
                     
@@ -220,7 +259,7 @@ def match_firms_to_inciweb(inciweb_df, firms_df, max_distance_km=50):
                 print(f"  Error processing incident {incident.get('name', 'Unknown')}: {e}")
                 continue
         
-        # Mark incidents without recent hotspots as potentially inactive
+        # Calculate final statistics
         active_count = (inciweb_df['is_active'] == True).sum()
         total_with_coords = len(incidents_with_coords)
         
@@ -230,14 +269,20 @@ def match_firms_to_inciweb(inciweb_df, firms_df, max_distance_km=50):
         
     except Exception as e:
         print(f"Error in match_firms_to_inciweb: {e}")
-        # Return original dataframe with safety columns
-        inciweb_df['firms_hotspots'] = 0
-        inciweb_df['total_frp'] = 0.0
-        inciweb_df['avg_confidence'] = 0.0
-        inciweb_df['latest_hotspot'] = None
-        inciweb_df['is_active'] = False
-        inciweb_df['hotspot_coords'] = None
-        inciweb_df['activity_level'] = 'Unknown'
+        # Return original dataframe with safety columns if matching completely fails
+        inciweb_df = inciweb_df.copy()
+        for col in ['firms_hotspots', 'total_frp', 'avg_confidence', 'latest_hotspot', 'is_active', 'hotspot_coords', 'activity_level']:
+            if col not in inciweb_df.columns:
+                if col in ['firms_hotspots']:
+                    inciweb_df[col] = 0
+                elif col in ['total_frp', 'avg_confidence']:
+                    inciweb_df[col] = 0.0
+                elif col in ['is_active']:
+                    inciweb_df[col] = False
+                elif col in ['activity_level']:
+                    inciweb_df[col] = 'Unknown'
+                else:
+                    inciweb_df[col] = None
         return inciweb_df
 
 # Function to scrape InciWeb data from the accessible view page