Update app.py

app.py

@@ -27,9 +27,9 @@ def get_raw_data(station_id):
         'Accept': 'application/json'
     }
     
-    # Calculate correct date range for last 3 days
+    # Calculate date range for last 3 days
     end_time = datetime.utcnow()
-    start_time = end_time - timedelta(hours=72)  # Last 3 days
+    start_time = end_time - timedelta(hours=72)
     
     params = {
         'start': start_time.isoformat() + 'Z',
@@ -56,11 +56,10 @@ def get_raw_data(station_id):
             if len(data['features']) > 0:
                 print("\nFirst observation properties:")
                 print(json.dumps(data['features'][0]['properties'], indent=2))
-                
-                print("\nAll available property keys:")
                 keys = set()
                 for feature in data['features']:
                     keys.update(feature['properties'].keys())
+                print("\nAll available property keys:")
                 print(sorted(list(keys)))
         
         return data
@@ -73,7 +72,7 @@ def get_raw_data(station_id):
 def scrape_snow_depth():
     """
     Uses Selenium with a headless browser to load the weather.gov timeseries page,
-    waits for the table to load, then extracts the snow depth data.
+    waits until an element containing 'Snow Depth' is present, then extracts the table data.
     Returns a DataFrame with columns "timestamp" and "snowDepth".
     """
     url = ("https://www.weather.gov/wrh/timeseries?"
@@ -86,20 +85,20 @@ def scrape_snow_depth():
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--disable-dev-shm-usage")
     
-    # Initialize Chrome using webdriver_manager so that the correct driver is installed automatically
+    # Initialize Chrome using webdriver_manager
     driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
     
     driver.get(url)
     
     try:
-        # Wait (up to 15 seconds) for at least one <table> element to appear in the DOM
-        WebDriverWait(driver, 15).until(
-            EC.presence_of_element_located((By.TAG_NAME, "table"))
+        # Wait up to 30 seconds for any element containing the text "Snow Depth" to appear
+        WebDriverWait(driver, 30).until(
+            EC.presence_of_element_located((By.XPATH, "//*[contains(text(), 'Snow Depth')]"))
         )
     except Exception as e:
-        print("Timeout waiting for table to load:", e)
+        print("Timeout waiting for 'Snow Depth' element to appear:", e)
     
-    # Give extra time for any dynamic data to load
+    # Allow extra time for dynamic content to load
     time.sleep(5)
     
     page_source = driver.page_source
@@ -107,26 +106,29 @@ def scrape_snow_depth():
     
     soup = BeautifulSoup(page_source, 'html.parser')
     
-    # Look through all tables for one with headers that include both "time" and "snow"
+    # Look through all tables for one that contains "Snow Depth" in its text
     tables = soup.find_all("table")
     target_table = None
     for table in tables:
-        header_row = table.find("tr")
-        if not header_row:
-            continue
-        headers = [th.get_text(strip=True) for th in header_row.find_all("th")]
-        print("Found table headers:", headers)
-        if any("time" in h.lower() for h in headers) and any("snow" in h.lower() for h in headers):
+        table_text = table.get_text()
+        print("Found table text snippet:", table_text[:100])
+        if "Snow Depth" in table_text:
             target_table = table
             break
     
     if target_table is None:
-        print("No table with required headers found.")
+        print("No table with 'Snow Depth' found in the page.")
         return pd.DataFrame()
     
-    # Determine column indices using case-insensitive matching
+    # Look for header cells in the table
     header_row = target_table.find("tr")
+    if not header_row:
+        print("No header row found in the table.")
+        return pd.DataFrame()
     headers = [th.get_text(strip=True) for th in header_row.find_all("th")]
+    print("Table headers found:", headers)
+    
+    # Identify column indices (using case-insensitive match)
     time_index = None
     snow_index = None
     for i, header in enumerate(headers):
@@ -135,10 +137,10 @@ def scrape_snow_depth():
         if "snow" in header.lower():
             snow_index = i
     if time_index is None or snow_index is None:
-        print("Required columns not found in the table headers.")
+        print("Required columns ('Time' and 'Snow Depth') not found in the table headers.")
         return pd.DataFrame()
     
-    # Extract data rows (skipping the header row)
+    # Extract rows (skip header)
     data = []
     rows = target_table.find_all("tr")[1:]
     for row in rows:
@@ -150,18 +152,19 @@ def scrape_snow_depth():
         data.append((time_text, snow_text))
     
     df = pd.DataFrame(data, columns=["Time", "Snow Depth"])
-    # Convert "Time" to datetime objects
+    # Convert the "Time" column to datetime
     df["Time"] = pd.to_datetime(df["Time"], errors="coerce")
-    # Convert "Snow Depth" to numeric values (in inches)
+    # Convert "Snow Depth" to numeric (in inches)
     df["Snow Depth"] = pd.to_numeric(df["Snow Depth"], errors="coerce")
     print("Scraped snow depth data:")
     print(df.head())
-    # Rename columns to match our API data
+    
+    # Rename columns to match API data
     return df.rename(columns={"Time": "timestamp", "Snow Depth": "snowDepth"})
 
 def parse_raw_data(data):
     """
-    Parse the raw JSON data from the API into a DataFrame.
+    Parse the raw JSON API data into a DataFrame.
     """
     if not data or 'features' not in data:
         return None
@@ -169,8 +172,7 @@ def parse_raw_data(data):
     records = []
     for feature in data['features']:
         props = feature['properties']
-        
-        # Extract any properties with "snow" in their key (if present)
+        # Extract any snow-related fields if present
         snow_fields = {k: v for k, v in props.items() if 'snow' in k.lower()}
         if snow_fields:
             print("\nFound snow-related fields:")
@@ -183,8 +185,7 @@ def parse_raw_data(data):
             'wind_speed': props.get('windSpeed', {}).get('value'),
             'wind_direction': props.get('windDirection', {}).get('value')
         }
-        
-        # Add any snow-related fields to the record
+        # Add any snow fields
         for k, v in snow_fields.items():
             if isinstance(v, dict) and 'value' in v:
                 record[k] = v['value']
@@ -192,9 +193,8 @@ def parse_raw_data(data):
                 record[k] = v
         
         records.append(record)
-
-    df = pd.DataFrame(records)
     
+    df = pd.DataFrame(records)
     print("\nDataFrame columns from API:")
     print(df.columns.tolist())
     print("\nSample of raw API data:")
@@ -228,7 +228,7 @@ def create_wind_rose(ax, data, title):
     Create a wind rose subplot.
     """
     if data.empty or data['wind_direction'].isna().all() or data['wind_speed'].isna().all():
-        ax.text(0.5, 0.5, 'No wind data available', 
+        ax.text(0.5, 0.5, 'No wind data available',
                 horizontalalignment='center',
                 verticalalignment='center',
                 transform=ax.transAxes)
@@ -236,44 +236,35 @@ def create_wind_rose(ax, data, title):
         return
 
     plot_data = data.copy()
-    
     direction_bins = np.arange(0, 361, 45)
     directions = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']
-    
     mask = plot_data['wind_direction'].notna() & plot_data['wind_speed'].notna()
     plot_data = plot_data[mask]
-    
     if plot_data.empty:
-        ax.text(0.5, 0.5, 'No valid wind data', 
+        ax.text(0.5, 0.5, 'No valid wind data',
                 horizontalalignment='center',
                 verticalalignment='center',
                 transform=ax.transAxes)
         ax.set_title(title)
         return
-    
-    plot_data.loc[:, 'direction_bin'] = pd.cut(plot_data['wind_direction'], 
-                                               bins=direction_bins, 
+    plot_data.loc[:, 'direction_bin'] = pd.cut(plot_data['wind_direction'],
+                                               bins=direction_bins,
                                                labels=directions,
                                                include_lowest=True)
-    
     wind_stats = plot_data.groupby('direction_bin', observed=True)['wind_speed'].mean()
-    
     all_directions = pd.Series(0.0, index=directions)
     wind_stats = wind_stats.combine_first(all_directions)
-    
     angles = np.linspace(0, 2*np.pi, len(directions), endpoint=False)
     values = [wind_stats[d] for d in directions]
-    
     if any(v > 0 for v in values):
         ax.bar(angles, values, width=0.5, alpha=0.6)
         ax.set_xticks(angles)
         ax.set_xticklabels(directions)
     else:
-        ax.text(0.5, 0.5, 'No significant wind', 
+        ax.text(0.5, 0.5, 'No significant wind',
                 horizontalalignment='center',
                 verticalalignment='center',
                 transform=ax.transAxes)
-    
     ax.set_title(title)
 
 def create_visualizations(df):
@@ -302,12 +293,11 @@ def create_visualizations(df):
     ax2.set_xlabel('')
     ax2.grid(True)
     
-    # Plot snow depth if available
     if 'snowDepth' in df.columns and not df['snowDepth'].isna().all():
         ax3.plot(df['timestamp'], df['snowDepth'], linewidth=2)
-        ax3.set_ylim(0, 80)  # Fixed y-axis limit to 80 inches
+        ax3.set_ylim(0, 80)
     else:
-        ax3.text(0.5, 0.5, 'No snow depth data available', 
+        ax3.text(0.5, 0.5, 'No snow depth data available',
                  horizontalalignment='center',
                  verticalalignment='center',
                  transform=ax3.transAxes)
@@ -316,7 +306,6 @@ def create_visualizations(df):
     ax3.set_xlabel('')
     ax3.grid(True)
     
-    # Format x-axis labels
     for ax in [ax1, ax2, ax3]:
         ax.tick_params(axis='x', rotation=45)
         ax.xaxis.set_major_formatter(plt.matplotlib.dates.DateFormatter('%Y-%m-%d %H:%M'))
@@ -329,7 +318,6 @@ def create_visualizations(df):
             create_wind_rose(ax, day_data, pd.to_datetime(date).strftime('%Y-%m-%d'))
     
     plt.tight_layout()
-    
     return fig
 
 def get_weather_data(station_id, hours):
@@ -338,34 +326,29 @@ def get_weather_data(station_id, hours):
     Combines API data and scraped snow depth data.
     """
     try:
-        # Get raw data from the API
         raw_data = get_raw_data(station_id)
         if raw_data is None:
             return None, "Failed to fetch data from API"
-
-        # Parse raw API data
+        
         df = parse_raw_data(raw_data)
         if df is None:
             return None, "Failed to parse API data"
-
-        # Process the API data
+        
         df = process_weather_data(df)
         if df is None:
             return None, "Failed to process API data"
         
-        # Scrape snow depth data using Selenium and merge with API data
+        # Attempt to scrape snow depth data using Selenium
         snow_df = scrape_snow_depth()
         if not snow_df.empty:
             df = df.sort_values('timestamp')
             snow_df = snow_df.sort_values('timestamp')
-            # Merge using nearest timestamp within a 30-minute tolerance
             df = pd.merge_asof(df, snow_df, on='timestamp', tolerance=pd.Timedelta('30min'), direction='nearest')
         
         print("\nProcessed combined data sample:")
         print(df.head())
         
         return df, None
-
     except Exception as e:
         return None, f"Error: {str(e)}"
 
@@ -374,30 +357,23 @@ def fetch_and_display(station_id, hours):
     Fetch data and create visualization.
     """
     df, error = get_weather_data(station_id, hours)
-    
     if error:
         return None, error
-    
     if df is not None and not df.empty:
         fig = create_visualizations(df)
         return fig, "Data fetched successfully!"
-    
     return None, "No data available for the specified parameters."
 
 # Create Gradio interface
 with gr.Blocks() as demo:
     gr.Markdown("# Weather Data Viewer")
     gr.Markdown("Displays temperature, wind speed, and snow depth from NWS stations.")
-    
     with gr.Row():
         station_id = gr.Textbox(label="Station ID", value="YCTIM")
         hours = gr.Slider(minimum=24, maximum=168, value=72, label="Hours of Data", step=24)
-    
     fetch_btn = gr.Button("Fetch Data")
-    
     plot_output = gr.Plot()
     message = gr.Textbox(label="Status")
-    
     fetch_btn.click(
         fn=fetch_and_display,
         inputs=[station_id, hours],