import pandas as pd import numpy as np from scipy.ndimage import gaussian_filter1d from sklearn.preprocessing import MinMaxScaler import torch import torch.nn as nn from torch.utils.data import Dataset, DataLoader import random import matplotlib.pyplot as plt from torch.utils.data import DataLoader, TensorDataset from pathlib import Path import matplotlib.dates as mdates # --- Utility Functions --- def set_seed(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # --- Data Loading and Initial Processing (from original) --- def get_data_building_weather_weekly(): # path = "C:\\Software\\Probabilistic_Forecasting\\Data\\ashrae-energy-prediction" # df_train = pd.read_csv(path + "\\train.csv") # df_weather = pd.read_csv(path + "\\weather_train.csv") # df_meta = pd.read_csv(path + "\\building_metadata.csv") df_train = pd.read_csv("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/ashrae-energy-prediction/train.csv") df_weather = pd.read_csv("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/ashrae-energy-prediction/weather_train.csv") df_meta = pd.read_csv("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/ashrae-energy-prediction/building_metadata.csv") df = df_train.merge(df_meta, on='building_id').merge(df_weather, on=['site_id', 'timestamp']) df['timestamp'] = pd.to_datetime(df['timestamp']) # Filter for a specific building, meter, and a reduced date range for faster processing if needed df = df[(df['building_id'] == 2) & (df['meter'] == 0)] df = df[(df['timestamp'] >= '2016-01-04') & (df['timestamp'] < '2017-01-04')] # Ensure enough data for ~50 weeks df['Date'] = df['timestamp'].dt.date df['day_of_week'] = df['timestamp'].dt.dayofweek # Monday=0, Sunday=6 def get_season(month): return {12: 0, 1: 0, 2: 0, 3: 1, 4: 1, 5: 1, 6: 2, 7: 2, 8: 2, 9: 3, 10: 3, 11: 3}[month] # Ensure 'meter_reading' and 'air_temperature' are present and numeric df['meter_reading'] = pd.to_numeric(df['meter_reading'], errors='coerce').fillna(0) df['air_temperature'] = pd.to_numeric(df['air_temperature'], errors='coerce').fillna(method='ffill').fillna( method='bfill').fillna(15) measurement_columns = ['meter_reading', 'air_temperature', 'Date', 'timestamp'] # Ensure columns exist, add placeholders if not for col in measurement_columns: if col not in df.columns and col not in ['Date']: # Date is derived df[col] = 0 if col != 'timestamp' else pd.NaT grouped = df.groupby('Date')[measurement_columns + ['day_of_week']] array_3d, labels_3d, seasons_3d = [], [], [] dates = sorted(grouped.groups.keys()) if not dates: print("Warning: No data after filtering in get_data_building_weather_weekly.") # Return empty arrays with expected dimensions to avoid downstream errors immediately return np.array([]), np.array([]), np.array([]), np.array([]), np.array([]) for date_val in dates: group_df = grouped.get_group(date_val) if group_df.empty or len(group_df) != 24: # Assuming hourly data, fill if not # Create a full day template full_day_timestamps = pd.to_datetime([f"{date_val} {h:02d}:00:00" for h in range(24)]) template_df = pd.DataFrame({'timestamp': full_day_timestamps}) group_df = pd.merge(template_df, group_df, on='timestamp', how='left') group_df['Date'] = group_df['timestamp'].dt.date group_df['day_of_week'] = group_df['timestamp'].dt.dayofweek for col in ['meter_reading', 'air_temperature']: group_df[col] = group_df[col].interpolate(method='linear').fillna(method='ffill').fillna(method='bfill') group_df = group_df.fillna({'meter_reading': 0, 'air_temperature': 15}) # final fallback arr = group_df[measurement_columns].values label = 0 if group_df['day_of_week'].iloc[0] < 5 else 1 # Weekday/Weekend season = get_season(group_df['timestamp'].iloc[0].month) array_3d.append(arr) labels_3d.append(np.full(len(arr), label)) seasons_3d.append(np.full(len(arr), season)) n_full_weeks = len(array_3d) // 7 if n_full_weeks == 0: print("Warning: Not enough daily data to form even one full week.") return np.array([]), np.array([]), np.array([]), np.array([]), np.array([]) energy, temp, times, workday, season_feat = [], [], [], [], [] for w in range(n_full_weeks): wk = slice(w * 7, (w + 1) * 7) week_data = array_3d[wk] week_labels = labels_3d[wk] week_seasons = seasons_3d[wk] e = np.concatenate([np.asarray(d[:, 0], dtype=float) for d in week_data]) t = np.concatenate([np.asarray(d[:, 1], dtype=float) for d in week_data]) ts = np.concatenate([np.asarray(d[:, 3]) for d in week_data]) # timestamp objects wl = np.concatenate([np.asarray(lbl, dtype=int) for lbl in week_labels]) sl = np.concatenate([np.asarray(seas, dtype=int) for seas in week_seasons]) if e.shape[0] != 168: # Skip incomplete weeks silently or handle # print(f"Skipping week {w} due to incomplete data: {e.shape[0]} points") continue e = gaussian_filter1d(e, sigma=1) t = gaussian_filter1d(t, sigma=1) energy.append(e) temp.append(t) times.append(ts) workday.append(wl) season_feat.append(sl) return np.array(times, dtype=object), np.array(energy), np.array(temp), np.array(workday), np.array(season_feat) def gaussian_nll_loss(mu, logvar, target): # mu, logvar, target → same shape [B, L+1, output_len, output_dim] nll = 0.5 * (logvar + np.log(2 * np.pi) + ((target - mu) ** 2) / logvar.exp()) return nll.mean() # average over all elements def kl_loss(mu_z, logvar_z): return -0.5 * torch.mean(1 + logvar_z - mu_z.pow(2) - logvar_z.exp()) def process_seq2seq_data( feature_dict, *, train_ratio = 0.7, norm_features = ('load', 'temp'), output_len = 24, # how many steps each decoder step predicts encoder_len_weeks = 1, decoder_len_weeks = 1, num_in_week = 168, # ← NEW: default parameter device = None): # ---------------------------------------------------------- # 1. flatten, scale, keep 1‑D per feature # ---------------------------------------------------------- processed, scalers = {}, {} for k, arr in feature_dict.items(): if arr.size == 0: raise ValueError(f"feature '{k}' is empty.") vec = arr.astype(float).flatten() # weeks → long vector if k in norm_features: sc = MinMaxScaler() processed[k] = sc.fit_transform(vec.reshape(-1, 1)).flatten() scalers[k] = sc else: processed[k] = vec scalers[k] = None n_weeks = feature_dict['load'].shape[0] need_weeks = encoder_len_weeks + decoder_len_weeks if n_weeks < need_weeks: raise ValueError(f"Need ≥{need_weeks} consecutive weeks, found {n_weeks}.") enc_seq_len = encoder_len_weeks * num_in_week dec_seq_len = decoder_len_weeks * num_in_week L = dec_seq_len - output_len if L <= 0: raise ValueError("`output_len` must be smaller than decoder sequence length.") # ---------------------------------------------------------- # 2. build samples (stride = 1 week) # ---------------------------------------------------------- X_enc_l, X_enc_t, X_enc_w, X_enc_s = [], [], [], [] X_dec_in_l, X_dec_in_t, X_dec_in_w, X_dec_in_s = [], [], [], [] Y_dec_target_l = [] last_start = n_weeks - need_weeks # inclusive for w in range(last_start + 1): enc_start = w * num_in_week enc_end = (w + encoder_len_weeks) * num_in_week dec_start = enc_end dec_end = dec_start + dec_seq_len # exclusive # -- encoder -- X_enc_l.append(processed['load' ][enc_start:enc_end]) X_enc_t.append(processed['temp' ][enc_start:enc_end]) X_enc_w.append(processed['workday'][enc_start:enc_end]) X_enc_s.append(processed['season' ][enc_start:enc_end]) # -- decoder input (teacher forcing) -- X_dec_in_l.append(processed['load' ][dec_start : dec_start + L]) X_dec_in_t.append(processed['temp' ][dec_start : dec_start + L]) X_dec_in_w.append(processed['workday'][dec_start : dec_start + L]) X_dec_in_s.append(processed['season' ][dec_start : dec_start + L]) # -- decoder targets (sliding output_len window) -- load_dec_full = processed['load'][dec_start: dec_end] targets = np.stack([ load_dec_full[i: i + output_len] for i in range(L+1)], axis=0) Y_dec_target_l.append(targets) # ---------------------------------------------------------- # 3. pack → tensors # ---------------------------------------------------------- to_tensor = lambda lst: torch.tensor(lst, dtype=torch.float32).unsqueeze(-1).to(device) data_tensors = { 'X_enc_l' : to_tensor(X_enc_l), # [B, enc_seq_len, 1] 'X_enc_t' : to_tensor(X_enc_t), 'X_enc_w' : to_tensor(X_enc_w), 'X_enc_s' : to_tensor(X_enc_s), 'X_dec_in_l' : to_tensor(X_dec_in_l), # [B, L, 1] 'X_dec_in_t' : to_tensor(X_dec_in_t), 'X_dec_in_w' : to_tensor(X_dec_in_w), 'X_dec_in_s' : to_tensor(X_dec_in_s), 'Y_dec_target_l': torch.tensor( Y_dec_target_l, dtype=torch.float32).unsqueeze(-1).to(device) # [B, L, output_len, 1] } # quick check for k, v in data_tensors.items(): print(f"{k:15s} {tuple(v.shape)}") # ---------------------------------------------------------- # 4. train / test split # ---------------------------------------------------------- B = data_tensors['X_enc_l'].shape[0] split = int(train_ratio * B) train_dict = {k: v[:split] for k, v in data_tensors.items()} test_dict = {k: v[split:] for k, v in data_tensors.items()} return train_dict, test_dict, scalers def visualise_one_sample(data_dict, sample_idx=0): """Draw a single figure with three subplots: 1) encoder load, 2) decoder load, 3) heat‑map of Y_dec_target_l.""" enc = data_dict['X_enc_t'][sample_idx].cpu().numpy().squeeze(-1) dec = data_dict['X_dec_in_t'][sample_idx].cpu().numpy().squeeze(-1) tgt = data_dict['Y_dec_target_l'][sample_idx].cpu().numpy().squeeze(-1) # [L, output_len] fig, axes = plt.subplots(3, 1, figsize=(14, 10), constrained_layout=True) axes[0].plot(enc) axes[0].set_title("Encoder input") axes[0].set_xlabel("Time step"); axes[0].set_ylabel("scaled") axes[1].plot(dec) axes[1].set_title("Decoder input") axes[1].set_xlabel("Time step") axes[2].plot(tgt[0]) axes[2].plot(tgt[1]) axes[2].plot(tgt[2]) axes[2].set_title("Decoder target") axes[2].set_xlabel("Time step") plt.show() def make_loader(data_dict, batch_size, shuffle=True): """ Returns: batch = (enc_l, enc_t, enc_w, enc_s, dec_l, dec_t, dec_w, dec_s, tgt) Shapes: enc_* : [B, enc_seq, 1] dec_* : [B, L, 1] tgt : [B, L+1, output_len, 1] """ tensors = ( data_dict['X_enc_l'], data_dict['X_enc_t'], data_dict['X_enc_w'], data_dict['X_enc_s'], data_dict['X_dec_in_l'], data_dict['X_dec_in_t'], data_dict['X_dec_in_w'], data_dict['X_dec_in_s'], data_dict['Y_dec_target_l'] ) ds = TensorDataset(*tensors) return DataLoader(ds, batch_size=batch_size, shuffle=shuffle) def reconstruct_sequence(pred_seq): """ Averages overlapping predictions from [L+1, output_len] into [L+output_len] Args: pred_seq: [L+1, output_len] – single sample prediction Returns: avg_pred: [L+output_len] – averaged sequence """ L_plus_1, output_len = pred_seq.shape total_len = L_plus_1 + output_len - 1 sum_seq = torch.zeros(total_len, device=pred_seq.device) count_seq = torch.zeros(total_len, device=pred_seq.device) for t in range(L_plus_1): sum_seq[t:t+output_len] += pred_seq[t] count_seq[t:t+output_len] += 1 return sum_seq / count_seq # [L+output_len] def get_load_temperature_spanish(): ''' https://www.kaggle.com/datasets/nicholasjhana/energy-consumption-generation-prices-and-weather ''' # Load the energy dataset and weather features energy_df = pd.read_csv('/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/kaggle/energy_dataset.csv') weather_df = pd.read_csv('/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/kaggle/weather_features.csv') # Convert timestamp columns to datetime format for easier merging and plotting energy_df['time'] = pd.to_datetime(energy_df['time']) weather_df['time'] = pd.to_datetime(weather_df['dt_iso']) # Merge datasets on the 'timestamp' column merged_df = pd.merge(energy_df, weather_df, on='time', how='inner') merged_df = merged_df[['time', 'temp', 'total load actual']].dropna() merged_df = merged_df[::5] time = merged_df["time"].values print(time) exit() temperature = (merged_df["temp"] - 273.15).values # from Kelvin (K) to degrees Celsius (°C), load = merged_df["total load actual"].values/1000 # from MW to (×10³ MW) temperature = gaussian_filter1d(temperature, sigma=2) load = gaussian_filter1d(load, sigma=2) # Plotting temperature and load on the same figure fig, ax1 = plt.subplots(figsize=(14, 6)) # Plot temperature with left y-axis ax1.plot(time, temperature, label='Temperature', color='orange', linewidth=2) ax1.set_ylabel('Temperature (°C)', color='orange', fontsize=20) ax1.tick_params(axis='y', labelcolor='orange', labelsize=20) ax1.tick_params(axis='x', labelsize=20) # Create a second y-axis for load ax2 = ax1.twinx() ax2.plot(time, load, label='Power Load', color='darkblue', linewidth=2) ax2.set_ylabel('Power Load (×10³ MW)', color='darkblue', fontsize=20) ax2.tick_params(axis='y', labelcolor='darkblue', labelsize=20) ax2.tick_params(axis='x', labelsize=20) # Title and layout adjustments fig.suptitle('Temperature and Power Load Over Time', fontsize=20) fig.autofmt_xdate(rotation=45) plt.tight_layout() # plt.savefig("./results/raw_load_temp_spanish.pdf") plt.show() print(time.shape, load.shape, temperature.shape) return time, load, temperature def get_data_spanish_weekly(): """ Weekly load-temperature slices for Spain ————————————————————————————————————————————————— Returns ------- times : np.ndarray, dtype=object, shape (n_weeks,) energy : np.ndarray, shape (n_weeks, 168) temp : np.ndarray, shape (n_weeks, 168) workday : np.ndarray, shape (n_weeks, 168) season_feat : np.ndarray, shape (n_weeks, 168) """ # ---------- raw files -------------------------------------------------- p_energy = "/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/kaggle/energy_dataset.csv" p_weather = "/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/kaggle/weather_features.csv" # ---------- pre-processing & merge ------------------------------------ energy_df = pd.read_csv(p_energy) weather_df = pd.read_csv(p_weather) energy_df["time"] = pd.to_datetime(energy_df["time"], utc=True) weather_df["time"] = pd.to_datetime(weather_df["dt_iso"], utc=True) df = pd.merge(energy_df, weather_df, on="time", how="inner") df = df[::5] df = df[1:] df["time"] = df["time"].dt.tz_convert(None) # or .dt.tz_localize(None) df = df[["time", "temp", "total load actual"]].dropna() df["Date"] = df["time"].dt.date # now works df["day_of_week"] = df["time"].dt.dayofweek df["air_temperature"] = (df["temp"] - 273.15).astype(float) df["meter_reading"] = (df["total load actual"] / 1000).astype(float) # ---------- season helper --------------------------------------------- def get_season(month: int) -> int: return {12: 0, 1: 0, 2: 0, 3: 1, 4: 1, 5: 1, 6: 2, 7: 2, 8: 2, 9: 3, 10: 3, 11: 3}[month] # ---------- daily grouping (24 samples each) --------------------------- meas_cols = ["meter_reading", "air_temperature", "Date", "time"] grouped = df.groupby("Date")[meas_cols + ["day_of_week"]] array_3d, labels_3d, seasons_3d = [], [], [] for date_val in sorted(grouped.groups.keys()): gdf = grouped.get_group(date_val) # make sure we have *exactly* 24 hourly rows if len(gdf) != 24: full_hours = pd.date_range(start=f"{date_val} 00:00:00", end=f"{date_val} 23:00:00", freq="H") tmpl = pd.DataFrame({"time": full_hours}) gdf = pd.merge(tmpl, gdf, on="time", how="left") gdf["Date"] = gdf["time"].dt.date gdf["day_of_week"] = gdf["time"].dt.dayofweek for c in ["meter_reading", "air_temperature"]: gdf[c] = (gdf[c] .interpolate("linear") .ffill() .bfill() ) gdf.fillna({"meter_reading": 0, "air_temperature": 15}, inplace=True) arr = gdf[meas_cols].values w_label = 0 if gdf["day_of_week"].iloc[0] < 5 else 1 season = get_season(gdf["time"].iloc[0].month) array_3d.append(arr) labels_3d.append(np.full(len(arr), w_label)) seasons_3d.append(np.full(len(arr), season)) # ---------- pack consecutive days into full weeks --------------------- n_full_weeks = len(array_3d) // 7 if n_full_weeks == 0: return (np.array([]),) * 5 energy, temp, times, workday, season_feat = [], [], [], [], [] for w in range(n_full_weeks): wk = slice(w * 7, (w + 1) * 7) week_d = array_3d[wk] w_lbls = labels_3d[wk] w_seas = seasons_3d[wk] e = np.concatenate([d[:, 0].astype(float) for d in week_d]) t = np.concatenate([d[:, 1].astype(float) for d in week_d]) ts = np.concatenate([d[:, 3] for d in week_d]) # timestamps wl = np.concatenate([lbl.astype(int) for lbl in w_lbls]) sl = np.concatenate([s.astype(int) for s in w_seas]) if e.size != 168: # incomplete week – skip continue energy.append(gaussian_filter1d(e, sigma=1)) temp.append(gaussian_filter1d(t, sigma=1)) times.append(ts) workday.append(wl) season_feat.append(sl) return (np.array(times, dtype=object), np.array(energy), np.array(temp), np.array(workday), np.array(season_feat)) def get_data_power_consumption(): """ https://www.kaggle.com/datasets/fedesoriano/electric-power-consumption Loads a CSV containing at least: ['Date Time', 'Temperature', 'Zone 1 Power Consumption'] and does a simple time-series plot of Zone 1 vs. Temperature. """ # 1) Load data file_path = "/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/powerconsumption/powerconsumption.csv" # <-- Adjust to your actual CSV df = pd.read_csv(file_path) # 2) Parse datetime and sort # We assume a combined 'Date Time' column, like '2020-01-01 00:10:00' df['Date Time'] = pd.to_datetime(df['Datetime']) df.sort_values(by='Date Time', inplace=True) # 3) Select only needed columns # We pick 'Zone 1 Power Consumption' & 'Temperature' df_filtered = df[['Date Time', 'Temperature', 'PowerConsumption_Zone1']].copy() # 4) Convert to numeric (in case CSV has strings) # Coerce errors => NaN df_filtered['Temperature'] = pd.to_numeric(df_filtered['Temperature'], errors='coerce') df_filtered['Zone 1 Power Consumption'] = pd.to_numeric(df_filtered['PowerConsumption_Zone1'], errors='coerce') # 5) Drop rows with missing values if needed df_filtered.dropna(subset=['Temperature', 'Zone 1 Power Consumption'], inplace=True) scaler = MinMaxScaler() df_filtered[['Temperature', 'Zone 1 Power Consumption']] = scaler.fit_transform( df_filtered[['Temperature', 'Zone 1 Power Consumption']] ) # 6) Simple Plot: Time series of Zone1 and Temperature fig, ax1 = plt.subplots(figsize=(10, 5)) # Plot Zone 1 Power on ax1 color1 = 'tab:blue' ax1.set_xlabel('Date Time') ax1.set_ylabel('Zone 1 Power Consumption', color=color1) ax1.plot(df_filtered['Date Time'], df_filtered['Zone 1 Power Consumption'], color=color1, label='Zone1 Power') ax1.tick_params(axis='y', labelcolor=color1) # Create a second y-axis for Temperature ax2 = ax1.twinx() # shares x-axis color2 = 'tab:red' ax2.set_ylabel('Temperature', color=color2) ax2.plot(df_filtered['Date Time'], df_filtered['Temperature'], color=color2, label='Temperature') ax2.tick_params(axis='y', labelcolor=color2) plt.title('Zone 1 Power Consumption and Temperature Over Time') fig.tight_layout() # -------------------------------------------------------- # 7) Reshape the data: separate by date # => new shape: [#dates, #values_in_one_day] # -------------------------------------------------------- # Extract the date and the time of day (as a string HH:MM:SS) df_filtered['Date'] = df_filtered['Date Time'].dt.date df_filtered['TimeOfDay'] = df_filtered['Date Time'].dt.strftime('%H:%M:%S') # Pivot so each row is one date, each column is a time of day pivot_time = df_filtered.pivot(index='Date', columns='TimeOfDay', values='Date Time') pivot_power = df_filtered.pivot(index='Date', columns='TimeOfDay', values='Zone 1 Power Consumption') pivot_temp = df_filtered.pivot(index='Date', columns='TimeOfDay', values='Temperature') # Sort the columns so time-of-day is in ascending order (00:00:00 < 00:10:00 < ...) pivot_time = pivot_time.reindex(sorted(pivot_time.columns), axis=1) pivot_power = pivot_power.reindex(sorted(pivot_power.columns), axis=1) pivot_temp = pivot_temp.reindex(sorted(pivot_temp.columns), axis=1) # 9) Create workday/weekend label workday_label = np.array([ [1 if pd.Timestamp(date).weekday() >= 5 else 0] * pivot_power.shape[1] for date in pivot_power.index ]) # -------------------------------------------------------- # 8) Plot daily profiles (one line per date) # -------------------------------------------------------- # Plot Zone 1 Power plt.figure(figsize=(10,4)) for date_idx in pivot_power.index: plt.plot(pivot_power.columns, pivot_power.loc[date_idx, :], label=str(date_idx), alpha=0.4, color="gray") plt.title("Daily Profile of Zone 1 Power Consumption") plt.xlabel("Time of Day (HH:MM:SS)") plt.ylabel("Scaled Power Consumption") # Uncomment to show legend with all dates # plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left') plt.tight_layout() # Plot Temperature plt.figure(figsize=(10,4)) for date_idx in pivot_temp.index: plt.plot(pivot_temp.columns, pivot_temp.loc[date_idx, :], label=str(date_idx), alpha=0.4, color="green") plt.title("Daily Profile of Temperature") plt.xlabel("Time of Day (HH:MM:SS)") plt.ylabel("Scaled Temperature") # plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left') plt.tight_layout() # 10) Visualize one week of Power, Temperature, and Workday Label week_index = 0 # Change this to shift the week (e.g., 7 for second week) days_to_plot = 7 power_week = pivot_power.iloc[week_index:week_index+days_to_plot, :].to_numpy().flatten() temp_week = pivot_temp.iloc[week_index:week_index+days_to_plot, :].to_numpy().flatten() label_week = workday_label[week_index:week_index+days_to_plot, :].flatten() time_axis = np.arange(len(power_week)) # X-axis for plotting plt.figure(figsize=(12, 4)) plt.plot(time_axis, power_week, label='Power', linewidth=1) plt.plot(time_axis, temp_week, label='Temperature', linewidth=1) plt.plot(time_axis, label_week, label='Workday Label', linewidth=2, linestyle='--') plt.title("One Week of Power, Temperature, and Workday Labels") plt.xlabel("10-minute Intervals over 7 Days") plt.ylabel("Normalized Value") plt.legend() plt.grid(True) plt.tight_layout() # plt.savefig("results/one_week_data.pdf") plt.show() return np.array(pivot_time), np.array(pivot_power), np.array(pivot_temp) def plot_data(times, energy, temp, workday, season_feat, alpha=0.5, lw=1.0, cmap="viridis"): """ Overlay *all* weeks in four side-by-side sub-figures. Parameters ---------- times, energy, temp, workday, season_feat : list/ndarray Output from your get_data_…_weekly routine. alpha : float Per-curve transparency (≤1). Lower → less clutter. lw : float Line width. cmap : str or matplotlib Colormap Used to give each week a slightly different colour. """ n_weeks = len(times) if n_weeks == 0: print("Nothing to plot.") return # colour map to distinguish weeks (wraps if >256) colours = plt.cm.get_cmap(cmap, n_weeks) fig, axes = plt.subplots( nrows=1, ncols=4, figsize=(22, 4), sharex=False, sharey=False, gridspec_kw={"wspace": 0.25}) date_fmt = mdates.DateFormatter("%b\n%d") # ------------------------------------------------------------- # iterate once, plotting the same week on all four axes # ------------------------------------------------------------- for w in range(n_weeks): c = colours(w) axes[0].plot(times[w], energy[w], color=c, alpha=alpha, lw=lw) axes[1].plot(times[w], temp[w], color=c, alpha=alpha, lw=lw) axes[2].step(times[w], workday[w], where="mid", color=c, alpha=alpha, lw=lw) axes[3].step(times[w], season_feat[w], where="mid", color=c, alpha=alpha, lw=lw) # ------------------------------------------------------------- # cosmetics # ------------------------------------------------------------- axes[0].set_title("Energy (norm.)") axes[0].set_ylabel("0–1") axes[1].set_title("Temperature (norm.)") axes[2].set_title("Weekend flag") axes[2].set_ylim(-0.1, 1.1) axes[3].set_title("Season (0–3)") axes[3].set_ylim(-0.2, 3.2) for ax in axes: ax.xaxis.set_major_formatter(date_fmt) ax.tick_params(axis="x", rotation=45, labelsize=8) fig.suptitle(f"Overlay of {n_weeks} weeks", fontsize=15, y=1.02) plt.tight_layout() plt.show() ## fig, axes = plt.subplots( nrows=1, ncols=4, figsize=(22, 4), sharex=False, sharey=False, gridspec_kw={"wspace": 0.25}) date_fmt = mdates.DateFormatter("%b\n%d") # ------------------------------------------------------------- # iterate once, plotting the same week on all four axes # ------------------------------------------------------------- for w in range(n_weeks): c = colours(w) axes[0].plot(energy[w], color=c, alpha=alpha, lw=lw) axes[1].plot(temp[w], color=c, alpha=alpha, lw=lw) axes[2].plot(workday[w], color=c, alpha=alpha, lw=lw) axes[3].plot(season_feat[w], color=c, alpha=alpha, lw=lw) # ------------------------------------------------------------- # cosmetics # ------------------------------------------------------------- axes[0].set_title("Energy (norm.)") axes[0].set_ylabel("0–1") axes[1].set_title("Temperature (norm.)") axes[2].set_title("Weekend flag") axes[2].set_ylim(-0.1, 1.1) axes[3].set_title("Season (0–3)") axes[3].set_ylim(-0.2, 3.2) for ax in axes: ax.xaxis.set_major_formatter(date_fmt) ax.tick_params(axis="x", rotation=45, labelsize=8) fig.suptitle(f"Overlay of {n_weeks} weeks", fontsize=15, y=1.02) plt.tight_layout() plt.show() def get_data_power_consumption_weekly(): """ Weekly load-temperature slices (Zone-1 household data) ------------------------------------------------------ Returns ------- times : ndarray[object] – n_weeks, each element len = points_per_day*7 energy : ndarray[float] – n_weeks × (points_per_day*7) temp : ndarray[float] – idem workday : ndarray[int] – idem (0 weekday, 1 weekend) season_feat : ndarray[int] – idem (0-winter … 3-autumn) """ csv_path = Path("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/powerconsumption/powerconsumption.csv") # ── 1. Read & basic cleaning ────────────────────────────────────────── df = pd.read_csv(csv_path) # column names vary slightly across versions → be defensive time_col = next(c for c in df.columns if c.lower().startswith(("date time", "datetime"))) temp_col = next(c for c in df.columns if "temp" in c.lower()) power_col= next(c for c in df.columns if "zone1" in c.lower()) df["time"] = pd.to_datetime(df[time_col]) df["air_temperature"] = pd.to_numeric(df[temp_col], errors="coerce") df["meter_reading"] = pd.to_numeric(df[power_col], errors="coerce") df = df[["time", "air_temperature", "meter_reading"]].dropna() df = df[::6] # print(df) df.sort_values("time", inplace=True) for c in ["air_temperature", "meter_reading"]: col_min, col_max = df[c].min(), df[c].max() df[c] = (df[c] - col_min) / (col_max - col_min) # ── 2. Identify full days & points-per-day ──────────────────────────── df["date"] = df["time"].dt.date day_counts = df.groupby("date").size() points_per_day = int(day_counts.mode().iloc[0]) # most common daily length full_dates = day_counts[day_counts == points_per_day].index df = df[df["date"].isin(full_dates)].copy() # ── 3. Season & weekday helpers ─────────────────────────────────────── def get_season(month): return {12:0,1:0,2:0,3:1,4:1,5:1,6:2,7:2,8:2,9:3,10:3,11:3}[month] # ── 4. Daily arrays (guaranteed length = points_per_day) ────────────── meas_cols = ["meter_reading", "air_temperature", "date", "time"] grouped = df.groupby("date")[meas_cols] array_3d, labels_3d, seasons_3d = [], [], [] for d in sorted(grouped.groups.keys()): g = grouped.get_group(d).sort_values("time") # (No need to re-index; we already filtered to full days.) arr = g[meas_cols].values w_label = 0 if g["time"].dt.dayofweek.iloc[0] < 5 else 1 season = get_season(g["time"].iloc[0].month) array_3d.append(arr) labels_3d.append(np.full(points_per_day, w_label)) seasons_3d.append(np.full(points_per_day, season)) # ── 5. Pack into complete weeks (7 consecutive full days) ───────────── n_full_weeks = len(array_3d) // 7 if n_full_weeks == 0: return (np.array([]),) * 5 sigma = max(1, points_per_day // 24) # ≈ 1-hour smoothing energy, temp, times, workday, season_feat = [], [], [], [], [] for w in range(n_full_weeks): wk = slice(w*7, (w+1)*7) week_d, w_lbls, w_seas = array_3d[wk], labels_3d[wk], seasons_3d[wk] e = np.asarray(np.concatenate([d[:, 0] for d in week_d]), dtype=float) t = np.asarray(np.concatenate([d[:, 1] for d in week_d]), dtype=float) ts = np.concatenate([d[:,3] for d in week_d]) wl = np.concatenate(w_lbls) sl = np.concatenate(w_seas) energy.append(gaussian_filter1d(e, sigma=sigma)) temp.append(gaussian_filter1d(t, sigma=sigma)) times.append(ts) workday.append(wl) season_feat.append(sl) # plot_data(times, energy, temp, workday, season_feat) return (np.array(times, dtype=object), np.array(energy), np.array(temp), np.array(workday), np.array(season_feat)) def get_data_kaggle_2(): """ https://www.kaggle.com/datasets/srinuti/residential-power-usage-3years-data-timeseries Loads the 'power_usage_2016_to_2020.csv' and 'weather_2016_2020_daily.csv' datasets, merges them by date, creates daily profiles, and plots a single week of data (Power, Temperature, Workday Label) in a flattened time series. """ load_file = "/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/Kaggle_2/power_usage_2016_to_2020.csv" df_load = pd.read_csv(load_file) df_load['DateTime'] = pd.to_datetime(df_load['StartDate']) df_load['Date'] = df_load['DateTime'].dt.date df_load.rename(columns={'Value (kWh)': 'Power'}, inplace=True) weather_file = "/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/Kaggle_2/weather_2016_2020_daily.csv" df_weather = pd.read_csv(weather_file) df_weather['Date'] = pd.to_datetime(df_weather['Date']).dt.date df_weather.rename(columns={'Temp_avg': 'Temperature'}, inplace=True) df_weather = df_weather[['Date', 'Temperature']] df_merged = pd.merge(df_load, df_weather, on='Date', how='left') df_merged.sort_values(by='DateTime', inplace=True) df_merged.dropna(subset=['Power', 'Temperature'], inplace=True) scaler = MinMaxScaler() df_merged[['Power', 'Temperature']] = scaler.fit_transform(df_merged[['Power', 'Temperature']]) df_merged['TimeOfDay'] = df_merged['DateTime'].dt.strftime('%H:%M:%S') pivot_power = df_merged.pivot(index='Date', columns='TimeOfDay', values='Power') pivot_temp = df_merged.pivot(index='Date', columns='TimeOfDay', values='Temperature') pivot_time = df_merged.pivot(index='Date', columns='TimeOfDay', values='DateTime') # Sort columns so time-of-day is in ascending order pivot_power = pivot_power.reindex(sorted(pivot_power.columns), axis=1) pivot_temp = pivot_temp.reindex(sorted(pivot_temp.columns), axis=1) pivot_time = pivot_time.reindex(sorted(pivot_time.columns), axis=1) pivot_dates = pivot_power.index # these are datetime.date objects df_day = df_load.groupby('Date')['day_of_week'].first().reindex(pivot_dates) weekend_indicator = df_day.isin([5, 6]).astype(int).values # 1 if day_of_week in [6,7], else 0 workday_label_2D = np.array([ [weekend_indicator[i]] * pivot_power.shape[1] for i in range(len(pivot_dates)) ]) print(workday_label_2D) plt.figure(figsize=(10, 4)) for date_idx in pivot_power.index: plt.plot( pivot_power.columns, pivot_power.loc[date_idx, :], label=str(date_idx), alpha=0.4, color="gray" ) plt.title("Daily Profile of Power") plt.xlabel("Time of Day") plt.ylabel("Scaled Power") plt.tight_layout() plt.show() # 7b) Plot daily temperature profiles plt.figure(figsize=(10, 4)) for date_idx in pivot_temp.index: plt.plot( pivot_temp.columns, pivot_temp.loc[date_idx, :], label=str(date_idx), alpha=0.4, color="blue" ) plt.title("Daily Profile of Temperature") plt.xlabel("Time of Day") plt.ylabel("Scaled Temperature") # plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left') plt.tight_layout() plt.show() # ------------------------------------------------ # 8) Select ONE WEEK of data and flatten it into a single time-series # ------------------------------------------------ # Let's say we pick the first 7 days in the pivot: week_index = 10 # which chunk of 7 days to pick days_to_plot = 7 chosen_dates = pivot_power.index[week_index:week_index + days_to_plot] power_week = pivot_power.loc[chosen_dates, :].to_numpy().flatten() temp_week = pivot_temp.loc[chosen_dates, :].to_numpy().flatten() label_week = workday_label_2D[week_index:week_index + days_to_plot, :].flatten() # The X-axis will be one point per hour (or half-hour, etc.) times 7 days time_axis = np.arange(len(power_week)) # ------------------------------------------------ # 9) Plot one-week time series of Power, Temperature, Workday # ------------------------------------------------ plt.figure(figsize=(12, 4)) plt.plot(time_axis, power_week, label='Power', linewidth=1) plt.plot(time_axis, temp_week, label='Temperature', linewidth=1) plt.plot(time_axis, label_week, label='Workday Label', linewidth=2, linestyle='--') print(list(power_week)) plt.title("One Week of Power, Temperature, and Workday Labels") plt.xlabel("Hourly Points over 7 Days") plt.ylabel("Scaled Value / Label") plt.legend() plt.grid(True) plt.tight_layout() plt.show() return pivot_power, pivot_temp, workday_label_2D def get_data_residential_weekly(): """ Residential power-usage data (2016-2020) → weekly slices. Returns ------- times : np.ndarray (dtype=object) – shape (n_weeks,) each element is a 1-D array of datetime stamps energy : np.ndarray, shape (n_weeks, points_per_day*7) temp : np.ndarray, same shape workday : np.ndarray, same shape, int {0,1} season_feat : np.ndarray, same shape, int {0,1,2,3} """ # ── paths ────────────────────────────────────────────────────────────── p_load = Path("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/Kaggle_2/power_usage_2016_to_2020.csv") p_weather = Path("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/Kaggle_2/weather_2016_2020_daily.csv") # ── 1. read & basic merge (load = hourly, weather = daily) ─────────── df_load = pd.read_csv(p_load) df_load["time"] = pd.to_datetime(df_load["StartDate"]) df_load["date"] = df_load["time"].dt.date df_load.rename(columns={"Value (kWh)": "meter_reading"}, inplace=True) df_weather = pd.read_csv(p_weather) df_weather["date"] = pd.to_datetime(df_weather["Date"]).dt.date df_weather.rename(columns={"Temp_avg": "air_temperature"}, inplace=True) df = pd.merge(df_load[["time", "date", "meter_reading", "day_of_week"]], df_weather[["date", "air_temperature"]], on="date", how="left") # ── 2. keep numeric & drop NaN ───────────────────────────────────────── df["meter_reading"] = pd.to_numeric(df["meter_reading"], errors="coerce") df["air_temperature"] = pd.to_numeric(df["air_temperature"], errors="coerce") df.dropna(subset=["meter_reading", "air_temperature"], inplace=True) df.sort_values("time", inplace=True) # min-max normalise both variables globally for c in ["meter_reading", "air_temperature"]: v_min, v_max = df[c].min(), df[c].max() df[c] = (df[c] - v_min) / (v_max - v_min) # ── 3. ensure full-day rows & discover points_per_day ───────────────── day_counts = df.groupby("date").size() points_per_day = int(day_counts.mode().iloc[0]) # most common length full_dates = day_counts[day_counts == points_per_day].index df = df[df["date"].isin(full_dates)].copy() # ── 4. helpers ───────────────────────────────────────────────────────── def get_season(m): # 0=winter … 3=autumn return {12:0,1:0,2:0,3:1,4:1,5:1,6:2,7:2,8:2,9:3,10:3,11:3}[m] meas_cols = ["meter_reading", "air_temperature", "date", "time"] grouped = df.groupby("date")[meas_cols] # ── 5. daily arrays (guaranteed identical length) ───────────────────── daily, d_labels, d_seasons = [], [], [] for d in sorted(grouped.groups.keys()): g = grouped.get_group(d).sort_values("time") arr = g[meas_cols].values weekend = 1 if g["time"].dt.dayofweek.iloc[0] >= 5 else 0 season = get_season(g["time"].iloc[0].month) daily.append(arr) d_labels.append(np.full(points_per_day, weekend, dtype=int)) d_seasons.append(np.full(points_per_day, season, dtype=int)) # ── 6. build consecutive 7-day blocks starting at 00:00 ─────────────── n_full_weeks = len(daily) // 7 if n_full_weeks == 0: return (np.array([]),) * 5 # sigma = max(1, points_per_day // 24) # ≈ 1-hour smoothing energy, temp, times, workday, season_feat = [], [], [], [], [] for w in range(n_full_weeks): sl = slice(w*7, (w+1)*7) week_d, w_lbl, w_sea = daily[sl], d_labels[sl], d_seasons[sl] e = np.asarray(np.concatenate([d[:,0] for d in week_d]), dtype=float) t = np.asarray(np.concatenate([d[:,1] for d in week_d]), dtype=float) ts = np.concatenate([d[:,3] for d in week_d]) wl = np.concatenate(w_lbl) sf = np.concatenate(w_sea) energy.append(gaussian_filter1d(e, sigma=1)) temp.append(gaussian_filter1d(t, sigma=1)) times.append(ts) workday.append(wl) season_feat.append(sf) # plot_data(times, energy, temp, workday, season_feat) return (np.array(times, dtype=object), np.array(energy), np.array(temp), np.array(workday), np.array(season_feat)) def get_data_solar_weather_weekly(): """ Returns ------- times : np.ndarray (dtype=object) shape (n_weeks,) energy : np.ndarray shape (n_weeks, points_per_day*7) temp : np.ndarray same shape workday : np.ndarray same shape, int {0,1} season_feat : np.ndarray same shape, int {0,1,2,3} """ # ── 1. read & basic cleaning ───────────────────────────────────────── p_csv = Path("/Users/muhaoguo/Documents/study/Paper_Projects/PESGM/data/solar_weather.csv") df = pd.read_csv(p_csv, parse_dates=["Time"]) # you sampled 1000:10000 in the draft – keep that if desired # df = df.iloc[1000:10000].copy() df = df.iloc[::4].copy() # keep two numeric columns & drop NaN df = df[["Time", "Energy delta[Wh]", "temp"]].rename( columns={"Energy delta[Wh]": "meter_reading", "temp": "air_temperature"}) df["meter_reading"] = pd.to_numeric(df["meter_reading"], errors="coerce") df["air_temperature"] = pd.to_numeric(df["air_temperature"], errors="coerce") df.dropna(inplace=True) df.sort_values("Time", inplace=True) # print(df) # ── 2. global min-max normalisation ───────────────────────────────── for c in ["meter_reading", "air_temperature"]: vmin, vmax = df[c].min(), df[c].max() df[c] = (df[c] - vmin) / (vmax - vmin) # ── 3. identify full days / sample rate ───────────────────────────── df["date"] = df["Time"].dt.date day_counts = df.groupby("date").size() pts_per_day = int(day_counts.mode().iloc[0]) # modal length full_dates = day_counts[day_counts == pts_per_day].index df = df[df["date"].isin(full_dates)].copy() # ── 4. helpers ────────────────────────────────────────────────────── def get_season(m): # 0=winter,1=spring,2=summer,3=autumn return {12:0,1:0,2:0,3:1,4:1,5:1,6:2,7:2,8:2,9:3,10:3,11:3}[m] meas_cols = ["meter_reading", "air_temperature", "date", "Time"] grouped = df.groupby("date")[meas_cols] daily, d_wd, d_sea = [], [], [] for d in sorted(grouped.groups.keys()): g = grouped.get_group(d).sort_values("Time") arr = g[meas_cols].values wd_flag = 1 if g["Time"].dt.dayofweek.iloc[0] >= 5 else 0 season = get_season(g["Time"].iloc[0].month) daily.append(arr) d_wd.append(np.full(pts_per_day, wd_flag, dtype=int)) d_sea.append(np.full(pts_per_day, season, dtype=int)) # ── 5. consecutive 7-day blocks, starting at 00:00 ────────────────── n_full_weeks = len(daily) // 7 if n_full_weeks == 0: return (np.array([]),)*5 sigma = max(1, pts_per_day // 24) # ≈ one-hour smoothing energy, temp, times, workday, season_feat = [], [], [], [], [] for w in range(n_full_weeks): sl = slice(w*7, (w+1)*7) wk_d, wk_wd, wk_sea = daily[sl], d_wd[sl], d_sea[sl] e = np.asarray(np.concatenate([d[:,0] for d in wk_d]), dtype=float) t = np.asarray(np.concatenate([d[:,1] for d in wk_d]), dtype=float) ts = np.concatenate([d[:,3] for d in wk_d]) wl = np.concatenate(wk_wd) sf = np.concatenate(wk_sea) energy.append(gaussian_filter1d(e, sigma=sigma)) temp.append(gaussian_filter1d(t, sigma=sigma)) times.append(ts) workday.append(wl) season_feat.append(sf) # plot_data(times, energy, temp, workday, season_feat) return (np.array(times, dtype=object), np.array(energy), np.array(temp), np.array(workday), np.array(season_feat)) if __name__ == "__main__": times, energy, temp, workday, season_feat = get_data_building_weather_weekly() print(times.shape, energy.shape, temp.shape, workday.shape, season_feat.shape) times, energy, temp, workday, season_feat = get_data_spanish_weekly() print(times.shape, energy.shape, temp.shape, workday.shape, season_feat.shape) times, energy, temp, workday, season_feat = get_data_power_consumption_weekly() print(times.shape, energy.shape, temp.shape, workday.shape, season_feat.shape) times, energy, temp, workday, season_feat = get_data_residential_weekly() print(times.shape, energy.shape, temp.shape, workday.shape, season_feat.shape) times, energy, temp, workday, season_feat = get_data_solar_weather_weekly() print(times.shape, energy.shape, temp.shape, workday.shape, season_feat.shape)