process_hp_data_4.py

# -*- coding: utf-8 -*-
"""
Created on Wed Jan 22 15:17:22 2025

@author: luisfernando
"""

import pandas as pd
import os
import sys
import gc
import time
import joblib  # Import joblib for saving/loading model

from sklearn.preprocessing import StandardScaler

from sklearn.model_selection import train_test_split

# Scikit-learn models
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsRegressor

import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Function 1: evaluate all models
def evaluate_model(model, X_train, X_test, y_train, y_test, model_name="Model", 
                   feature_set_name="FeatureSet", target_name="Target",
                   output_dir='plots'
                   ):
    """
    Generic function to train, evaluate, and plot results for a regression model.
    
    Parameters:
        model: The regression model (e.g., LinearRegression(), RandomForestRegressor()).
        X_train: Training features.
        X_test: Testing features.
        y_train: Training target values.
        y_test: Testing target values.
        model_name: Name of the model for display purposes.
        feature_set_name: Name of the feature set for identification in the plot.
        target_name: Name of the target variable for identification in the plot.
        output_dir: Directory where plots will be saved.
    """
    # Train the model
    model.fit(X_train, y_train)
    
    # Make predictions
    y_pred = model.predict(X_test)
    
    # Clip negative predictions to 0
    y_pred_clipped = np.maximum(y_pred, 0)
    
    # Calculate metrics
    mse = mean_squared_error(y_test, y_pred_clipped)
    rmse = np.sqrt(mse)
    mae = mean_absolute_error(y_test, y_pred_clipped)
    r2 = r2_score(y_test, y_pred_clipped)
    
    # Print metrics
    # Prepare metrics string
    metrics_str = (
        f"\n{model_name} Metrics ({feature_set_name} -> {target_name}):\n"
        f"Mean Squared Error (MSE): {mse:.2f}\n"
        f"Root Mean Squared Error (RMSE): {rmse:.2f}\n"
        f"Mean Absolute Error (MAE): {mae:.2f}\n"
        f"R-squared (R²): {r2:.2f}\n"
    )
    
    # Print metrics to console
    print(metrics_str)
    
    '''
    The figure below takes too long, we better simplify that process
    '''
    '''
    # Plot True vs Predicted Values
    plt.figure(figsize=(10, 6))
    plt.scatter(y_test, y_pred_clipped, alpha=0.7, label="Predictions")
    plt.plot([min(y_test), max(y_test)], [min(y_test), max(y_test)], 
             color='red', linestyle='--', label="Perfect Prediction")
    plt.xlabel("True Values (y_test)")
    plt.ylabel("Predicted Values (y_pred_clipped)")
    plt.title(f"True vs Predicted ({model_name}, {feature_set_name} -> {target_name})")
    plt.legend()
    plt.grid(True)
    plt.show()
    '''
    # ---- Limit the Data for Plotting ----
    # Select a subset for visualization (e.g., 1000 points or less)
    subset_size = min(50000, len(y_test))  # Limit to 1000 points if dataset is larger
    sampled_indices = np.random.choice(len(y_test), size=subset_size, replace=False)

    # Get the sampled true and predicted values
    y_test_sampled = np.array(y_test)[sampled_indices]
    y_pred_sampled = np.array(y_pred_clipped)[sampled_indices]

    # Plot True vs Predicted Values (sampled)
    plt.figure(figsize=(10, 6))
    plt.scatter(y_test_sampled, y_pred_sampled, alpha=0.7, label="Predictions (Sampled)")
    plt.plot([min(y_test_sampled), max(y_test_sampled)], 
             [min(y_test_sampled), max(y_test_sampled)], 
             color='red', linestyle='--', label="Perfect Prediction")
    plt.xlabel("True Values (y_test)")
    plt.ylabel("Predicted Values (y_pred_clipped)")
    plt.title(f"True vs Predicted ({model_name}, {feature_set_name} -> {target_name})")
    plt.legend()
    plt.grid(True)
    #plt.show()
    #plt.pause(0.001)  # Pause briefly to allow the figure to render

    print('Figure has plotted, now to printing')
    # Save the plot
    os.makedirs(output_dir, exist_ok=True)
    filename = f"{output_dir}/{model_name}_{feature_set_name}_{target_name}.png"
    plt.savefig(filename)
    print(f"Plot saved as: {filename}")
    plt.close()  # Close the plot to avoid display overlap
    # print('This happened?***')

    return metrics_str


# Function 2: save the model
def save_model(model, filename, compress=3):
    try:
        # Ensure directory exists
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        
        # Save with compression
        joblib.dump(model, filename, compress=compress)
        print(f"Model saved successfully: {filename}")
    except Exception as e:
        print(f"Error saving model {filename}: {e}")

'''
Start the process below.
'''
START_PROCESS = time.time()

# Define models
models = {
    'Linear Regression': LinearRegression(),
    'Decision Tree': DecisionTreeRegressor(max_depth=5, random_state=555),
    'Random Forest': RandomForestRegressor(n_estimators=100, random_state=555),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=100, random_state=555),
    'SVR': SVR(kernel='rbf', C=1.0),
    'KNN': KNeighborsRegressor(n_neighbors=5)
}

# Initialize scaler
scaler = StandardScaler()

JSON_STR = "4"  # Replace with your desired string

# 1) Read the parquet files to process the model (check first):
directory = "./"

# parquet_files = [f for f in os.listdir(directory) if f.startswith("simple_parquet") and f.endswith(".parquet") and len(f.split('_')) == 3]
parquet_files = [f for f in os.listdir(directory) if f.startswith("simple_parquet") and f.endswith(".parquet") and JSON_STR in f.split('_')[-1]]

# Append multiple dataframes to print csv and obseve data.
BOOL_DF_APPEND = True
BOOL_VISUAL_DF_APPEND = False
dataframes = []

if BOOL_DF_APPEND:
    for file in parquet_files:
        try:
            # Read the Parquet file and filter for non-efficiency factors
            df = pd.read_parquet(directory + file)
            df_select = df[['timestamp', 
                           'out.electricity.heating.energy_consumption',
                           'out.electricity.heating_hp_bkup.energy_consumption',
                           'Diff',
                           'sqft',
                           'representative_income',
                           'occupants',
                           'usage_level',
                           'rmb_heating_primary',
                           'rmb_heatpump_backup',
                            'vintage',
                            'rmb_window_area',
                            'duct_leakage_and_insulation',
                            'heating_setpoint',
                            'hvac_heating_efficiency']]

            # print('Stop here to check')
            # sys.exit()

            # Append the DataFrame to the list
            dataframes.append(df_select)
            print(f"Successfully read {file}")
        except Exception as e:
            print(f"Error reading {file}: {e}")

    # Concatenate them
    combined_df = pd.concat(dataframes, ignore_index=True)

    if BOOL_VISUAL_DF_APPEND:
        # Create the filename
        filename = f"all_states_{JSON_STR}.csv"

        # Save the DataFrame as a CSV
        combined_df.to_csv(filename, index=False)

# Explicitly delete the `dataframes` list
del dataframes

# Force garbage collection
gc.collect()

# Randomly sample 1 million rows (or all rows if fewer than 1M exist)
# num_samples = min(1_000_000, len(df))
num_samples = min(100_000, len(df))
df_sampled = combined_df.sample(n=num_samples, random_state=555)

# print('Stop here to print csv that allows good visualzations.')
# sys.exit()

# Define the feature sets
feature_sets = {
    "Set 1": [
        'out.electricity.heating.energy_consumption',
        'Diff'
    ],
    "Set 2": [
        #'timestamp', 
        'out.electricity.heating.energy_consumption',
        'Diff',
        'sqft',
        'representative_income',
        'occupants',
        'usage_level',
        'rmb_heating_primary',
        'rmb_heatpump_backup'
    ],
    "Set 3": [
        #'timestamp', 
        'out.electricity.heating.energy_consumption',
        'Diff',
        'vintage',
        'rmb_window_area',
        'duct_leakage_and_insulation',
        'heating_setpoint',
        'hvac_heating_efficiency'
    ],
    "Set 4": [
        #'timestamp', 
        'out.electricity.heating.energy_consumption',
        'Diff',
        'sqft',
        'representative_income',
        'occupants',
        'usage_level',
        'rmb_heating_primary',
        'rmb_heatpump_backup',
        'vintage',
        'rmb_window_area',
        'duct_leakage_and_insulation',
        'heating_setpoint',
        'hvac_heating_efficiency'
    ]
}

# Define the targets
targets = {
    "Backup Heat": 'out.electricity.heating_hp_bkup.energy_consumption',
    "Main Heat": 'out.electricity.heating.energy_consumption'
}

# Models to evaluate
models = {
    "Linear Regression": LinearRegression(),
    "Random Forest": RandomForestRegressor(n_estimators=100, random_state=555),
    "Gradient Boosting": GradientBoostingRegressor(n_estimators=100, random_state=555),
    "SVR": SVR(kernel='rbf'),
    "Decision Tree": DecisionTreeRegressor(random_state=555),
    "K-Neighbors": KNeighborsRegressor(n_neighbors=5)
}

# Create a directory to store models
model_dir = "saved_models_" + JSON_STR
os.makedirs(model_dir, exist_ok=True)

# Initialize scaler
scaler = StandardScaler()

# Define the output file name using JSON_STR
output_file = f"model_evaluation_{JSON_STR}.txt"

# Open the output file in write mode before the loops
with open(output_file, "w") as f:
    # Iterate through feature sets
    for set_name, features in feature_sets.items():
        print("--------------------------------------------------")
        print(f"\nHeader for Feature Set: {set_name} ({features})")

        # Iterate through targets
        for target_name, target in targets.items():
            print("--------------------------------------------------")
            print(f"\nHeader for Target: {target_name}")

            # Prepare the data
            X = df_sampled[features]
            y = df_sampled[target]

            # Handle timestamp if included
            if 'Main Heat' == target_name:
                # X['timestamp'] = pd.to_datetime(X['timestamp'])
                # X['hour'] = X['timestamp'].dt.hour
                # X['month'] = X['timestamp'].dt.month
                X = X.drop(columns=['out.electricity.heating.energy_consumption'])

            # Scale features
            X_scaled = scaler.fit_transform(X)

            # Split the data
            X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=555)

            # Evaluate models
            for model_name, model in models.items():
                print("--------------------------------------------------")
                print(f"\nEvaluating Feature Set: {set_name} ({features})")
                print(f"\nTarget: {target_name}")
                print(f"\nModel: {model_name}")

                f.write("--------------------------------------------------\n")
                f.write(f"Evaluating Feature Set: {set_name} ({features})\n")
                f.write(f"Target: {target_name}\n")
                f.write(f"Model: {model_name}\n")

                metrics_str = evaluate_model(
                    model, X_train, X_test, y_train, y_test, 
                    model_name=model_name, 
                    feature_set_name=set_name, 
                    target_name=target_name, 
                    output_dir="plots" + '_' + JSON_STR
                )
                # Write the metrics to the file
                f.write(metrics_str + "\n")

                '''
                # Save the model
                model_filename = f"{model_dir}/{model_name}_{set_name}_{target_name}.joblib"
                joblib.dump(model, model_filename)
                print(f"Model saved as: {model_filename}")
                '''
                # Example usage in the loop
                model_filename = f"{model_dir}/{model_name}_{set_name}_{target_name}.joblib"
                print(model_filename)
                save_model(model, model_filename)
                print(f"Model saved as: {model_filename}")
                f.write(f"Model saved as: {model_filename}\n")

END_PROCESS = time.time()
TIME_ELAPSED = -START_PROCESS + END_PROCESS
print('\n TIME ELAPSED:')
print(str(TIME_ELAPSED) + ' seconds /', str(TIME_ELAPSED/60) + ' minutes.')