# - Random Forest Regression on California Housing Dataset
# - RandomizedSearchCV vs GridSearchCV:
# - GridSearchCV exhaustively tries every combination of hyperparameters in the provided grid.
# - RandomizedSearchCV samples a fixed number of random combinations from the grid, making it faster for large search spaces.
# - Both are used for hyperparameter optimization, but RandomizedSearchCV is more efficient when the grid is large or when you want a quick search.
import numpy as np  # For numerical operations
import pandas as pd  # For data manipulation
import matplotlib.pyplot as plt  # For plotting
import seaborn as sns  # For advanced visualizations
from sklearn.datasets import fetch_california_housing  # To load the dataset
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV  # For splitting and hyperparameter search
from sklearn.ensemble import RandomForestRegressor  # Random Forest regression model
from sklearn.metrics import mean_squared_error, r2_score  # For regression metrics
import warnings  # To suppress warnings
warnings.filterwarnings('ignore')  # Ignore warnings for cleaner output
# Load California housing dataset
cal_data = fetch_california_housing()  # Fetch the dataset
X = pd.DataFrame(cal_data.data, columns=cal_data.feature_names)  # Features as DataFrame
y = cal_data.target  # Target variable (median house value)
# Train-test split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)  # Split data into train and test sets
# Fixed lists for hyperparameters
param_list = {
    'n_estimators': [50, 100, 150, 200],  # Number of trees in the forest
    'max_depth': [5, 10, 15, None],  # Maximum depth of the tree
    'min_samples_split': [2, 4, 6, 8],  # Minimum samples required to split a node
    'min_samples_leaf': [1, 2, 3],  # Minimum samples required at a leaf node
    'max_features': ['sqrt', 'log2', None],  # Number of features to consider at each split
    'bootstrap': [True, False]  # Whether bootstrap samples are used
}
# RandomizedSearchCV for Random Forest Regressor
rf = RandomForestRegressor(random_state=42)  # Initialize Random Forest Regressor
random_search = RandomizedSearchCV(
    rf, param_distributions=param_list, n_iter=20, cv=3, scoring='neg_mean_squared_error', n_jobs=-1, random_state=42)  # Randomized hyperparameter search
random_search.fit(X_train, y_train)  # Fit model to training data
print('Best parameters (RandomizedSearchCV):', random_search.best_params_)  # Print best parameters
print('Best score (RandomizedSearchCV):', -random_search.best_score_)  # Print best score (MSE)
rf_random = random_search.best_estimator_  # Get best model
# GridSearchCV for Random Forest Regressor (commented out)
# grid_search = GridSearchCV(
#     rf, param_grid=param_list, cv=3, scoring='neg_mean_squared_error', n_jobs=-1)  # Grid hyperparameter search
# grid_search.fit(X_train, y_train)  # Fit model to training data
# print('Best parameters (GridSearchCV):', grid_search.best_params_)  # Print best parameters
# print('Best score (GridSearchCV):', -grid_search.best_score_)  # Print best score (MSE)
# rf_grid = grid_search.best_estimator_  # Get best model
# Evaluate RandomizedSearchCV model
y_pred = rf_random.predict(X_test)  # Predict on test set
mse = mean_squared_error(y_test, y_pred)  # Calculate mean squared error
r2 = r2_score(y_test, y_pred)  # Calculate R^2 score
print(f'\nRandomizedSearchCV Test MSE: {mse:.4f}')  # Print test MSE
print(f'RandomizedSearchCV Test R2: {r2:.4f}')  # Print test R^2
plt.figure(figsize=(6,5))  # Set figure size
plt.scatter(y_test, y_pred, alpha=0.5)  # Scatter plot of actual vs predicted
plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--')  # Diagonal reference line
plt.xlabel('Actual Values')  # X-axis label
plt.ylabel('Predicted Values')  # Y-axis label
plt.title('RandomizedSearchCV Actual vs Predicted')  # Plot title
plt.tight_layout()  # Adjust layout
plt.show()  # Show plot
# Feature importance visualization for best RandomizedSearchCV model
importances = rf_random.feature_importances_  # Get feature importances
indices = np.argsort(importances)[::-1]  # Sort features by importance
plt.figure(figsize=(8,5))  # Set figure size
plt.bar(range(X.shape[1]), importances[indices], align='center')  # Bar plot of importances
plt.xticks(range(X.shape[1]), [X.columns[i] for i in indices], rotation=45)  # Feature names as x-ticks
plt.title('Feature Importances (RandomizedSearchCV Best Model)')  # Plot title
plt.tight_layout()  # Adjust layout
plt.show()  # Show plot
# Visualize a single tree from the Random Forest
from sklearn.tree import plot_tree
plt.figure(figsize=(20,10))
plot_tree(rf_random.estimators_[0], feature_names=X.columns, filled=True, max_depth=2)
plt.title('Random Forest Regression: Example Tree (Depth=2)')
plt.show()

# Titanic Classification with Random Forest (Simplified)
import pandas as pd
import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn.preprocessing import LabelEncoder, StandardScaler
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.tree import plot_tree
# Load and preprocess Titanic dataset
X = sns.load_dataset('titanic').drop(['survived', 'deck', 'embark_town', 'alive', 'class', 'who'], axis=1)
X['age'] = X['age'].fillna(X['age'].median())
X['fare'] = X['fare'].fillna(X['fare'].median())
X['embarked'] = X['embarked'].fillna(X['embarked'].mode()[0])
X['alone'] = X['alone'].fillna(X['alone'].mode()[0])
for col in ['sex', 'embarked', 'alone']:
    X[col] = LabelEncoder().fit_transform(X[col])
X = X.drop(['adult_male'], axis=1)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
y = sns.load_dataset('titanic')['survived']
# Train-test split
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.3, random_state=42, stratify=y)
# Random Forest with RandomizedSearchCV
rf = RandomForestClassifier(random_state=42)
param_dist = {
    'n_estimators': [100, 200, 300],
    'max_depth': [5, 10, None],
    'min_samples_split': [2, 4, 6],
    'min_samples_leaf': [1, 2, 3],
    'max_features': ['sqrt', 'log2']
}
random_search = RandomizedSearchCV(rf, param_distributions=param_dist, n_iter=15, cv=3, scoring='accuracy', n_jobs=-1, random_state=42)
random_search.fit(X_train, y_train)
rf_best = random_search.best_estimator_
# Evaluation
y_pred = rf_best.predict(X_test)
print('Test Accuracy:', accuracy_score(y_test, y_pred))
print(classification_report(y_test, y_pred))
cm = confusion_matrix(y_test, y_pred)
plt.figure(figsize=(5,4))
sns.heatmap(cm, annot=True, fmt='d', cmap='Greens')
plt.title('Titanic Classification Confusion Matrix')
plt.xlabel('Predicted')
plt.ylabel('Actual')
plt.show()
# Feature importance
importances = rf_best.feature_importances_
indices = np.argsort(importances)[::-1]
plt.figure(figsize=(8,5))
plt.bar(range(X.shape[1]), importances[indices], align='center')
plt.xticks(range(X.shape[1]), [X.columns[i] for i in indices], rotation=45)
plt.title('Feature Importances (Titanic Random Forest)')
plt.tight_layout()
plt.show()
# Visualize tree with most important root split
root_feature = indices[0]
tree_idx = next((i for i, est in enumerate(rf_best.estimators_) if est.tree_.feature[0] == root_feature), 0)
plt.figure(figsize=(20,10))
plot_tree(rf_best.estimators_[tree_idx], feature_names=X.columns, filled=True, max_depth=2)
plt.title(f'Titanic Random Forest: Example Tree (Root={X.columns[root_feature]})')
plt.show()