Add application file

app.py

@@ -0,0 +1,296 @@
+"""
+Gradio Interface for Training Regression Models
+
+This script provides a Gradio-based user interface to train regression models using various datasets 
+and algorithms. It enables seamless interaction by allowing users to select models, preprocess data, 
+and specify hyperparameters through an intuitive UI.
+
+Features:
+- **Model Selection**: Choose from a list of available regression algorithms located in `models/supervised/regression`.
+- **Dataset Input Options**:
+  - Upload a local CSV file.
+  - Specify a path to a dataset.
+  - Download datasets from Kaggle using the `kaggle.json` API credentials.
+- **Hyperparameter Customization**: Modify model parameters like test size, random state, cross-validation folds, 
+  and more directly in the interface.
+- **Visualizations**: Generate plots like actual vs. predicted graphs after training.
+- **Live Feedback**: Outputs training metrics, best hyperparameters, and paths to saved models.
+
+Structure:
+1. **Helper Functions**:
+   - `get_model_modules`: Dynamically fetches available regression models.
+   - `download_kaggle_data`: Handles Kaggle dataset downloads.
+   - `train_model`: Constructs and executes the command for training models.
+   - `get_columns_from_data`: Extracts column names from the dataset for UI selection.
+
+2. **Gradio UI Components**:
+   - Allows users to toggle between different dataset input methods.
+   - Updates column dropdowns dynamically based on the dataset.
+   - Executes the training script and displays results and visualizations.
+
+Usage:
+- Place this script in the `interfaces/gradio/` directory of the project.
+- Ensure that the project structure adheres to the specified layout.
+- Run the script, and a Gradio interface will be launched for training models interactively.
+
+Requirements:
+- Python 3.7 or higher
+- Required Python libraries specified in `requirements.txt`
+- Properly structured project with `train_regression_model.py` and model modules.
+
+"""
+
+import gradio as gr
+import pandas as pd
+import os
+import subprocess
+import sys
+import glob
+import re
+
+# Add the project root directory to the Python path
+current_dir = os.path.dirname(os.path.abspath(__file__))
+project_root = os.path.abspath(os.path.join(current_dir, '../../'))
+sys.path.append(project_root)
+
+def get_model_modules():
+    # Get the list of available model modules
+    models_dir = os.path.join(project_root, 'models', 'supervised', 'regression')
+    model_files = glob.glob(os.path.join(models_dir, '*.py'))
+
+    # Debugging: Print the models directory and found files
+    print(f"Looking for model files in: {models_dir}")
+    print(f"Found model files: {model_files}")
+
+    models = [os.path.splitext(os.path.basename(f))[0] for f in model_files if not f.endswith('__init__.py')]
+    model_modules = [f"{model}" for model in models]
+    return model_modules
+
+def download_kaggle_data(json_path, competition_name):
+    # Import the get_kaggle_data function
+    from data.datasets.kaggle_data import get_kaggle_data
+
+    data_path = get_kaggle_data(json_path=json_path, data_name=competition_name, is_competition=True)
+    return data_path
+
+def train_model(model_module, data_option, data_file, data_path, data_name_kaggle, kaggle_json_file, competition_name,
+                target_variable, drop_columns, test_size, random_state, log_transform, cv_folds,
+                scoring_metric, model_save_path, results_save_path, visualize):
+
+    # Determine data_path
+    if data_option == 'Upload Data File':
+        if data_file is None:
+            return "Please upload a data file.", None
+        data_path = data_file  # data_file is the path to the uploaded file
+    elif data_option == 'Provide Data Path':
+        if not os.path.exists(data_path):
+            return "Provided data path does not exist.", None
+    elif data_option == 'Download from Kaggle':
+        if kaggle_json_file is None:
+            return "Please upload your kaggle.json file.", None
+        else:
+            # Save the kaggle.json file to ~/.kaggle/kaggle.json
+            import shutil
+            kaggle_config_dir = os.path.expanduser('~/.kaggle')
+            os.makedirs(kaggle_config_dir, exist_ok=True)
+            kaggle_json_path = os.path.join(kaggle_config_dir, 'kaggle.json')
+            shutil.copy(kaggle_json_file.name, kaggle_json_path)
+            os.chmod(kaggle_json_path, 0o600)
+        data_dir = download_kaggle_data(json_path=kaggle_json_path, competition_name=competition_name)
+        if data_dir is None:
+            return "Failed to download data from Kaggle.", None
+        # Use the specified data_name_kaggle
+        data_path = os.path.join(data_dir, data_name_kaggle)
+        if not os.path.exists(data_path):
+            return f"{data_name_kaggle} not found in the downloaded Kaggle data.", None
+    else:
+        return "Invalid data option selected.", None
+
+    # Prepare command-line arguments
+    cmd = [sys.executable, os.path.join(project_root, 'scripts', 'train_regression_model.py')]
+    cmd.extend(['--model_module', model_module])
+    cmd.extend(['--data_path', data_path])
+    cmd.extend(['--target_variable', target_variable])
+
+    if drop_columns:
+        cmd.extend(['--drop_columns', ','.join(drop_columns)])
+    if test_size != 0.2:
+        cmd.extend(['--test_size', str(test_size)])
+    if random_state != 42:
+        cmd.extend(['--random_state', str(int(random_state))])
+    if log_transform:
+        cmd.append('--log_transform')
+    if cv_folds != 5:
+        cmd.extend(['--cv_folds', str(int(cv_folds))])
+    if scoring_metric:
+        cmd.extend(['--scoring_metric', scoring_metric])
+    if model_save_path:
+        cmd.extend(['--model_path', model_save_path])
+    if results_save_path:
+        cmd.extend(['--results_path', results_save_path])
+    if visualize:
+        cmd.append('--visualize')
+
+    # Debugging: Print the command being executed
+    print(f"Executing command: {' '.join(cmd)}")
+
+    # Execute the command
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        output = result.stdout
+        errors = result.stderr
+        if result.returncode != 0:
+            return f"Error during training:\n{errors}", None
+        else:
+            # Delete usless "Figure (600x400)" text
+            output = re.sub(r"Figure\(\d+x\d+\)", "", output).strip()
+            # Try to load the plot image
+            if results_save_path:
+                plot_image_path = os.path.join(results_save_path, 'actual_vs_predicted.png')
+            else:
+                # Default path if results_save_path is not provided
+                plot_image_path = output.split('Visualization saved to ')[1].strip()
+            if os.path.exists(plot_image_path):
+                return f"Training completed successfully.\n\n{output}", plot_image_path
+            else:
+                return f"Training completed successfully.\n\n{output}", None
+    except Exception as e:
+        return f"An error occurred:\n{str(e)}", None
+
+def get_columns_from_data(data_option, data_file, data_path, data_name_kaggle, kaggle_json_file, competition_name):
+    # Determine data_path
+    if data_option == 'Upload Data File':
+        if data_file is None:
+            return []
+        data_path = data_file
+    elif data_option == 'Provide Data Path':
+        if not os.path.exists(data_path):
+            return []
+    elif data_option == 'Download from Kaggle':
+        if kaggle_json_file is None:
+            return []
+        else:
+            # Save the kaggle.json file to ~/.kaggle/kaggle.json
+            import shutil
+            kaggle_config_dir = os.path.expanduser('~/.kaggle')
+            os.makedirs(kaggle_config_dir, exist_ok=True)
+            kaggle_json_path = os.path.join(kaggle_config_dir, 'kaggle.json')
+            shutil.copy(kaggle_json_file.name, kaggle_json_path)
+            os.chmod(kaggle_json_path, 0o600)
+        data_dir = download_kaggle_data(json_path=kaggle_json_path, competition_name=competition_name)
+        if data_dir is None:
+            return []
+        data_path = os.path.join(data_dir, data_name_kaggle)
+        if not os.path.exists(data_path):
+            return []
+    else:
+        return []
+
+    try:
+        data = pd.read_csv(data_path)
+        columns = data.columns.tolist()
+        return columns
+    except Exception as e:
+        print(f"Error reading data file: {e}")
+        return []
+
+# Define Gradio interface components
+
+def update_columns(data_option, data_file, data_path, data_name_kaggle, kaggle_json_file, competition_name):
+    columns = get_columns_from_data(data_option, data_file, data_path, data_name_kaggle, kaggle_json_file, competition_name)
+    if not columns:
+        return gr.update(choices=[]), gr.update(choices=[])
+    else:
+        return gr.update(choices=columns), gr.update(choices=columns)
+
+model_modules = get_model_modules()
+
+if not model_modules:
+    print("No model modules found. Please check the 'models/supervised/regression' directory.")
+    # You can handle this case appropriately, e.g., show an error message in the interface or exit.
+
+with gr.Blocks() as demo:
+    gr.Markdown("# Train a Regression Model")
+
+    with gr.Row():
+        model_module_input = gr.Dropdown(choices=model_modules, label="Select Model Module")
+        scoring_metric_input = gr.Textbox(value='neg_root_mean_squared_error', label="Scoring Metric")
+
+    with gr.Row():
+        test_size_input = gr.Slider(minimum=0.1, maximum=0.5, step=0.05, value=0.2, label="Test Size")
+        random_state_input = gr.Number(value=42, label="Random State")
+        cv_folds_input = gr.Number(value=5, label="CV Folds", precision=0)
+
+    log_transform_input = gr.Checkbox(label="Log Transform Target Variable", value=False)
+    visualize_input = gr.Checkbox(label="Generate Visualizations", value=True)
+
+    with gr.Row():
+        model_save_path_input = gr.Textbox(value='', label="Model Save Path (optional)")
+        results_save_path_input = gr.Textbox(value='', label="Results Save Path (optional)")
+
+    with gr.Tab("Data Input"):
+        data_option_input = gr.Radio(choices=['Upload Data File', 'Provide Data Path', 'Download from Kaggle'], label="Data Input Option", value='Upload Data File')
+
+        upload_data_col = gr.Column(visible=True)
+        with upload_data_col:
+            data_file_input = gr.File(label="Upload CSV Data File", type="filepath")
+
+        data_path_col = gr.Column(visible=False)
+        with data_path_col:
+            data_path_input = gr.Textbox(value='', label="Data File Path")
+
+        kaggle_data_col = gr.Column(visible=False)
+        with kaggle_data_col:
+            kaggle_json_file_input = gr.File(label="Upload kaggle.json File", type="filepath")
+            competition_name_input = gr.Textbox(value='house-prices-advanced-regression-techniques', label="Kaggle Competition Name")
+            data_name_kaggle_input = gr.Textbox(value='train.csv', label="Data File Name (in Kaggle dataset)")
+
+    def toggle_data_input(option):
+        if option == 'Upload Data File':
+            return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
+        elif option == 'Provide Data Path':
+            return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+        elif option == 'Download from Kaggle':
+            return gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+
+    data_option_input.change(
+        fn=toggle_data_input,
+        inputs=[data_option_input],
+        outputs=[upload_data_col, data_path_col, kaggle_data_col]
+    )
+
+    update_cols_btn = gr.Button("Update Columns")
+
+    target_variable_input = gr.Dropdown(choices=[], label="Select Target Variable")
+    drop_columns_input = gr.CheckboxGroup(choices=[], label="Columns to Drop")
+
+    update_cols_btn.click(
+        fn=update_columns,
+        inputs=[data_option_input, data_file_input, data_path_input, data_name_kaggle_input, kaggle_json_file_input, competition_name_input],
+        outputs=[target_variable_input, drop_columns_input]
+    )
+
+    train_btn = gr.Button("Train Model")
+    output_display = gr.Textbox(label="Output")
+    image_display = gr.Image(label="Visualization", visible=True)
+
+    def run_training(*args):
+        output_text, plot_image_path = train_model(*args)
+        if plot_image_path and os.path.exists(plot_image_path):
+            return output_text, plot_image_path
+        else:
+            return output_text, None
+
+    train_btn.click(
+        fn=run_training,
+        inputs=[
+            model_module_input, data_option_input, data_file_input, data_path_input,
+            data_name_kaggle_input, kaggle_json_file_input, competition_name_input,
+            target_variable_input, drop_columns_input, test_size_input, random_state_input, log_transform_input, cv_folds_input,
+            scoring_metric_input, model_save_path_input, results_save_path_input, visualize_input
+        ],
+        outputs=[output_display, image_display]
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)

data/README.md

@@ -0,0 +1 @@
+# data

data/datasets/README.md

@@ -0,0 +1,21 @@
+# Datasets Utilities
+
+This folder contains utility scripts for handling datasets, including downloading data from Kaggle.
+
+## 📄 Scripts
+
+### `kaggle_data.py`
+
+- **Description**: A Python script to download Kaggle datasets or competition data seamlessly, supporting Google Colab, local Linux/Mac, and Windows environments.
+- **Path**: [`data/datasets/kaggle_data.py`](kaggle_data.py)
+- **Key Function**: `get_kaggle_data(json_path, data_name, is_competition=False, output_dir='data/raw')`
+- **Example**:
+
+  ```python
+  from kaggle_data import get_kaggle_data
+
+  # Download a standard Kaggle dataset
+  dataset_path = get_kaggle_data("kaggle.json", "paultimothymooney/chest-xray-pneumonia")
+
+  # Download competition data
+  competition_path = get_kaggle_data("kaggle.json", "house-prices-advanced-regression-techniques", is_competition=True)

data/datasets/kaggle_data.py

@@ -0,0 +1,115 @@
+"""
+This module provides a utility function to download Kaggle datasets or competition data.
+
+The function automatically detects whether it is running in a Google Colab environment, a local Linux/Mac environment, or a Windows environment, and sets up the Kaggle API accordingly.
+
+Requirements:
+    - Kaggle API installed (`pip install kaggle`)
+    - Kaggle API key (`kaggle.json`) with appropriate permissions.
+
+Environment Detection:
+    - Google Colab: Uses `/root/.config/kaggle/kaggle.json`.
+    - Local Linux/Mac: Uses `~/.kaggle/kaggle.json`.
+    - Windows: Uses `C:\\Users\\<Username>\\.kaggle\\kaggle.json`.
+
+Functions:
+    get_kaggle_data(json_path: str, data_name: str, is_competition: bool = False, output_dir: str = "data/raw") -> str
+"""
+
+import os
+import zipfile
+import sys
+import shutil
+import platform
+
+def get_kaggle_data(json_path: str, data_name: str, is_competition: bool = False, output_dir: str = "data/raw") -> str:
+    """
+    Downloads a Kaggle dataset or competition data using the Kaggle API in Google Colab, local Linux/Mac, or Windows environment.
+
+    Parameters:
+        json_path (str): Path to your 'kaggle.json' file.
+        data_name (str): Kaggle dataset or competition name (e.g., 'paultimothymooney/chest-xray-pneumonia' or 'house-prices-advanced-regression-techniques').
+        is_competition (bool): Set to True if downloading competition data. Default is False (for datasets).
+        output_dir (str): Directory to save and extract the data. Default is 'data'.
+
+    Returns:
+        str: Path to the extracted dataset folder.
+
+    Raises:
+        OSError: If 'kaggle.json' is not found or cannot be copied.
+        Exception: If there is an error during download or extraction.
+
+    Example of Usage:
+        # For downloading a standard dataset
+        dataset_path = get_kaggle_data("kaggle.json", "paultimothymooney/chest-xray-pneumonia")
+        print(f"Dataset is available at: {dataset_path}")
+
+        # For downloading competition data
+        competition_path = get_kaggle_data("kaggle.json", "house-prices-advanced-regression-techniques", is_competition=True)
+        print(f"Competition data is available at: {competition_path}")
+    """
+    # Detect environment (Colab, local Linux/Mac, or Windows)
+    is_colab = "google.colab" in sys.modules
+    is_windows = platform.system() == "Windows"
+
+    # Step 1: Setup Kaggle API credentials
+    try:
+        if is_colab:
+            config_dir = "/root/.config/kaggle"
+            os.makedirs(config_dir, exist_ok=True)
+            print("Setting up Kaggle API credentials for Colab environment.")
+            shutil.copy(json_path, os.path.join(config_dir, "kaggle.json"))
+            os.chmod(os.path.join(config_dir, "kaggle.json"), 0o600)
+        else:
+            # For both local Linux/Mac and Windows, use the home directory
+            config_dir = os.path.join(os.path.expanduser("~"), ".kaggle")
+            os.makedirs(config_dir, exist_ok=True)
+            print("Setting up Kaggle API credentials for local environment.")
+            kaggle_json_dest = os.path.join(config_dir, "kaggle.json")
+            if not os.path.exists(kaggle_json_dest):
+                shutil.copy(json_path, kaggle_json_dest)
+                if not is_windows:
+                    os.chmod(kaggle_json_dest, 0o600)
+    except Exception as e:
+        raise OSError(f"Could not set up Kaggle API credentials: {e}")
+
+    # Step 2: Create output directory
+    dataset_dir = os.path.join(output_dir, data_name.split('/')[-1])
+    os.makedirs(dataset_dir, exist_ok=True)
+    original_dir = os.getcwd()
+    os.chdir(dataset_dir)
+
+    # Step 3: Download the dataset or competition data
+    try:
+        if is_competition:
+            print(f"Downloading competition data: {data_name}")
+            cmd = f"kaggle competitions download -c {data_name}"
+        else:
+            print(f"Downloading dataset: {data_name}")
+            cmd = f"kaggle datasets download -d {data_name}"
+        os.system(cmd)
+    except Exception as e:
+        print(f"Error during download: {e}")
+        os.chdir(original_dir)
+        return None
+
+    # Step 4: Unzip all downloaded files
+    zip_files = [f for f in os.listdir() if f.endswith(".zip")]
+    if not zip_files:
+        print("No zip files found. Please check the dataset or competition name.")
+        os.chdir(original_dir)
+        return None
+
+    for zip_file in zip_files:
+        try:
+            with zipfile.ZipFile(zip_file, "r") as zip_ref:
+                zip_ref.extractall()
+            print(f"Extracted: {zip_file}")
+            os.remove(zip_file)
+        except Exception as e:
+            print(f"Error extracting {zip_file}: {e}")
+
+    # Step 5: Navigate back to the original directory
+    os.chdir(original_dir)
+
+    return dataset_dir

data/preprocessing/README.md

@@ -0,0 +1 @@
+# preprocessing

data/raw/README.md

@@ -0,0 +1 @@
+# raw

data/utils/README.md

@@ -0,0 +1 @@
+# utils

models/README.md

@@ -0,0 +1 @@
+# models

models/computer_vision/README.md

@@ -0,0 +1 @@
+# computer_vision

models/deep_learning/README.md

@@ -0,0 +1 @@
+# deep_learning

models/nlp/README.md

@@ -0,0 +1 @@
+# nlp

models/reinforcement_learning/README.md

@@ -0,0 +1 @@
+# reinforcement_learning

models/supervised/classification/README.md

@@ -0,0 +1 @@
+# classification

models/supervised/regression/README.md

@@ -0,0 +1 @@
+# regression

models/supervised/regression/adaboost_regressor.py

@@ -0,0 +1,30 @@
+
+"""
+This module sets up an AdaBoost Regressor with hyperparameter tuning.
+
+Features:
+- Uses `AdaBoostRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for boosting parameters.
+- Combines weak learners to form a strong predictor.
+
+Special Considerations:
+- Sensitive to outliers.
+- Not sensitive to feature scaling.
+- Base estimator is a Decision Tree by default.
+"""
+
+from sklearn.ensemble import AdaBoostRegressor
+
+# Define the estimator
+estimator = AdaBoostRegressor(random_state=42)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_estimators': [50, 100],  # Focus on a narrower range of estimators
+    'model__learning_rate': [0.001, 0.01, 0.1, 1.0],  # Keep a good spread for learning rates
+    'model__loss': ['linear'],  # Focus on the most commonly used loss function
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/catboost_regressor.py

@@ -0,0 +1,32 @@
+
+"""
+This module sets up a CatBoost Regressor with hyperparameter tuning.
+
+Features:
+- Uses `CatBoostRegressor` estimator from CatBoost.
+- Defines a hyperparameter grid for boosting parameters.
+- Handles categorical features natively.
+
+Special Considerations:
+- Requires the `catboost` library (`pip install catboost`).
+- Adjust the preprocessing pipeline to skip encoding categorical features.
+- Not sensitive to feature scaling.
+- Can be slower to train compared to other boosting algorithms.
+"""
+
+from catboost import CatBoostRegressor
+
+# Define the estimator
+estimator = CatBoostRegressor(random_state=42, verbose=0)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__iterations': [500],  # Fixed to a reasonable value for faster tuning
+    'model__learning_rate': [0.05, 0.1],  # Common learning rates
+    'model__depth': [6, 8],  # Typical depths for balance between speed and accuracy
+    'model__l2_leaf_reg': [3],  # Most impactful regularization value
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/decision_tree_regressor.py

@@ -0,0 +1,31 @@
+
+"""
+This module sets up a Decision Tree Regressor with hyperparameter tuning.
+
+Features:
+- Uses `DecisionTreeRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for tree-specific parameters.
+- Handles non-linear relationships and interactions.
+
+Special Considerations:
+- Decision Trees are not affected by feature scaling.
+- Can easily overfit; control tree depth and splitting criteria.
+- No need for scaling transformers in the preprocessing pipeline.
+"""
+
+from sklearn.tree import DecisionTreeRegressor
+
+# Define the estimator
+estimator = DecisionTreeRegressor(random_state=42)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__criterion': ['squared_error', 'absolute_error'],  # Only two key criteria
+    'model__max_depth': [5, 10, 20, None],  # Depth variations
+    'model__min_samples_split': [2, 10],  # Commonly used values
+    'model__min_samples_leaf': [1, 4],  # Few values for leaves
+    'preprocessor__num__imputer__strategy': ['mean'],  # Focused on a single strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/elasticnet_regression.py

@@ -0,0 +1,35 @@
+
+"""
+This module sets up an ElasticNet Regression model with hyperparameter tuning.
+
+Features:
+- Uses `ElasticNet` estimator from scikit-learn.
+- Combines L1 and L2 regularization.
+- Increases `max_iter` to address convergence warnings.
+
+Special Considerations:
+- May produce convergence warnings if `max_iter` is insufficient.
+- Adjust `l1_ratio` to balance between Lasso and Ridge penalties.
+- Applying a log transformation (`log_transform`) to the target variable can be beneficial if it's skewed.
+- Ensure `OneHotEncoder` outputs dense arrays.
+"""
+
+from sklearn.linear_model import ElasticNet
+
+# Define the estimator
+estimator = ElasticNet()
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__alpha': [0.01, 0.1, 1.0, 10.0],  # Regularization strength
+    'model__l1_ratio': [0.2, 0.5, 0.8],  # Balance between L1 (Lasso) and L2 (Ridge)
+    'model__max_iter': [5000],  # Sufficient to avoid convergence warnings
+    'model__fit_intercept': [True],  # Assume intercept is important
+    'model__selection': ['cyclic'],  # Focus on the default cyclic selection
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+    'preprocessor__num__scaler__with_mean': [True],  # StandardScaler
+    'preprocessor__num__scaler__with_std': [True],  # StandardScaler
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/extra_trees_regressor.py

@@ -0,0 +1,33 @@
+
+"""
+This module sets up an Extra Trees Regressor with hyperparameter tuning.
+
+Features:
+- Uses `ExtraTreesRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for ensemble parameters.
+- Similar to Random Forest but uses random thresholds for splitting.
+
+Special Considerations:
+- Not sensitive to feature scaling.
+- Can handle large datasets efficiently.
+- Less prone to overfitting compared to single decision trees.
+"""
+
+from sklearn.ensemble import ExtraTreesRegressor
+
+# Define the estimator
+estimator = ExtraTreesRegressor(random_state=42, n_jobs=-1)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_estimators': [100, 200],  # Common range for estimators
+    'model__criterion': ['squared_error'],  # Focus on the most widely used criterion
+    'model__max_depth': [None, 10, 20],  # Unrestricted depth and reasonable constraints
+    'model__min_samples_split': [2, 5],  # Commonly used values
+    'model__min_samples_leaf': [1, 2],  # Prevent overfitting with larger leaves
+    'model__max_features': ['sqrt', 'log2'],  # Reduce to most common feature sampling strategies
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/gradient_boosting_regressor.py

@@ -0,0 +1,33 @@
+
+"""
+This module sets up a Gradient Boosting Regressor with hyperparameter tuning.
+
+Features:
+- Uses `GradientBoostingRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for boosting parameters.
+- Builds sequential models to minimize errors.
+
+Special Considerations:
+- Sensitive to overfitting; tune `n_estimators` and `learning_rate`.
+- Not sensitive to feature scaling.
+- Longer training times compared to other models.
+"""
+
+from sklearn.ensemble import GradientBoostingRegressor
+
+# Define the estimator
+estimator = GradientBoostingRegressor(random_state=42)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_estimators': [100, 200],  # Focused range of estimators
+    'model__learning_rate': [0.001, 0.01, 0.1, 1],  # Commonly used learning rates
+    'model__max_depth': [3, 5],  # Standard depth values
+    'model__subsample': [0.8],  # Single value to focus on speed
+    'model__min_samples_split': [2],  # Default value
+    'model__min_samples_leaf': [1],  # Default value
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/knn_regressor.py

@@ -0,0 +1,34 @@
+
+"""
+This module sets up a K-Nearest Neighbors Regressor with hyperparameter tuning.
+
+Features:
+- Uses `KNeighborsRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for neighbor parameters.
+- Non-parametric method useful for capturing local patterns.
+
+Special Considerations:
+- Feature scaling is crucial for KNN.
+- Sensitive to the choice of `n_neighbors`.
+- Training is fast, but prediction can be slow on large datasets.
+"""
+
+from sklearn.neighbors import KNeighborsRegressor
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+# Define the estimator
+estimator = KNeighborsRegressor(n_jobs=-1)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_neighbors': [3, 5, 7],  # Focus on common neighbor values
+    'model__weights': ['uniform', 'distance'],  # Standard options
+    'model__algorithm': ['auto', 'ball_tree'],  # Reduce algorithms to commonly used ones
+    'model__p': [1, 2],  # Manhattan and Euclidean distances
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+    'preprocessor__num__scaler__with_mean': [True],  # StandardScaler
+    'preprocessor__num__scaler__with_std': [True],  # StandardScaler
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/lasso_regression.py

@@ -0,0 +1,33 @@
+
+"""
+This module sets up a Lasso Regression model with hyperparameter tuning.
+
+Features:
+- Uses `Lasso` estimator from scikit-learn.
+- Defines a hyperparameter grid for preprocessing and model-specific parameters.
+- Increases `max_iter` to address convergence warnings.
+
+Special Considerations:
+- Lasso Regression may produce convergence warnings if `max_iter` is insufficient.
+- Applying a log transformation (`log_transform`) to the target variable can be beneficial if it's skewed.
+- Ensure `OneHotEncoder` outputs dense arrays to avoid compatibility issues.
+"""
+
+from sklearn.linear_model import Lasso
+
+# Define the estimator
+estimator = Lasso()
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__alpha': [0.01, 0.1, 1.0, 10.0],  # Regularization strength
+    'model__max_iter': [5000],  # Single value to ensure convergence
+    'model__fit_intercept': [True],  # Assume the intercept is important
+    'model__selection': ['cyclic'],  # Focus on the default cyclic selection
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+    'preprocessor__num__scaler__with_mean': [True],  # StandardScaler
+    'preprocessor__num__scaler__with_std': [True],  # StandardScaler
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/lightgbm_regressor.py

@@ -0,0 +1,37 @@
+
+"""
+This module sets up a LightGBM Regressor with hyperparameter tuning.
+
+Features:
+- Uses `LGBMRegressor` estimator from LightGBM.
+- Defines a hyperparameter grid for boosting parameters.
+- Optimized for speed and performance.
+
+Special Considerations:
+- Requires the `lightgbm` library (`pip install lightgbm`).
+- Can handle categorical features if provided appropriately.
+- Not sensitive to feature scaling.
+"""
+
+from lightgbm import LGBMRegressor
+
+# Define the estimator
+estimator = LGBMRegressor(
+    random_state=42,
+    n_jobs=-1,
+    verbose=-1
+)
+
+# Define hyperparameter grid
+param_grid = {
+    'model__n_estimators': [100, 200],
+    'model__learning_rate': [0.01, 0.05],
+    'model__num_leaves': [15, 31],
+    'model__max_depth': [10, 20],
+    'model__min_data_in_leaf': [20, 50],
+    'model__colsample_bytree': [0.8],
+    'preprocessor__num__imputer__strategy': ['mean'],
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/linear_regression.py

@@ -0,0 +1,29 @@
+
+"""
+This module defines the setup for performing Linear Regression with hyperparameter tuning.
+
+Features:
+- Sets up a `LinearRegression` estimator from scikit-learn.
+- Defines a hyperparameter grid for preprocessing and model-specific parameters.
+- Specifies an optional default scoring metric for evaluating the model.
+
+Special Considerations:
+- Linear Regression doesn't typically require special handling.
+- Applying a log transformation to the target variable (`log_transform`) can be beneficial if it's skewed.
+"""
+
+from sklearn.linear_model import LinearRegression
+
+# Define the estimator
+estimator = LinearRegression()
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__fit_intercept': [True, False],
+    'preprocessor__num__imputer__strategy': ['mean', 'median'],
+    'preprocessor__num__scaler__with_mean': [True, False],
+    'preprocessor__num__scaler__with_std': [True, False],
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/mlp_regressor.py

@@ -0,0 +1,35 @@
+
+"""
+This module sets up a Multilayer Perceptron Regressor with hyperparameter tuning.
+
+Features:
+- Uses `MLPRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for neural network parameters.
+- Capable of capturing complex non-linear relationships.
+
+Special Considerations:
+- Feature scaling is crucial for MLP.
+- May produce convergence warnings; increase `max_iter` to address this.
+- Can be sensitive to hyperparameter settings; tuning is important.
+"""
+
+from sklearn.neural_network import MLPRegressor
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+# Define the estimator
+estimator = MLPRegressor(random_state=42, max_iter=1000)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__hidden_layer_sizes': [(50,), (100,), (50, 50)],  # Simplified layer sizes
+    'model__activation': ['relu'],  # Focused on ReLU, the most commonly effective activation
+    'model__solver': ['adam'],  # Retain 'adam' for efficiency; drop 'lbfgs' (slower for larger datasets)
+    'model__alpha': [0.0001, 0.001],  # Regularization strengths
+    'model__learning_rate': ['constant', 'adaptive'],  # Common learning rate strategies
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+    'preprocessor__num__scaler__with_mean': [True],  # StandardScaler
+    'preprocessor__num__scaler__with_std': [True],  # StandardScaler
+}
+
+# Optional: Default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/random_forest_regressor.py

@@ -0,0 +1,32 @@
+
+"""
+This module sets up a Random Forest Regressor with hyperparameter tuning.
+
+Features:
+- Uses `RandomForestRegressor` estimator from scikit-learn.
+- Defines a hyperparameter grid for ensemble parameters.
+- Handles non-linear relationships and reduces overfitting through averaging.
+
+Special Considerations:
+- Random Forests are robust to outliers and can handle non-linear data.
+- Not sensitive to feature scaling.
+- Set `n_jobs=-1` to utilize all available CPU cores.
+"""
+
+from sklearn.ensemble import RandomForestRegressor
+
+# Define the estimator
+estimator = RandomForestRegressor(random_state=42, n_jobs=-1)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_estimators': [100, 200],  # Focus on a small range of estimators
+    'model__max_depth': [10, 20, None],  # Commonly used depth variations
+    'model__min_samples_split': [2, 5],  # Commonly used split values
+    'model__min_samples_leaf': [1, 2],  # Focused leaf size
+    'model__max_features': ['sqrt'],  # "sqrt" is often optimal for Random Forests
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/ridge_regression.py

@@ -0,0 +1,33 @@
+
+"""
+This module sets up a Ridge Regression model with hyperparameter tuning.
+
+Features:
+- Uses `Ridge` estimator from scikit-learn.
+- Defines a hyperparameter grid for preprocessing and model-specific parameters.
+- Addresses potential convergence warnings by increasing `max_iter`.
+- Considers solvers compatible with dense data after modifying `OneHotEncoder`.
+
+Special Considerations:
+- Ridge Regression may produce convergence warnings if `max_iter` is insufficient.
+- Applying a log transformation (`log_transform`) to the target variable can be beneficial if it's skewed.
+- Ensure `OneHotEncoder` outputs dense arrays to avoid solver compatibility issues.
+"""
+
+from sklearn.linear_model import Ridge
+
+# Define the estimator
+estimator = Ridge()
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__alpha': [0.1, 1.0, 10.0],
+    'model__solver': ['auto', 'svd', 'cholesky'],
+    'model__max_iter': [1000, 5000],
+    'preprocessor__num__imputer__strategy': ['mean', 'median'],
+    'preprocessor__num__scaler__with_mean': [True, False],
+    'preprocessor__num__scaler__with_std': [True, False],
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/support_vector_regressor.py

@@ -0,0 +1,34 @@
+
+"""
+This module sets up a Support Vector Regressor (SVR) with hyperparameter tuning.
+
+Features:
+- Uses `SVR` estimator from scikit-learn.
+- Defines a hyperparameter grid for kernel parameters.
+- Effective in high-dimensional spaces.
+
+Special Considerations:
+- Feature scaling is crucial for SVR.
+- Training time can be significant for large datasets.
+- Applying a log transformation (`log_transform`) can be beneficial if the target variable is skewed.
+"""
+
+from sklearn.svm import SVR
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+
+# Define the estimator
+estimator = SVR()
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__kernel': ['rbf'],  # Stick to the most effective kernel
+    'model__C': [0.1, 1.0, 10.0],  # Focus on a narrower range
+    'model__epsilon': [0.1, 0.2, 0.5],  # Retain small deviations
+    'model__gamma': ['scale', 0.1],  # Simplify gamma
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+    'preprocessor__num__scaler__with_mean': [True],
+    'preprocessor__num__scaler__with_std': [True],
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/supervised/regression/xgboost_regressor.py

@@ -0,0 +1,35 @@
+
+"""
+This module sets up an XGBoost Regressor with hyperparameter tuning.
+
+Features:
+- Uses `XGBRegressor` estimator from XGBoost.
+- Defines a hyperparameter grid for boosting parameters.
+- Efficient and scalable implementation of gradient boosting.
+
+Special Considerations:
+- Requires the `xgboost` library (`pip install xgboost`).
+- Handles missing values internally.
+- Not sensitive to feature scaling.
+- May require setting `tree_method` to 'gpu_hist' for GPU acceleration if available.
+"""
+
+from xgboost import XGBRegressor
+
+# Define the estimator
+estimator = XGBRegressor(random_state=42, n_jobs=-1, verbosity=0)
+
+# Define the hyperparameter grid
+param_grid = {
+    'model__n_estimators': [100, 200],  # Common range for estimators
+    'model__learning_rate': [0.05, 0.1],  # Common learning rates
+    'model__max_depth': [3, 5],  # Typical depth for gradient boosting
+    'model__subsample': [0.8],  # Fixed subsample value to reduce complexity
+    'model__colsample_bytree': [0.8],  # Fixed colsample value to reduce complexity
+    'model__reg_alpha': [0, 0.1],  # Focus on smaller values for L1 regularization
+    'model__reg_lambda': [1],  # Default L2 regularization
+    'preprocessor__num__imputer__strategy': ['mean'],  # Single imputation strategy
+}
+
+# Optional: Define the default scoring metric
+default_scoring = 'neg_root_mean_squared_error'

models/unsupervised/README.md

@@ -0,0 +1 @@
+# unsupervised

requirements.txt

@@ -0,0 +1,12 @@
+pandas==2.2.2
+numpy==1.26.4
+matplotlib==3.8.0
+seaborn==0.13.2
+kaggle==1.6.17
+scikit-learn==1.5.2
+catboost==1.2.7
+dask[dataframe]==2024.10.0
+xgboost==2.1.2
+lightgbm==4.5.0
+joblib==1.4.2
+gradio==5.7.1

scripts/README.md

@@ -0,0 +1,52 @@
+# Scripts
+
+This directory contains executable scripts for training, testing, and other tasks related to model development and evaluation.
+
+## Contents
+
+- [`train_regression_model.py`](#train_regression_model.py)
+
+### `train_regression_model.py`
+
+A script for training supervised learning regression models using scikit-learn. It handles data loading, preprocessing, optional log transformation, hyperparameter tuning, model evaluation, and saving of models, metrics, and visualizations.
+
+#### Features
+
+- Supports various regression models defined in the `models/supervised/regression` directory.
+- Performs hyperparameter tuning using grid search cross-validation.
+- Saves trained models and evaluation metrics.
+- Generates visualizations if specified.
+
+#### Usage
+
+```bash
+python train_regression_model.py --model_module MODEL_MODULE \
+    --data_path DATA_PATH/DATA_NAME.csv \
+    --target_variable TARGET_VARIABLE [OPTIONS]
+
+```
+
+- **Required Arguments:**
+- `model_module`: Name of the model module to import (e.g., `linear_regression`).
+- `data_path`: Path to the dataset directory, including the data file name.
+- `target_variable`: Name of the target variable.
+
+- **Optional Arguments:**
+- `test_size`: Proportion of the dataset to include in the test split (default: 0.2).
+- `random_state`: Random seed for reproducibility (default: 42).
+- `log_transform`: Apply log transformation to the target variable (regression only).
+- `cv_folds`: Number of cross-validation folds (default: 5).
+- `scoring_metric`: Scoring metric for model evaluation.
+- `model_path`: Path to save the trained model.
+- `results_path`: Path to save results and metrics.
+- `visualize`: Generate and save visualizations.
+- `drop_columns`: Comma-separated column names to drop from the dataset.
+
+#### Usage Example
+
+```bash
+python train_regression_model.py --model_module linear_regression \
+    --data_path data/house_prices/train.csv \
+    --target_variable SalePrice --drop_columns Id \
+    --log_transform --visualize
+```

scripts/train_classification_model.py

@@ -0,0 +1,185 @@
+"""
+This script trains classification models using scikit-learn.
+It includes data loading, preprocessing, encoding of target variable,
+hyperparameter tuning, model evaluation, and saving of models, metrics,
+and visualizations.
+
+Usage:
+    python train_classification_model.py --model_module MODEL_MODULE --data_path DATA_PATH/DATA_NAME.csv
+                                         --target_variable TARGET_VARIABLE
+
+Optional arguments:
+    --test_size TEST_SIZE
+    --random_state RANDOM_STATE
+    --cv_folds CV_FOLDS
+    --scoring_metric SCORING_METRIC
+    --model_path MODEL_PATH
+    --results_path RESULTS_PATH
+    --visualize
+    --drop_columns COLUMN_NAMES
+
+Example:
+    python train_classification_model.py --model_module logistic_regression
+                                         --data_path data/titanic/train.csv
+                                         --target_variable Survived --drop_columns PassengerId
+                                         --visualize
+"""
+
+import os
+import sys
+import argparse
+import importlib
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import (accuracy_score, precision_score, recall_score, f1_score,
+                             confusion_matrix, ConfusionMatrixDisplay)
+import joblib
+
+def main(args):
+    # Change to the root directory of the project
+    project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    os.chdir(project_root)
+    sys.path.insert(0, project_root)
+
+    # Import the hyperparameter tuning and the model modules
+    from utils.supervised_hyperparameter_tuning import classification_hyperparameter_tuning
+    model_module_path = f"models.supervised.classification.{args.model_module}"
+    model_module = importlib.import_module(model_module_path)
+    
+    # Get the model estimator, parameters grid, and the scoring metric
+    estimator = model_module.estimator
+    param_grid = model_module.param_grid
+    scoring_metric = args.scoring_metric or getattr(model_module, 'default_scoring', 'accuracy')
+    model_name = estimator.__class__.__name__
+
+    # Set default paths if not provided
+    args.model_path = args.model_path or os.path.join('saved_models', model_name)
+    args.results_path = args.results_path or os.path.join('results', model_name)
+    os.makedirs(args.results_path, exist_ok=True)
+
+    # Load the dataset
+    df = pd.read_csv(os.path.join(args.data_path))
+
+    # Drop specified columns
+    if args.drop_columns:
+        columns_to_drop = args.drop_columns.split(',')
+        df = df.drop(columns=columns_to_drop)
+
+    # Define target variable and features
+    target_variable = args.target_variable
+    X = df.drop(columns=[target_variable])
+    y = df[target_variable]
+
+    # Ensure target variable is categorical
+    if np.issubdtype(y.dtype, np.number) and len(np.unique(y)) > 20:
+        raise ValueError(f"The target variable '{target_variable}' seems to be continuous. Please ensure it's categorical for classification tasks.")
+
+    # Encode target variable if not numeric
+    if y.dtype == 'object' or not np.issubdtype(y.dtype, np.number):
+        from sklearn.preprocessing import LabelEncoder
+        le = LabelEncoder()
+        y = le.fit_transform(y)
+        # Save label encoder for inverse transformation
+        joblib.dump(le, os.path.join(args.model_path, 'label_encoder.pkl'))
+
+    # Split the data
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=args.test_size, random_state=args.random_state, stratify=y)
+
+    # Perform hyperparameter tuning
+    best_model, best_params = classification_hyperparameter_tuning(
+        X_train, y_train, estimator, param_grid,
+        cv=args.cv_folds, scoring=scoring_metric)
+
+    # Evaluate the best model on the test set
+    y_pred = best_model.predict(X_test)
+    y_test_actual = y_test
+
+    # Save the trained model
+    model_output_path = os.path.join(args.model_path, 'best_model.pkl')
+    os.makedirs(args.model_path, exist_ok=True)
+    joblib.dump(best_model, model_output_path)
+    print(f"Trained model saved to {model_output_path}")
+
+    # Calculate metrics
+    accuracy = accuracy_score(y_test_actual, y_pred)
+    precision = precision_score(y_test_actual, y_pred, average='weighted', zero_division=0)
+    recall = recall_score(y_test_actual, y_pred, average='weighted', zero_division=0)
+    f1 = f1_score(y_test_actual, y_pred, average='weighted', zero_division=0)
+    print(f"\n{model_name} Classification Metrics on Test Set:")
+    print(f"- Accuracy: {accuracy:.4f}")
+    print(f"- Precision: {precision:.4f}")
+    print(f"- Recall: {recall:.4f}")
+    print(f"- F1 Score: {f1:.4f}")
+    # Save metrics
+    metrics = {'Accuracy': [accuracy], 'Precision': [precision], 'Recall': [recall], 'F1 Score': [f1]}
+
+    # Save metrics to CSV
+    results_df = pd.DataFrame(metrics)
+    results_df.to_csv(os.path.join(args.results_path, 'metrics.csv'), index=False)
+    print(f"\nMetrics saved to {os.path.join(args.results_path, 'metrics.csv')}")
+
+    if args.visualize:
+        # Plot Classification Metrics
+        plt.figure(figsize=(8, 6))
+        # Extract metrics and values
+        metric_names = list(metrics.keys())
+        metric_values = [value[0] for value in metrics.values()]  # Extract the single value from each list
+
+        # Create bar chart
+        plt.bar(metric_names, metric_values, color='skyblue', alpha=0.8)
+        plt.ylim(0, 1)  # Metrics like accuracy, precision, etc., are between 0 and 1
+        plt.xlabel('Metrics')
+        plt.ylabel('Scores')
+        plt.title('Classification Metrics')
+
+        # Save and display the plot
+        plt.savefig(os.path.join(args.results_path, 'classification_metrics.png'))
+        plt.show()
+        print(f"Visualization saved to {os.path.join(args.results_path, 'classification_metrics.png')}")
+
+        # Display and save the confusion matrix
+        conf_matrix = confusion_matrix(y_test_actual, y_pred)
+        disp = ConfusionMatrixDisplay(confusion_matrix=conf_matrix)
+        disp.plot(cmap=plt.cm.Blues, values_format='d')  # Format as integers for counts
+        plt.title(f'{model_name} Confusion Matrix')
+
+        # Save the confusion matrix plot
+        conf_matrix_path = os.path.join(args.results_path, 'confusion_matrix.png')
+        plt.savefig(conf_matrix_path)
+        plt.show()
+        print(f"Confusion matrix saved to {conf_matrix_path}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Train a classification model.")
+    # Model module argument
+    parser.add_argument('--model_module', type=str, required=True,
+                        help='Name of the classification model module to import.')
+    # Data arguments
+    parser.add_argument('--data_path', type=str, required=True,
+                        help='Path to the dataset file including data name.')
+    parser.add_argument('--target_variable', type=str, required=True,
+                        help='Name of the target variable.')
+    parser.add_argument('--drop_columns', type=str, default='',
+                        help='Columns to drop from the dataset.')
+    # Model arguments
+    parser.add_argument('--test_size', type=float, default=0.2,
+                        help='Proportion for test split.')
+    parser.add_argument('--random_state', type=int, default=42,
+                        help='Random seed.')
+    parser.add_argument('--cv_folds', type=int, default=5,
+                        help='Number of cross-validation folds.')
+    parser.add_argument('--scoring_metric', type=str, default=None,
+                        help='Scoring metric for model evaluation.')
+    # Output arguments
+    parser.add_argument('--model_path', type=str, default=None,
+                        help='Path to save the trained model.')
+    parser.add_argument('--results_path', type=str, default=None,
+                        help='Path to save results and metrics.')
+    parser.add_argument('--visualize', action='store_true',
+                        help='Generate and save visualizations.')
+
+    args = parser.parse_args()
+    main(args)

scripts/train_regression_model.py

@@ -0,0 +1,200 @@
+"""
+This script trains regression models using scikit-learn.
+It includes data loading, preprocessing, optional log transformation,
+hyperparameter tuning, model evaluation, and saving of models, metrics,
+and visualizations.
+
+Usage:
+    python train_regression_model.py --model_module MODEL_MODULE --data_path DATA_PATH/DATA_NAME.csv
+                                     --target_variable TARGET_VARIABLE
+
+Optional arguments:
+    --test_size TEST_SIZE
+    --random_state RANDOM_STATE
+    --log_transform
+    --cv_folds CV_FOLDS
+    --scoring_metric SCORING_METRIC
+    --model_path MODEL_PATH
+    --results_path RESULTS_PATH
+    --visualize
+    --drop_columns COLUMN_NAMES
+
+Example:
+    python train_regression_model.py --model_module linear_regression
+                                     --data_path data/house_prices/train.csv
+                                     --target_variable SalePrice --drop_columns Id
+                                     --log_transform --visualize
+"""
+
+import os
+import sys
+import argparse
+import importlib
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import root_mean_squared_error, mean_squared_error, r2_score, mean_absolute_error
+import joblib
+from timeit import default_timer as timer
+
+def main(args):
+    # Change to the root directory of the project
+    project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    os.chdir(project_root)
+    sys.path.insert(0, project_root)
+
+    # Import the hyperparameter tuning and the model modules
+    from utils.supervised_hyperparameter_tuning import regression_hyperparameter_tuning
+    model_module_path = f"models.supervised.regression.{args.model_module}"
+    model_module = importlib.import_module(model_module_path)
+    
+    # Get the model estimator, parameters grid, and the scoring metric
+    estimator = model_module.estimator
+    param_grid = model_module.param_grid
+    scoring_metric = args.scoring_metric or getattr(model_module, 'default_scoring', 'neg_root_mean_squared_error')
+    model_name = estimator.__class__.__name__
+
+    # Set default paths if not provided
+    args.model_path = args.model_path or os.path.join('saved_models', model_name)
+    args.results_path = args.results_path or os.path.join('results', model_name)
+    os.makedirs(args.results_path, exist_ok=True)
+
+    # Load the dataset
+    df = pd.read_csv(os.path.join(args.data_path))
+
+    # Drop specified columns
+    if args.drop_columns:
+        columns_to_drop = args.drop_columns.split(',')
+        df = df.drop(columns=columns_to_drop)
+
+    # Define target variable and features
+    target_variable = args.target_variable
+    X = df.drop(columns=[target_variable])
+    y = df[target_variable]
+
+    # Ensure target variable is numeric
+    if not np.issubdtype(y.dtype, np.number):
+        raise ValueError(f"The target variable '{target_variable}' must be numeric for regression tasks.")
+
+    # Split the data
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y, test_size=args.test_size, random_state=args.random_state)
+
+    # Visualize target variable distribution
+    if args.visualize:
+        plt.figure(figsize=(6, 4))
+        sns.histplot(y_train, kde=True)
+        plt.title(f'{target_variable} Distribution Before Transformation')
+        plt.savefig(os.path.join(args.results_path, 'target_distribution_before.png'))
+        plt.show()
+
+    # Optional: Apply log transformation
+    if args.log_transform:
+        y_train_transformed = np.log1p(y_train)
+        y_test_transformed = np.log1p(y_test)
+        if args.visualize:
+            plt.figure(figsize=(6, 4))
+            sns.histplot(y_train_transformed, kde=True, color='green')
+            plt.title(f'{target_variable} Distribution After Log Transform')
+            plt.savefig(os.path.join(args.results_path, 'target_distribution_after.png'))
+            plt.show()
+    else:
+        y_train_transformed = y_train
+        y_test_transformed = y_test
+
+    # Start the timer
+    start_time = timer()
+
+    # Perform hyperparameter tuning
+    best_model, best_params = regression_hyperparameter_tuning(
+        X_train, y_train_transformed, estimator, param_grid,
+        cv=args.cv_folds, scoring=scoring_metric)
+
+    # End the timer and calculate how long it took
+    end_time = timer()
+    train_time = end_time-start_time
+
+    # Evaluate the best model on the test set
+    y_pred_transformed = best_model.predict(X_test)
+
+    # Reverse transformation if applied
+    if args.log_transform:
+        y_pred = np.expm1(y_pred_transformed)
+        y_test_actual = np.expm1(y_test_transformed)
+    else:
+        y_pred = y_pred_transformed
+        y_test_actual = y_test_transformed
+
+    # Save the trained model
+    model_output_path = os.path.join(args.model_path, 'best_model.pkl')
+    os.makedirs(args.model_path, exist_ok=True)
+    joblib.dump(best_model, model_output_path)
+    print(f"Trained model saved to {model_output_path}")
+
+    # Calculate metrics
+    rmse = root_mean_squared_error(y_test_actual, y_pred)
+    r2 = r2_score(y_test_actual, y_pred)
+    mae = mean_absolute_error(y_test_actual, y_pred)
+    mse = mean_squared_error(y_test_actual, y_pred)
+    print(f"\n{model_name} Regression Metrics on Test Set:")
+    print(f"- RMSE: {rmse:.4f}")
+    print(f"- R² Score: {r2:.4f}")
+    print(f"- MAE: {mae:.4f}")
+    print(f"- MSE: {mse:.4f}")
+    print(f"- Training time: {train_time:.4f} seconds")
+    # Save metrics
+    metrics = {'RMSE': [rmse], 'R2': [r2], 'MAE': [mae], 'MSE': [mse], 'train_time': [train_time]}
+
+    # Save metrics to CSV
+    results_df = pd.DataFrame(metrics)
+    results_df.to_csv(os.path.join(args.results_path, 'metrics.csv'), index=False)
+    print(f"\nMetrics saved to {os.path.join(args.results_path, 'metrics.csv')}")
+
+    if args.visualize:
+        # Plot Actual vs. Predicted
+        plt.figure(figsize=(8, 6))
+        plt.scatter(y_test_actual, y_pred, alpha=0.6, color='blue')
+        plt.plot([y_test_actual.min(), y_test_actual.max()],
+                [y_test_actual.min(), y_test_actual.max()], 'r--')
+        plt.xlabel(f'Actual {target_variable}')
+        plt.ylabel(f'Predicted {target_variable}')
+        plt.title(f'Actual vs. Predicted {target_variable}')
+        plt.savefig(os.path.join(args.results_path, 'actual_vs_predicted.png'))
+        plt.show()
+        print(f"Visualization saved to {os.path.join(args.results_path, 'actual_vs_predicted.png')}")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Train a regression model.")
+    # Model module argument
+    parser.add_argument('--model_module', type=str, required=True,
+                        help='Name of the regression model module to import.')
+    # Data arguments
+    parser.add_argument('--data_path', type=str, required=True,
+                        help='Path to the dataset file including data name.')
+    parser.add_argument('--target_variable', type=str, required=True,
+                        help='Name of the target variable.')
+    parser.add_argument('--drop_columns', type=str, default='',
+                        help='Columns to drop from the dataset.')
+    # Model arguments
+    parser.add_argument('--test_size', type=float, default=0.2,
+                        help='Proportion for test split.')
+    parser.add_argument('--random_state', type=int, default=42,
+                        help='Random seed.')
+    parser.add_argument('--log_transform', action='store_true',
+                        help='Apply log transformation to the target variable.')
+    parser.add_argument('--cv_folds', type=int, default=5,
+                        help='Number of cross-validation folds.')
+    parser.add_argument('--scoring_metric', type=str, default=None,
+                        help='Scoring metric for model evaluation.')
+    # Output arguments
+    parser.add_argument('--model_path', type=str, default=None,
+                        help='Path to save the trained model.')
+    parser.add_argument('--results_path', type=str, default=None,
+                        help='Path to save results and metrics.')
+    parser.add_argument('--visualize', action='store_true',
+                        help='Generate and save visualizations.')
+
+    args = parser.parse_args()
+    main(args)

utils/README.md

@@ -0,0 +1,45 @@
+# Utils
+
+This directory contains utility scripts and helper functions that are used throughout the project. These scripts provide common functionalities such as data preprocessing, hyperparameter tuning, and other support functions that assist in model training and evaluation.
+
+## Contents
+
+- [`supervised_hyperparameter_tuning.py`](#supervised_hyperparameter_tuning.py)
+
+### `supervised_hyperparameter_tuning.py`
+
+This script contains functions for performing hyperparameter tuning on supervised learning models using scikit-learn's `Pipeline` and `GridSearchCV`.
+
+#### Functions
+
+- **`regression_hyperparameter_tuning(X_train, y_train, estimator, param_grid, cv=5, scoring=None)`**
+
+  Performs hyperparameter tuning using grid search cross-validation.
+
+  - **Parameters:**
+    - `X_train`: Training features.
+    - `y_train`: Training target variable.
+    - `estimator`: A scikit-learn estimator (e.g., `LinearRegression()`).
+    - `param_grid`: Dictionary with parameters names (`str`) as keys and lists of parameter settings to try as values.
+    - `cv`: Number of cross-validation folds. Default is 5.
+    - `scoring`: Scoring metric to use. Default depends on the estimator.
+
+  - **Returns:**
+    - `best_model`: The estimator with the best found parameters.
+    - `best_params`: Dictionary of the best parameters.
+
+#### Usage Example
+
+```python
+from utils.supervised_hyperparameter_tuning import regression_hyperparameter_tuning
+from sklearn.linear_model import LinearRegression
+
+# Define estimator and parameter grid
+estimator = LinearRegression()
+param_grid = {
+    'model__fit_intercept': [True, False],
+    # Add other parameters
+}
+
+# Perform hyperparameter tuning
+best_model, best_params = regression_hyperparameter_tuning(X_train, y_train, estimator, param_grid)

utils/supervised_hyperparameter_tuning.py

@@ -0,0 +1,115 @@
+"""
+This module provides a function for hyperparameter tuning with preprocessing using scikit-learn's GridSearchCV specifically for regression models.
+
+Features:
+- Handles numerical and categorical preprocessing using pipelines.
+- Automates hyperparameter tuning for any scikit-learn regressor.
+- Uses GridSearchCV for cross-validation and hyperparameter search.
+- Applies algorithm-specific preprocessing when necessary.
+
+Functions:
+    - hyperparameter_tuning_model: Performs hyperparameter tuning on a given dataset and estimator.
+
+Example Usage:
+    from sklearn.ensemble import RandomForestRegressor
+    from supervised_hyperparameter_tuning import hyperparameter_tuning_model
+
+    X = ...  # Your feature DataFrame
+    y = ...  # Your target variable
+    param_grid = {
+        'model__n_estimators': [100, 200, 500],
+        'model__max_depth': [None, 10, 20]
+    }
+    best_model, best_params = hyperparameter_tuning_model(X, y, RandomForestRegressor(), param_grid, cv=5, scoring='neg_mean_squared_error')
+"""
+
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler
+from sklearn.model_selection import GridSearchCV, KFold
+
+def regression_hyperparameter_tuning(X, y, estimator, param_grid, cv=5, scoring=None):
+    """
+    Performs hyperparameter tuning for a given regression model using GridSearchCV with preprocessing.
+
+    Args:
+        X (pd.DataFrame): Features.
+        y (pd.Series): Target variable.
+        estimator: The scikit-learn regressor to use (e.g., LinearRegression(), RandomForestRegressor()).
+        param_grid (dict): Hyperparameter grid for GridSearchCV.
+        cv (int or cross-validation generator): Number of cross-validation folds or a cross-validation generator.
+        scoring (str or None): Scoring metric to use.
+
+    Returns:
+        best_model (Pipeline): Best model within a pipeline from GridSearch.
+        best_params (dict): Best hyperparameters.
+    """
+    # Identify numerical and categorical columns
+    numerical_cols = X.select_dtypes(include=['int64', 'float64']).columns.tolist()
+    categorical_cols = X.select_dtypes(include=['object', 'category']).columns.tolist()
+
+    # Define preprocessing for numerical data
+    numerical_transformer = Pipeline(steps=[
+        ('imputer', SimpleImputer(strategy='median')),
+        ('scaler', StandardScaler())
+    ])
+
+    # Conditional preprocessing for categorical data
+    estimator_name = estimator.__class__.__name__
+
+    if estimator_name in [
+        'DecisionTreeRegressor', 'RandomForestRegressor', 'ExtraTreesRegressor',
+        'GradientBoostingRegressor', 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor'
+    ]:
+        # Use Ordinal Encoding for tree-based models
+        categorical_transformer = Pipeline(steps=[
+            ('imputer', SimpleImputer(strategy='constant', fill_value='Missing')),
+            ('ordinal_encoder', OrdinalEncoder(handle_unknown='use_encoded_value', unknown_value=-1))
+        ])
+    else:
+        # Use OneHotEncoder for other models
+        categorical_transformer = Pipeline(steps=[
+            ('imputer', SimpleImputer(strategy='constant', fill_value='Missing')),
+            ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))
+        ])
+
+    # Create preprocessing pipeline
+    preprocessor = ColumnTransformer(
+        transformers=[
+            ('num', numerical_transformer, numerical_cols),
+            ('cat', categorical_transformer, categorical_cols)
+        ]
+    )
+
+    # Create a pipeline that combines preprocessing and the estimator
+    pipeline = Pipeline(steps=[
+        ('preprocessor', preprocessor),
+        ('model', estimator)
+    ])
+
+    # Define cross-validation strategy
+    if isinstance(cv, int):
+        cv = KFold(n_splits=cv, shuffle=True, random_state=42)
+
+    # Initialize GridSearchCV
+    grid_search = GridSearchCV(
+        estimator=pipeline,
+        param_grid=param_grid,
+        cv=cv,
+        scoring=scoring,
+        n_jobs=-1
+    )
+
+    # Perform Grid Search
+    grid_search.fit(X, y)
+
+    # Get the best model and parameters
+    best_model = grid_search.best_estimator_
+    best_params = grid_search.best_params_
+
+    print(f"Best Hyperparameters for {estimator_name}:")
+    for param_name in sorted(best_params.keys()):
+        print(f"{param_name}: {best_params[param_name]}")
+
+    return best_model, best_params