{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **About**\nThis notebook documents my approach and learnings from participating in a Kaggle competition focused on predicting insurance premium amounts. The evaluation metric for this competition is **Root Mean Squared Logrithmic Error (RMSLE).** The dataset is split into **three parts: train, test, and an original dataset:**\n* **Train:** **1.2 million** records\n* **Test:** **800,000** records\n* **Original:** **278,860** records\n\n### **Goal**\nThe objective is to accurately predict the **`premium_amount`** for each policyholder, optimizing the model for **RMSLE**.\n\n### **Challenges**\n* **Missing Data:** Several features contain missing values, ranging from **2% to over 30%.**\n* **Outliers:** Columns like **`annual_income`** and **`health_score`** contain **extreme values**. These values were retained as they likely reflect real-world variability across individuals.\n* **Weak Correlation with Target Variable:** The dataset lacks strong linear relationships with the target variable. For instance, the highest observed correlation was just **0.047** between **`previous_claim`** and **`premium_amount`**, indicating minimal direct predictive power.\n\n### **Approach**\nTo address these challenges and enhance model performance:\n\n#### **Exploratory Data Analysis (EDA):** \n* Conducted in-depth analysis to understand feature distributions, identify outliers, and assess feature-target relationships.\n\n#### **Feature Engineering:**\n\n✅ Created interaction and ratio-based features to uncover deeper relationships between variables.\n\n✅ Applied smoothed target encoding to categorical features and group-wise categorical combinations to enhance signal strength.\n\n✅ For high-cardinality numerical features, performed quantile binning followed by target-based statistical aggregations.\n\n#### **Handling Missing Values:** \n* Applied **median** imputation for numerical features and **mode** imputation for categorical features using **SimpleImputer** within each fold via **ColumnTransformer**, ensuring no data leakage during cross-validation.\n\n#### **Cross-Validation Strategy:**\n* Employed **5-fold cross-validation.**\n* All feature engineering steps including missing value imputation, target encoding, and aggregation were performed within each fold to prevent data leakage.\n\n### **Model**\n* Implemented and trained a **LightGBM model** using the engineered feature set.\n* Manually selected model hyperparameters through iterative experimentation and validation.","metadata":{}},{"cell_type":"code","source":"# Import Library    \nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt \n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer \nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error \nfrom xgboost import XGBRegressor  \nfrom catboost import CatBoostRegressor\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor, early_stopping\nfrom sklearn.model_selection import train_test_split, cross_val_score, KFold \nfrom sklearn.metrics import mean_squared_log_error, mean_squared_error, make_scorer\nfrom category_encoders import TargetEncoder\nimport optuna\n\n# Display numbers in standard notation\npd.set_option('display.float_format', '{:.2f}'.format) \n\n# Display all columns\npd.set_option('display.max_columns', None)\n\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.simplefilter(action='ignore', category=FutureWarning)  \nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:00.701314Z","iopub.execute_input":"2025-09-09T05:28:00.70226Z","iopub.status.idle":"2025-09-09T05:28:05.177575Z","shell.execute_reply.started":"2025-09-09T05:28:00.702197Z","shell.execute_reply":"2025-09-09T05:28:05.176663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Read Train Data & Drop `id` \ntrain_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntrain_df.drop('id', axis=1, inplace=True)\n\n# Read Test Data & Drop `id`  \ntest_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\n# Read Original Data \noriginal = pd.read_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv')       ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:05.179192Z","iopub.execute_input":"2025-09-09T05:28:05.18012Z","iopub.status.idle":"2025-09-09T05:28:15.166508Z","shell.execute_reply.started":"2025-09-09T05:28:05.180072Z","shell.execute_reply":"2025-09-09T05:28:15.165621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Original: ', original.shape)  \nprint('Train: ', train_df.shape)\nprint('Test: ', test_df.shape) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:15.167718Z","iopub.execute_input":"2025-09-09T05:28:15.168099Z","iopub.status.idle":"2025-09-09T05:28:15.173209Z","shell.execute_reply.started":"2025-09-09T05:28:15.168061Z","shell.execute_reply":"2025-09-09T05:28:15.172327Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:15.175065Z","iopub.execute_input":"2025-09-09T05:28:15.175344Z","iopub.status.idle":"2025-09-09T05:28:15.198919Z","shell.execute_reply.started":"2025-09-09T05:28:15.175319Z","shell.execute_reply":"2025-09-09T05:28:15.198199Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Data Cleaning**","metadata":{}},{"cell_type":"code","source":"# Change column names to lower_case & replace space with underscore\ntrain_df.columns = train_df.columns.str.replace(' ', '_').str.lower()\ntest_df.columns = test_df.columns.str.replace(' ', '_').str.lower()\noriginal.columns = original.columns.str.replace(' ', '_').str.lower()\n\nprint('Original:\\n', original.columns)\nprint('Train:\\n', train_df.columns)\nprint('Test:\\n', test_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:15.199724Z","iopub.execute_input":"2025-09-09T05:28:15.199946Z","iopub.status.idle":"2025-09-09T05:28:15.20654Z","shell.execute_reply.started":"2025-09-09T05:28:15.199924Z","shell.execute_reply":"2025-09-09T05:28:15.205815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:15.207642Z","iopub.execute_input":"2025-09-09T05:28:15.207877Z","iopub.status.idle":"2025-09-09T05:28:15.761703Z","shell.execute_reply.started":"2025-09-09T05:28:15.207854Z","shell.execute_reply":"2025-09-09T05:28:15.760829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Change Data type for `policy_start_date` in all three dataset\noriginal['policy_start_date'] = pd.to_datetime(original['policy_start_date'])\ntrain_df['policy_start_date'] = pd.to_datetime(train_df['policy_start_date'])\ntest_df['policy_start_date'] = pd.to_datetime(test_df['policy_start_date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:15.762589Z","iopub.execute_input":"2025-09-09T05:28:15.762831Z","iopub.status.idle":"2025-09-09T05:28:16.399173Z","shell.execute_reply.started":"2025-09-09T05:28:15.762807Z","shell.execute_reply":"2025-09-09T05:28:16.398443Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **We need to verify a few key aspects to ensure consistency:**\n\n**Consistency in Columns:** Ensure all datasets have the same number of columns.\n\n**Column Names and Data Types:** Confirm that column names and their data types match in all datasets.\n\n**Categorical Variables:** There are several categorical variables, so we must verify that they have the same number of unique categories to maintain consistency. ","metadata":{}},{"cell_type":"code","source":"# Define function to check for missing columns\ndef check_missing_columns(dataset1, dataset2, name_1 = 'dataset1', name_2 = 'dataset2'):\n    # Get column names\n    dataset1_col = set(dataset1.columns)\n    dataset2_col = set(dataset2.columns)\n    \n    # Find missing columns in each dataset\n    missing_col_dataset_1 = dataset2_col - dataset1_col\n    missing_col_dataset_2 = dataset1_col - dataset2_col\n\n    # Output the result\n    if not missing_col_dataset_1 and not missing_col_dataset_2:\n        print(f'Both {name_1} and {name_2} datasets contains the same columns.')\n    else:\n        if missing_col_dataset_1:\n            print(f'Comparing {name_1} & {name_2}: \\nThe following columns are missing in {name_1} dataset \\n', missing_col_dataset_1)\n        if missing_col_dataset_2:\n            print(f'Comparing {name_1} & {name_2}: \\nThe following columns are missing in {name_2} dataset \\n', missing_col_dataset_2)\n\n# Compare both dataset to check for missing columns \ncheck_missing_columns(original, train_df, name_1 = 'original', name_2 = 'train_df')\nprint()\ncheck_missing_columns(train_df, test_df, name_1 = 'train_df', name_2 = 'test_df')\nprint()\ncheck_missing_columns(original, test_df, name_1 = 'original', name_2 = 'test_df')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:16.400563Z","iopub.execute_input":"2025-09-09T05:28:16.400828Z","iopub.status.idle":"2025-09-09T05:28:16.407749Z","shell.execute_reply.started":"2025-09-09T05:28:16.400802Z","shell.execute_reply":"2025-09-09T05:28:16.40684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compare data types between two dataset \ndef compare_data_types(dataset1, dataset2, name_1 = 'dataset1', name_2 = 'dataset2'):\n    \"\"\"\n    This function checks if the data types of common columns between two DataFrames match\n    \"\"\"\n    # Both datasets have the same columns for comparison\n    common_columns = dataset1.columns.intersection(dataset2.columns)\n    dataset1 = dataset1[common_columns]\n    dataset2 = dataset2[common_columns]\n    \n    # Extract data types for both DataFrames\n    dtypes1 = dataset1.dtypes\n    dtypes2 = dataset2.dtypes\n\n    # Create DataFrame for comparing data types\n    dtype_comparison = pd.DataFrame({f'{name_1}': dtypes1, f'{name_2}': dtypes2})\n\n    # Compare data type: Identify columns with different data types\n    dtype_mismatch = dtype_comparison[dtype_comparison[f'{name_1}'] != dtype_comparison[f'{name_2}']]\n\n    # Compare data types between two datasets\n    if dtype_mismatch.empty:\n        print(f'Comparing between {name_1} & {name_2}: \\nAll common columns have matching data types between {name_1} and {name_2}')\n    else:\n        print(f'Columns with differing data types between {name_1} and {name_2}: \\n', dtype_mismatch)\n\ncompare_data_types(original, train_df, name_1 = 'original', name_2 = 'train_df')\nprint()\ncompare_data_types(train_df, test_df, name_1 = 'train_df', name_2 = 'test_df')\nprint()\ncompare_data_types(original, test_df, name_1 = 'original', name_2 = 'test_df') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:16.408818Z","iopub.execute_input":"2025-09-09T05:28:16.40905Z","iopub.status.idle":"2025-09-09T05:28:16.924857Z","shell.execute_reply.started":"2025-09-09T05:28:16.40902Z","shell.execute_reply":"2025-09-09T05:28:16.92399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert data types in original dataset \noriginal['insurance_duration'] = original['insurance_duration'].astype('float64')\noriginal['vehicle_age'] = original['vehicle_age'].astype('float64')\n\ncompare_data_types(original, train_df, name_1 = 'original', name_2 = 'train_df')\nprint()\ncompare_data_types(original, test_df, name_1 = 'original', name_2 = 'test_df') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:16.928817Z","iopub.execute_input":"2025-09-09T05:28:16.929415Z","iopub.status.idle":"2025-09-09T05:28:17.20163Z","shell.execute_reply.started":"2025-09-09T05:28:16.929386Z","shell.execute_reply":"2025-09-09T05:28:17.200738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for duplicates in each dataset\nprint('Original:', original.duplicated().sum())\nprint('Train:', train_df.duplicated().sum())\nprint('Test:', test_df.duplicated().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:17.202877Z","iopub.execute_input":"2025-09-09T05:28:17.203245Z","iopub.status.idle":"2025-09-09T05:28:19.340241Z","shell.execute_reply.started":"2025-09-09T05:28:17.203204Z","shell.execute_reply":"2025-09-09T05:28:19.339361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check for infinite values in the `premium_amount` column\ntrain_df['premium_amount'].isin([np.inf, -np.inf]).any()       ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:19.341839Z","iopub.execute_input":"2025-09-09T05:28:19.342214Z","iopub.status.idle":"2025-09-09T05:28:19.415605Z","shell.execute_reply.started":"2025-09-09T05:28:19.342173Z","shell.execute_reply":"2025-09-09T05:28:19.414883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract all catgorical variables \ncategorical = train_df.select_dtypes(include=['object']).columns.tolist()\n\n# Extract all numerical variables\nnumerical = train_df.select_dtypes(include=['int64', 'float64']).columns.tolist()\nnumerical.remove('premium_amount')     ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:19.416617Z","iopub.execute_input":"2025-09-09T05:28:19.41702Z","iopub.status.idle":"2025-09-09T05:28:19.902431Z","shell.execute_reply.started":"2025-09-09T05:28:19.416993Z","shell.execute_reply":"2025-09-09T05:28:19.90176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Numerical Features\nnumerical ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:19.903447Z","iopub.execute_input":"2025-09-09T05:28:19.903839Z","iopub.status.idle":"2025-09-09T05:28:19.909203Z","shell.execute_reply.started":"2025-09-09T05:28:19.903801Z","shell.execute_reply":"2025-09-09T05:28:19.908372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creat dictionary to hold unqiue categories for each categorical variable\ncat_unique = {var: train_df[var].dropna().unique().tolist() for var in categorical}\ncat_unique  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:19.910367Z","iopub.execute_input":"2025-09-09T05:28:19.9108Z","iopub.status.idle":"2025-09-09T05:28:20.951359Z","shell.execute_reply.started":"2025-09-09T05:28:19.910762Z","shell.execute_reply":"2025-09-09T05:28:20.950458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check consistency of unique categories across dataset \nfor var in categorical:\n    original_unique = set(original[var].dropna().unique())\n    train_unique = set(train_df[var].dropna().unique())\n    test_unique = set(test_df[var].dropna().unique())\n\n    # Check if the count of unique categories and the categories themselves match \n    if len(original_unique) == len(train_unique) == len(test_unique) and original_unique == train_unique == test_unique:\n        print(f'✅ Categories for {var} have same count & matches across all datasets')\n    elif len(original_unique) == len(train_unique) == len(test_unique):\n        print(f'Categories for {var} have same count in all datasets but categories do not match excatly')\n    else:\n        print(f'Number of unique categories for {var} does not match across all datasets')           ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:20.952393Z","iopub.execute_input":"2025-09-09T05:28:20.952669Z","iopub.status.idle":"2025-09-09T05:28:22.920355Z","shell.execute_reply.started":"2025-09-09T05:28:20.952642Z","shell.execute_reply":"2025-09-09T05:28:22.919475Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Distribution Analysis of Categorical Variables**","metadata":{}},{"cell_type":"code","source":"# Setup subplots\nfig, axes = plt.subplots(len(categorical), 3, figsize = (13, 5 * len(categorical)))\n        \n# Create subplots\nfor i, var in enumerate(categorical):\n    original_counts = original[var].value_counts()\n    train_counts = train_df[var].value_counts()\n    test_counts = test_df[var].value_counts()\n\n    # Plot pie chart for original\n    axes[i, 0].pie(original_counts,\n                   labels = original_counts.index,\n                   autopct = '%1.1f%%',\n                   labeldistance = 1.05,\n                   pctdistance = 0.75,\n                   textprops = {'fontsize': 10, 'fontweight': 'bold'},\n                   colors = plt.cm.Pastel1.colors)\n    axes[i, 0].set_title(f'Original: {var.capitalize()}', weight = 'bold')\n\n    # Plot pie chart for train_df\n    axes[i, 1].pie(train_counts,\n                   labels = train_counts.index,\n                   autopct = '%1.1f%%',\n                   labeldistance = 1.05,\n                   pctdistance = 0.75,\n                   textprops = {'fontsize': 10, 'fontweight': 'bold'},\n                   colors = plt.cm.Pastel1.colors)\n    axes[i, 1].set_title(f'Train: {var.capitalize()}', weight = 'bold')\n\n    # Plot pie chart for test_df\n    axes[i, 2].pie(test_counts,\n                   labels = test_counts.index,\n                   autopct = '%1.1f%%',\n                   labeldistance = 1.05,\n                   pctdistance = 0.75,\n                   textprops = {'fontsize': 10, 'fontweight': 'bold'},\n                   colors = plt.cm.Pastel1.colors)\n    axes[i, 2].set_title(f'Test: {var.capitalize()}', weight = 'bold')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:22.921387Z","iopub.execute_input":"2025-09-09T05:28:22.921674Z","iopub.status.idle":"2025-09-09T05:28:26.56892Z","shell.execute_reply.started":"2025-09-09T05:28:22.921645Z","shell.execute_reply":"2025-09-09T05:28:26.568142Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"All categorical variables in the train, test, and original datasets exhibit balanced distributions. For example, the gender variable is nearly evenly split, with **50.1%** male and **49.9%** female. Similarly, other categorical variables also have a fairly even distribution across their unique categories.","metadata":{}},{"cell_type":"markdown","source":"# **Distribution Analysis of Numerical Variables**\n\nIn this section, we will examine and compare the distribution of numerical vaiables across the original, train, and test datasets. This analysis aims to:\n- Ensure consistency in data distribution across the datasets.\n- Identify any potential outliers.","metadata":{}},{"cell_type":"code","source":"# Setup subplots\nfig, axes = plt.subplots(len(numerical), 2, figsize = (13, 5 * len(numerical)))\n\n# Color for datasets\npalette = {'Original':'blue', 'Train':'green', 'Test':'orange'}\n\n# Plot Histogram\nfor i, var in enumerate(numerical):\n    axes[i, 0].hist(original[var], alpha=0.5, color=palette['Original'], label='Original')\n    axes[i, 0].hist(train_df[var], alpha=0.5, color=palette['Train'], label='Train')\n    axes[i, 0].hist(test_df[var], alpha=0.5, color=palette['Test'], label='Test')\n    axes[i, 0].set_title(f'Histogram for {var}', weight='bold')\n    axes[i, 0].legend()\n\n    # Prepare data for boxplot \n    combined = pd.concat([original[var].to_frame().assign(dataset='Original'),\n                         train_df[var].to_frame().assign(dataset='Train'),\n                         test_df[var].to_frame().assign(dataset='Test')\n                         ])\n    # Plot Boxplot\n    sns.boxplot(data=combined, x='dataset', y=var, ax=axes[i, 1], palette=palette)\n    axes[i, 1].set_title(f'Boxplot for {var}', weight='bold')\n\nplt.tight_layout()\nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:26.570109Z","iopub.execute_input":"2025-09-09T05:28:26.570794Z","iopub.status.idle":"2025-09-09T05:28:36.978193Z","shell.execute_reply.started":"2025-09-09T05:28:26.570728Z","shell.execute_reply":"2025-09-09T05:28:36.977311Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **age, insurance_duration, vehicle_age, number_of_dependents**","metadata":{}},{"cell_type":"code","source":"print('Original Descriptive Stats:')  \nprint(original[['age', 'insurance_duration', 'vehicle_age', 'number_of_dependents']].describe().T)\nprint()\nprint('Train Descriptive Stats:')\nprint(train_df[['age', 'insurance_duration', 'vehicle_age', 'number_of_dependents']].describe().T)\nprint()\nprint('Test Descriptive Stats:')\nprint(test_df[['age', 'insurance_duration', 'vehicle_age', 'number_of_dependents']].describe().T)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:36.979566Z","iopub.execute_input":"2025-09-09T05:28:36.979914Z","iopub.status.idle":"2025-09-09T05:28:37.422547Z","shell.execute_reply.started":"2025-09-09T05:28:36.979878Z","shell.execute_reply":"2025-09-09T05:28:37.421743Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The shape of the data distribution in the histograms indicates that the distributions for all four variables - **`age`, `insurance_duration`, `vehicle_age`, and `number_of_dependents`** are similar across the three datasets. Boxplots show no outliers for these variables in any of the datasets. \n\nThe descriptive statistics and boxplots suggest that the distributions of these variables are consistent across the three datasets. This consistency is reflected in the approximately similar means, medians, quartiles, minimums, and maximums, as well as the absence of outliers. ","metadata":{}},{"cell_type":"markdown","source":"## **credit_score and health_score**","metadata":{}},{"cell_type":"code","source":"print('Original Descriptive Stats:')\nprint(original[['credit_score', 'health_score']].describe())\nprint()\nprint('Train Descriptive Stats:')\nprint(train_df[['credit_score', 'health_score']].describe())\nprint()\nprint('Test Descriptive Stats:')\nprint(test_df[['credit_score', 'health_score']].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:37.423738Z","iopub.execute_input":"2025-09-09T05:28:37.424103Z","iopub.status.idle":"2025-09-09T05:28:37.666509Z","shell.execute_reply.started":"2025-09-09T05:28:37.424061Z","shell.execute_reply":"2025-09-09T05:28:37.665633Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The descriptive statistics for **`credit_score`** and **`health_score`** across the three datasets show similar patterns, with means, medians, quartiles, minimums, and maximums being quite close, though not identical. The boxplots for **`credit_score`** indicate no outliers in any of the datasets. \n\nHowever, the box for the original dataset is slightly larger than those for the train and test datasets, which are of equal size. This suggests that while the overall distribution of **`credit_score`** is consistent, the original dataset has a slightly wider spread of values compared to the train and test datasets. \n\nThe credit scores in the train and test dataset are generally on the **lower side**, with a mean of **592** and a median of **595**. Descriptive statistics reveal that **75%** of individuals in this dataset have a credit score below **721**, further highlighting the **overall lower credit score distribution.**\n\nFor **`health_score`**, the histograms shows that the distribution is quite similar across the three datasets, with a **slightly longer tail** in the original dataset. The boxplot reveals that there are outliers in the **`health_score`** of the original dataset, while the train and test datasets do not have any outliers. ","metadata":{}},{"cell_type":"markdown","source":"## **annual_income and previous_claims**","metadata":{}},{"cell_type":"code","source":"print('Original Descriptive Stats:') \nprint(original[['annual_income', 'previous_claims']].describe())\nprint()\nprint('Train Descriptive Stats:')\nprint(train_df[['annual_income', 'previous_claims']].describe())\nprint()\nprint('Test Descriptive Stats:')\nprint(test_df[['annual_income', 'previous_claims']].describe())  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:37.667536Z","iopub.execute_input":"2025-09-09T05:28:37.667819Z","iopub.status.idle":"2025-09-09T05:28:37.929695Z","shell.execute_reply.started":"2025-09-09T05:28:37.667792Z","shell.execute_reply":"2025-09-09T05:28:37.928776Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The histogram shows that the distribution of **`annual_income`** is **right-skewed** for all the three datasets, with the shape being consistent across them. The boxplot reveals that the original dataset has a wider spread of values compared to the train and test datasets. Additionally, the boxplots indicate the presence of outliers in all datasets.\n\nThe boxplot of previous_claims reveals the presence of outliers across all three datasets. Descriptive statistics show that **75%** of policyholders have **two or fewer previous claims**, indicating that higher claim counts are relatively rare and may represent exceptional cases or outliers.","metadata":{}},{"cell_type":"markdown","source":"## **Analyzing the Target Variable - premium_amount**","metadata":{}},{"cell_type":"code","source":"# Set subplots \nfig, axes = plt.subplots(2, 2, figsize=(10, 4)) \n\n# Histogram for the original dataset \nsns.histplot(original['premium_amount'], ax=axes[0, 0], kde=False, bins=50) \naxes[0, 0].set_title('Histogram of Premium Amount (Original)',fontsize=10, weight='bold') \n\n# Histogram for the train_df dataset \nsns.histplot(train_df['premium_amount'], ax=axes[0, 1], kde=False, bins=50) \naxes[0, 1].set_title('Histogram of Premium Amount (Train)', fontsize=10, weight='bold') \n\n# Boxplot for the original dataset \nsns.boxplot(x=original['premium_amount'], ax=axes[1, 0]) \naxes[1, 0].set_title('Boxplot of Premium Amount (Original)', fontsize=10, weight='bold') \n\n# Boxplot for the train_df dataset \nsns.boxplot(x=train_df['premium_amount'], ax=axes[1, 1]) \naxes[1, 1].set_title('Boxplot of Premium Amount (Train)', fontsize=10, weight='bold') \n\n# Adjust layout \nplt.tight_layout() \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:37.930962Z","iopub.execute_input":"2025-09-09T05:28:37.931618Z","iopub.status.idle":"2025-09-09T05:28:39.573529Z","shell.execute_reply.started":"2025-09-09T05:28:37.931576Z","shell.execute_reply":"2025-09-09T05:28:39.572603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Original Descriptive Stats:') \nprint(original[['premium_amount']].describe())\nprint()\nprint('Train Descriptive Stats:')\nprint(train_df[['premium_amount']].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:39.574633Z","iopub.execute_input":"2025-09-09T05:28:39.574908Z","iopub.status.idle":"2025-09-09T05:28:39.643742Z","shell.execute_reply.started":"2025-09-09T05:28:39.57488Z","shell.execute_reply":"2025-09-09T05:28:39.642857Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The target variable **`premium_amount`** has a **right-skewed distribution** in both the original and train datasets. In the original dataset, the histogram shows a right-skewed distribution with **each bin decreasing in size**. In the train dataset, the distribution is also **right-skewed**, with a **slight peak between 600 and 700**. Outliers can be detected in the boxplots for both datasets.","metadata":{}},{"cell_type":"markdown","source":"# **Missing Value**","metadata":{}},{"cell_type":"code","source":"# Missing values in Original data  \noriginal_count = original.isna().sum()\noriginal_perct = round(original.isna().sum() / len(original) * 100, 2)\n\n# Missing values in Train data\ntrain_count = train_df.isna().sum()\ntrain_perct = round(train_df.isna().sum() / len(train_df) * 100, 2)\n\n# Missing values in Test data\ntest_count = test_df.isna().sum()\ntest_perct = round(test_df.isna().sum() / len(test_df) * 100, 2)\n\n# Create a DataFrame for missing value summary for three datasets\nmissing_summary = pd.DataFrame({\n    'original_count': original_count,\n    'original_perct': original_perct,\n    'train_count': train_count,\n    'train_perct': train_perct,\n    'test_count': test_count,\n    'test_perct': test_perct\n})\n\nprint('\\n' + '='*85) \nprint(f\"{'SUMMARY OF MISSING VALUES ACROSS THREE DATASETS':^85}\") \nprint('='*85 + '\\n')\n\nmissing_summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:39.644867Z","iopub.execute_input":"2025-09-09T05:28:39.645975Z","iopub.status.idle":"2025-09-09T05:28:41.456372Z","shell.execute_reply.started":"2025-09-09T05:28:39.645944Z","shell.execute_reply":"2025-09-09T05:28:41.455554Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The missing values across the three datasets reveals that the proportion of missing data ranges approximately from **2%** to **30%**. Importantly, the missing values percentages are consistent across all three datasets. While some features exhibit minimal or no missing values, others such as **`occupation`** and **`previous_claims`** have nearly **30%** missing data.\n\nTo effectively handle these missing values, it is essential to examine the missing value patterns. We need to determine whether these values are missing at random (MAR) or exibit correlation with other features. ","metadata":{}},{"cell_type":"code","source":"import missingno as msno\n\n# Visualize missing data patterns\nmsno.matrix(train_df)\nplt.title('Missing Data Pattern for Train Dataset', size=18, weight='bold')\nplt.show()\n\nmsno.matrix(train_df)\nplt.title('Missing Data Pattern Test Dataset', size=18, weight='bold')\nplt.show()\n\n# Visualize missing data correlations\nmsno.heatmap(train_df, figsize=(8, 4))\nplt.title('Missing Values Correlation for Train Dataset', size=10, weight='bold')\nplt.show()\n\nmsno.heatmap(test_df, figsize=(8, 4))\nplt.title('Missing Values Correlation for Test Dataset', size=10, weight='bold')\nplt.show()  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:41.457462Z","iopub.execute_input":"2025-09-09T05:28:41.45775Z","iopub.status.idle":"2025-09-09T05:28:55.21902Z","shell.execute_reply.started":"2025-09-09T05:28:41.457718Z","shell.execute_reply":"2025-09-09T05:28:55.218236Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The **`msno.matrix`** visualization indicates that the missing values across all features in both datasets appear more random with no clear pattern. Features such as **`previous_claims`** and **`occupation`** have significant missing values, but these values are scattered randomly rather than being clustered together. This suggests that the missingness is more likely to be **Missing at Random (MAR)**, as there doesn't seem to be a specific trend. \n\nThe **`msno.heatmap`** visualization shows white cells for all features, indicating no significant correlation between the missing values of different features. This indicates that the missing values are not correlated with each other. The missingness in one feature does not predict or depend on the missingness in another feature, further supporting the observation that the data is **Missing  at Random (MAR)**.","metadata":{}},{"cell_type":"markdown","source":"## **Concatenate Original & Train Datasets**\n\nAfter analyzing the data distribution of both categorical and numerical features, I decided **not to concatenate** the original and train datasets. The shapes of the data distributions are consistent for each variable across all three datasets, as indicated by the histograms. However, the boxplots for few variables in the original dataset show wider values and outliers that are not present in the train and test datasets.","metadata":{}},{"cell_type":"markdown","source":"# **Correlation**","metadata":{}},{"cell_type":"code","source":"def heatmap(df, df_name):\n    plt.figure(figsize=(14, 10))\n    sns.heatmap(df.corr(method='pearson', numeric_only=True), annot=True, cmap='coolwarm')\n    plt.title(f'Correlation Heatmap for {df_name} Data', fontsize=14, weight='bold')\n    plt.show();\n\nheatmap(train_df, 'Train')\nheatmap(test_df, 'Test')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:55.220244Z","iopub.execute_input":"2025-09-09T05:28:55.220636Z","iopub.status.idle":"2025-09-09T05:28:56.600393Z","shell.execute_reply.started":"2025-09-09T05:28:55.220599Z","shell.execute_reply":"2025-09-09T05:28:56.599546Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* The heatmap indicates that none of the features have a strong correlation with the target variable (**`premium_amount`**). **`previous_claims`** have a weak correlation of **0.047** with the **`premium_amount`** in the training dataset.\n  \n* **`credit_score`** and **`annual_income`** shows negative correlation coefficients of **-0.20** in both train and test dataset, indicating very weak relationship. The features do not show high correlation with the target variable (premium_amount), meaning linear relationships are weak or nonexistent.","metadata":{}},{"cell_type":"markdown","source":"# **Evaluating Premium Amount Trends by Various Factors**\nBy comparing premium amount trends across different factors such as property type, location, policy type, year, month, and previous claims, we can better understand their influence. This analysis can identify patterns and potential drivers of premium variations.","metadata":{}},{"cell_type":"code","source":"def boxplot(var):\n    plt.figure(figsize=(8, 4))\n    sns.boxplot(data=train_df, x=var, y='premium_amount', palette='Set2')\n    plt.title(f'Boxplot of {var} vs premium_amount', size=12, weight='bold') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:56.601668Z","iopub.execute_input":"2025-09-09T05:28:56.60209Z","iopub.status.idle":"2025-09-09T05:28:56.607695Z","shell.execute_reply.started":"2025-09-09T05:28:56.601986Z","shell.execute_reply":"2025-09-09T05:28:56.606825Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Premium Amount by Property Type, Policy Type, and Location**","metadata":{}},{"cell_type":"code","source":"boxplot('property_type')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:56.612154Z","iopub.execute_input":"2025-09-09T05:28:56.612486Z","iopub.status.idle":"2025-09-09T05:28:57.430977Z","shell.execute_reply.started":"2025-09-09T05:28:56.612461Z","shell.execute_reply":"2025-09-09T05:28:57.430038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `property_type`\ntrain_df.groupby('property_type')['premium_amount'].describe()   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:57.431923Z","iopub.execute_input":"2025-09-09T05:28:57.432188Z","iopub.status.idle":"2025-09-09T05:28:57.570673Z","shell.execute_reply.started":"2025-09-09T05:28:57.432162Z","shell.execute_reply":"2025-09-09T05:28:57.56985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('policy_type')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:57.571785Z","iopub.execute_input":"2025-09-09T05:28:57.572148Z","iopub.status.idle":"2025-09-09T05:28:58.327233Z","shell.execute_reply.started":"2025-09-09T05:28:57.57212Z","shell.execute_reply":"2025-09-09T05:28:58.326367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `policy_type`\ntrain_df.groupby('policy_type')['premium_amount'].describe() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:58.328345Z","iopub.execute_input":"2025-09-09T05:28:58.328629Z","iopub.status.idle":"2025-09-09T05:28:58.468369Z","shell.execute_reply.started":"2025-09-09T05:28:58.328602Z","shell.execute_reply":"2025-09-09T05:28:58.467566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('location')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:58.46957Z","iopub.execute_input":"2025-09-09T05:28:58.469984Z","iopub.status.idle":"2025-09-09T05:28:59.457465Z","shell.execute_reply.started":"2025-09-09T05:28:58.469942Z","shell.execute_reply":"2025-09-09T05:28:59.456623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `location`\ntrain_df.groupby('location')['premium_amount'].describe()  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:59.458672Z","iopub.execute_input":"2025-09-09T05:28:59.459043Z","iopub.status.idle":"2025-09-09T05:28:59.590234Z","shell.execute_reply.started":"2025-09-09T05:28:59.459001Z","shell.execute_reply":"2025-09-09T05:28:59.589331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* The descriptive statistics indicate that the premium amounts for different **property_types** (Apartment, Condo, and House) are quite similar. The means and standard deviations are almost identical, and the minimum, 25%, 50% (median), 75%, and maximum values are also very close to each other. This indicates that there is no significant difference in the premium amount across different property types within this dataset.\n\n* The boxplots for the property type exhibit a **consistent box size**, suggesting a relatively stable distribution of premium amounts across categories. However, the **presence of outliers** in each category indicates that some individuals have exceptionally high premiums compared to the majority. These outliers are common in real-world insurance data, where premium amounts are influenced by multiple complex factors beyond the visible categorical attributes.\n\n* This pattern extends to **location** and **policy_types**, suggesting that premium amounts do not significantly vary based on these factors within this dataset. It implies that these factors may not be influential in determining the premium amount.\n\n## **Premium Amount by Year**","metadata":{}},{"cell_type":"code","source":"def temporal(df):\n    # Extract temporal features from `policy_start_date` \n    df['year'] = df['policy_start_date'].dt.year\n    df['month'] = df['policy_start_date'].dt.month\n\ntemporal(train_df)\ntemporal(test_df)\n\n# Descriptive Statistics of `premium_amount` by `year`\ntrain_df.groupby('year')['premium_amount'].describe() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:59.591316Z","iopub.execute_input":"2025-09-09T05:28:59.591649Z","iopub.status.idle":"2025-09-09T05:28:59.839011Z","shell.execute_reply.started":"2025-09-09T05:28:59.591621Z","shell.execute_reply":"2025-09-09T05:28:59.838083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('year')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:28:59.840004Z","iopub.execute_input":"2025-09-09T05:28:59.84029Z","iopub.status.idle":"2025-09-09T05:29:00.277831Z","shell.execute_reply.started":"2025-09-09T05:28:59.840263Z","shell.execute_reply":"2025-09-09T05:29:00.276996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The analysis of **premium_amount** by year shows slight variations. While the means, medians, and standard deviations are fairly consistent, there are observable differences. In **2019**, the mean **(1189.56)**, median **(944)**, and standard deviation **(914.40)** are the highest. From **2020 to 2024**, the values are more consistent.\n\nWhile average premiums didn't change drastically year-to-year, outliers indicate that extreme premium values continue to exist in all years. These might reflect rare high-risk cases or premium for high-value policies. Although the variations are not drastic, they suggest there might be some yearly influence on premium amounts.\n\n## **Premium Amount by Months**","metadata":{}},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `month`\ntrain_df.groupby('month')['premium_amount'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:00.27899Z","iopub.execute_input":"2025-09-09T05:29:00.279369Z","iopub.status.idle":"2025-09-09T05:29:00.371408Z","shell.execute_reply.started":"2025-09-09T05:29:00.279328Z","shell.execute_reply":"2025-09-09T05:29:00.370548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('month')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:00.372459Z","iopub.execute_input":"2025-09-09T05:29:00.372737Z","iopub.status.idle":"2025-09-09T05:29:00.881477Z","shell.execute_reply.started":"2025-09-09T05:29:00.372711Z","shell.execute_reply":"2025-09-09T05:29:00.880555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The analysis of **premium_amount** by months **shows slight variations**. While the mean, median, and standard deviations are fairly consistent, there are some observable differences. Minor variations suggest that there might be some monthly influence on premium amounts, but the overall impact appears to be minimal.\n\n## **Premium Amount by Previous Claims**","metadata":{}},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `previous_claims`\ntrain_df.groupby('previous_claims')['premium_amount'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:00.882449Z","iopub.execute_input":"2025-09-09T05:29:00.882699Z","iopub.status.idle":"2025-09-09T05:29:00.97174Z","shell.execute_reply.started":"2025-09-09T05:29:00.882675Z","shell.execute_reply":"2025-09-09T05:29:00.970859Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('previous_claims')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:00.972858Z","iopub.execute_input":"2025-09-09T05:29:00.973133Z","iopub.status.idle":"2025-09-09T05:29:01.480346Z","shell.execute_reply.started":"2025-09-09T05:29:00.973108Z","shell.execute_reply":"2025-09-09T05:29:01.479565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate medians for premium amounts grouped by previous_claims \nmedian_values = train_df.groupby('previous_claims')['premium_amount'].median().reset_index()\n\n# Update data dictionary with these median values \ndata = {'previous_claims': median_values['previous_claims'].tolist(), \n        'median_premium': median_values['premium_amount'].tolist()}\n\ndf = pd.DataFrame(data)\n\n# Scatter plot with trend line \nplt.figure(figsize=(8, 3)) \nsns.regplot(data=df, x='previous_claims', y='median_premium', ci=None) \nplt.title('Median Premium Amount vs. Number of Previous Claims', \n          fontsize=12, weight='bold') \nplt.grid(True) \nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:01.481593Z","iopub.execute_input":"2025-09-09T05:29:01.481956Z","iopub.status.idle":"2025-09-09T05:29:01.744549Z","shell.execute_reply.started":"2025-09-09T05:29:01.481918Z","shell.execute_reply":"2025-09-09T05:29:01.743636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The scatterplot with a regression line indicates a generally positive trend between the **number of previous claims** and the **median premium amount**. This aligns with the expectation that individuals with more previous claims tend to have higher premiums. This observation supports the idea that insurers may increase premiums as the number of claims rises, likely due to the perceived risk associated with repeat claims.\n\nAdditionally, the plot shows a clear upward trend in the median premium as the number of previous claims increases. However, this trend becomes less reliable for higher claim counts, particularly for **previous_claims > 5.** This inconsistency may be attributed to the **presence of outliers** or **limited data** for higher claim counts.","metadata":{}},{"cell_type":"markdown","source":"## **Premium Amount by Occupation**","metadata":{}},{"cell_type":"code","source":"# Descriptive Statistics of `premium_amount` by `occupation`\ntrain_df.groupby('occupation')['premium_amount'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:01.745687Z","iopub.execute_input":"2025-09-09T05:29:01.745971Z","iopub.status.idle":"2025-09-09T05:29:01.884264Z","shell.execute_reply.started":"2025-09-09T05:29:01.745945Z","shell.execute_reply":"2025-09-09T05:29:01.883389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"boxplot('occupation')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:01.885303Z","iopub.execute_input":"2025-09-09T05:29:01.885611Z","iopub.status.idle":"2025-09-09T05:29:02.660752Z","shell.execute_reply.started":"2025-09-09T05:29:01.885583Z","shell.execute_reply":"2025-09-09T05:29:02.659837Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The **median premium amount** appears to be **roughly the same** for **Employed, Self-Employed**, and **Unemployed** policyholders. This suggests occupation does not significantly impact premiums. Similar spread indicate (Interquartile Range - IQR)  that the premium amounts are distributed similarly across all three groups, meaning there's **no distinct pattern linking occupation to large variations in premiums.** \n\nBoxplot indicates each category contains outliers (high premium amounts). While the dataset is synthetically generated, the presence of outliers aligns with expected trends in real-world insurance data, where extreme premium values often arise due to various risk-level factors.","metadata":{}},{"cell_type":"markdown","source":"## **Most Common Occupation by Income, Education, and Gender**","metadata":{}},{"cell_type":"code","source":"# Create income bins & label \nincome_bins = [0, 25000, 50000, 75000, 100000, float('inf')]\nincome_labels = ['Low', 'Low-Middle', 'Middle', 'Upper-Middle', 'High']\n\n# Create`income_bin`\ntrain_df['income_bin'] = pd.cut(train_df['annual_income'], bins=income_bins,\n                                  labels=income_labels, include_lowest=True)     \n\n# Group by `income_bin`, `education_level`, and `occupation` & count occurrences\noccupation_counts = train_df.groupby(\n    ['income_bin', 'education_level', 'gender'])['occupation'].value_counts().reset_index(name='count')\n\n# Find the mode for each group \nmode_occupation = occupation_counts.loc[occupation_counts.groupby(\n    ['income_bin', 'education_level', 'gender'])['count'].idxmax()]\n\n# List of unique income_bin\nincome_bins = mode_occupation['income_bin'].unique() \n\n# Define a custom color palette for occupations \noccupation_palette = { 'Employed': '#1f77b4', 'Self-Employed': '#ff7f0e', 'Unemployed': '#2ca02c'}\n\n# Set up the plot size & title\nfig, axes = plt.subplots(1, 5, figsize=(25, 6), sharey=True) \nfig.suptitle('Most Common Occupations by Income Bin, Education Level, and Gender', fontsize=20, weight='bold') \n\n# Plotting \nfor i, income_bin in enumerate(income_bins):\n    ax = axes[i] \n    subset = mode_occupation[mode_occupation['income_bin'] == income_bin] \n    sns.barplot(data=subset, x='education_level', y='count', palette=occupation_palette,\n                hue='occupation', dodge=True, ax=ax) \n    ax.set_title(f'{income_bin} Income Group', size=15, weight='bold') \n    ax.set_xlabel('Education Level', fontdict=dict(weight='bold')) \n    ax.legend() \n\n    # Set the x-axis labels in bold\n    for label in ax.get_xticklabels():\n        label.set_weight('bold')\n    \n# Adjust layout \nplt.tight_layout(rect=[0, 0, 1, 0.95]) \nplt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:02.661836Z","iopub.execute_input":"2025-09-09T05:29:02.662108Z","iopub.status.idle":"2025-09-09T05:29:04.13433Z","shell.execute_reply.started":"2025-09-09T05:29:02.662082Z","shell.execute_reply":"2025-09-09T05:29:04.133447Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The above plots display bar plots for each **income_bin**, showing the most common occupation for each combination of **education_level** and **gender**. Each education level contains two bars, one for each gender, colored differently based on the occupation. If both genders have the same occupation within the same education level and income bin, the bars will appear together (single bar). Otherwise, there will be two distinct bars indicating the occupations.\n\nThe above plots clearly show that our dataset has **significant variability in occupation distributions** across groups defined by **income_bin**, **education_level**, and **gender**. The plots indicate that the proportions of occupations differ not only by income level and education level but also by gender within those groups. \n\nWhile gender may not directly define occupation, it is relevant in the context of our dataset as it contributes to the observed variability in occupations within different income levels and education groups.","metadata":{}},{"cell_type":"markdown","source":"# **Model Construction**\n## **Feature Creation**\n\nThere are several ordinal features, such as **`education_level`**, **`customer_feedback`**, and others, which can be meaningfully converted to numerical values due to their inherent order.\n\nThe **`create_features`** function generates row-wise interaction and ratio-based features, these do not rely on global dataset statistics. While these features are not at risk of causing data leakage, they are created inside the KFold loop because they depend on input features that may contain missing values. By imputing missing data within each fold first, we ensure that these derived features are based on clean and reliable inputs. \n\nIn addition, we use a **`group_feature`** function to construct group-wise combinations, for example, combining **`previous_claims`** and **`number_of_dependents`**, which are then used for categorical target encoding. This approach is motivated by our earlier analysis showing that individual categorical features have **weak correlation** with the target variable. Grouping them might introduce more granularity and potentially providing additional predictive power to the model when used with **TargetEncoder**.","metadata":{}},{"cell_type":"code","source":"def oridinal(df):\n    # Mapping for oridinal encoding\n    policy_type_mapping = {'Basic': 1, 'Premium': 2, 'Comprehensive': 3}\n    customer_feedback_mapping = {'Poor': 1, 'Average': 2, 'Good': 3}\n    education_level_mapping = {'High School': 1, \"Bachelor's\": 2, \"Master's\": 3, 'PhD': 4}\n    location_mapping = {'Rural': 1, 'Suburban': 2, 'Urban': 3}\n    exercise_mapping = {'Rarely': 1, 'Monthly': 2, 'Weekly': 3, 'Daily': 4}\n    \n    df['policy_type_enco'] = df['policy_type'].map(policy_type_mapping)\n    df['customer_feedback_enco'] = df['customer_feedback'].map(customer_feedback_mapping)\n    df['education_level_enco'] = df['education_level'].map(education_level_mapping)\n    df['location_enco'] = df['location'].map(location_mapping)\n    df['exercise_enco'] = df['exercise_frequency'].map(exercise_mapping)   \n    \n    # Encode smoking status as binary\n    df['smoking_enco'] = np.where(df['smoking_status'] == 'Yes', 1, 0).astype(int) \n\n# Apply to train_df & test_df\noridinal(train_df)\noridinal(test_df) \n\n\nnumerical = numerical + ['year', 'month', 'policy_type_enco', 'customer_feedback_enco', 'previous_claim_te', \n                         'education_level_enco', 'location_enco', 'exercise_enco', 'smoking_enco', 'month_te', \n                         'year_te', 'number_of_dependents_te']\n\n# Remove 'smoking_status' from the categorical list \ncategorical.remove('smoking_status')   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:04.135304Z","iopub.execute_input":"2025-09-09T05:29:04.135561Z","iopub.status.idle":"2025-09-09T05:29:04.840642Z","shell.execute_reply.started":"2025-09-09T05:29:04.135536Z","shell.execute_reply":"2025-09-09T05:29:04.839927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create interaction and ratio-based features \ndef create_features(df):   \n    # Create Claim Frequency \n    df['claim_frequency'] = df['previous_claims'] / df['insurance_duration']\n    # Create Policy Tenure Ratio\n    df['policy_tenure_ratio'] = df['insurance_duration'] / df['age']\n    # Create Claim Ratio per Year \n    df['claim_ratio_per_year'] = df['previous_claims'] / (df['insurance_duration'] / 12)\n    # Create Credit Health Interaction\n    df['credit_health_interaction'] = df['credit_score'] * df['health_score']\n    # Create Claims to Income Ratio\n    df['claims_to_income_ratio'] = (df['previous_claims'] / df['annual_income']) * 10000\n    # Dependents to Income Ratio\n    df['dependents_income_ratio'] = (df['number_of_dependents'] / df['annual_income']) * 10000\n    # Claims per dependent\n    df['claims_per_dependents'] = df['previous_claims'] / (df['number_of_dependents'] + 1)  \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:04.841606Z","iopub.execute_input":"2025-09-09T05:29:04.84185Z","iopub.status.idle":"2025-09-09T05:29:04.847806Z","shell.execute_reply.started":"2025-09-09T05:29:04.841825Z","shell.execute_reply":"2025-09-09T05:29:04.846931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create group features - this will be used for categorical target encoding \ndef group_feature(df):\n    df['month_year'] = df['month'].astype(str) + '_' + df['year'].astype(str)\n    df['previous_claims_dependents'] = df['previous_claims'].astype(str) + '_' + df['number_of_dependents'].astype(str)\n    df['gender_education'] = df['gender'].astype(str) + '_' + df['education_level'].astype(str)\n    df['gender_occupation'] = df['gender'].astype(str) + '_' + df['occupation'].astype(str)\n    return df\n\ncategorical_2 = ['previous_claim_te', 'month_te', 'year_te', 'number_of_dependents_te', \n                 'month_year', 'previous_claims_dependents', 'gender_education', 'gender_occupation']  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:04.848915Z","iopub.execute_input":"2025-09-09T05:29:04.849159Z","iopub.status.idle":"2025-09-09T05:29:04.861351Z","shell.execute_reply.started":"2025-09-09T05:29:04.849134Z","shell.execute_reply":"2025-09-09T05:29:04.860443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Covert data type to category from object\ndef change_type(df):\n    for col in df.select_dtypes('object').columns:\n        df[col] = df[col].astype('category')\n\nchange_type(train_df)\nchange_type(test_df)   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:04.862299Z","iopub.execute_input":"2025-09-09T05:29:04.86264Z","iopub.status.idle":"2025-09-09T05:29:06.668988Z","shell.execute_reply.started":"2025-09-09T05:29:04.862595Z","shell.execute_reply":"2025-09-09T05:29:06.668048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Custom RMSLE \ndef rmsle(y_true, y_pred):\n    y_true_original = np.expm1(y_true)\n    y_pred_original = np.expm1(y_pred)\n    y_pred_original = np.maximum(y_pred_original, 0)\n    return np.sqrt(mean_squared_log_error(y_true_original, y_pred_original))   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:06.669921Z","iopub.execute_input":"2025-09-09T05:29:06.670153Z","iopub.status.idle":"2025-09-09T05:29:06.674888Z","shell.execute_reply.started":"2025-09-09T05:29:06.670129Z","shell.execute_reply":"2025-09-09T05:29:06.673902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Imputation \nnum_imputer = SimpleImputer(strategy='median')\ncat_imputer = SimpleImputer(strategy='most_frequent')\n\npreprocessor = ColumnTransformer(transformers=[\n    ('num', num_imputer, numerical),\n    ('cat', cat_imputer, categorical)\n])   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:06.67604Z","iopub.execute_input":"2025-09-09T05:29:06.676652Z","iopub.status.idle":"2025-09-09T05:29:06.690061Z","shell.execute_reply.started":"2025-09-09T05:29:06.676587Z","shell.execute_reply":"2025-09-09T05:29:06.689286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Features & Target \nX = train_df.drop(columns=['premium_amount', 'policy_start_date', 'smoking_status'])\nX['previous_claim_te'] = X['previous_claims']\nX['month_te'] = X['month']\nX['year_te'] = X['year']\nX['number_of_dependents_te'] = X['number_of_dependents']\n\n# Log Transform target variable\ny = np.log1p(train_df['premium_amount'])\n\n# Log Transform annual_income \nX['annual_income'] = np.log1p(X['annual_income'])\n\nX_test = test_df.drop(columns=['id', 'policy_start_date', 'smoking_status'])   \nX_test['previous_claim_te'] = X_test['previous_claims'] \nX_test['month_te'] = X_test['month']\nX_test['year_te'] = X_test['year']      \nX_test['number_of_dependents_te'] = X_test['number_of_dependents']\n\n# Log Transform annual_income and health_score\nX_test['annual_income'] = np.log1p(X_test['annual_income'])   \n\n# Hyperparmeters \nparams = {\n    'n_estimators': 12000,\n    'learning_rate': 0.01,\n    'subsample': 0.75, \n    'colsample_bytree': 0.75,\n    'reg_lambda': 0.3, \n    'reg_alpha': 0.3, \n    'num_leaves': 64, \n    'max_depth': 10, \n    'min_child_samples': 30, \n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'device': 'gpu',\n    'random_state': 42\n}         ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:06.69114Z","iopub.execute_input":"2025-09-09T05:29:06.691466Z","iopub.status.idle":"2025-09-09T05:29:06.855458Z","shell.execute_reply.started":"2025-09-09T05:29:06.69143Z","shell.execute_reply":"2025-09-09T05:29:06.854509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.shape, X_test.shape, y.shape   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:06.856679Z","iopub.execute_input":"2025-09-09T05:29:06.857043Z","iopub.status.idle":"2025-09-09T05:29:06.863449Z","shell.execute_reply.started":"2025-09-09T05:29:06.857005Z","shell.execute_reply":"2025-09-09T05:29:06.862483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize OOF arrays      \noof_preds = np.zeros(len(X))\ntest_preds = np.zeros(len(X_test))\nrmsle_scores = []\n\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\n# Function for binned target encoding\ndef add_binned_target_encoding(df, df_list, target, features, q=100): \n    for feature in features:\n        df[f'{feature}_bin'] = pd.qcut(df[feature], q=q, duplicates='drop')\n\n        aggs = ['mean', 'median', 'min', 'max', 'skew', 'std', 'nunique']   \n        stats = df.groupby(f'{feature}_bin')[target].agg(aggs).reset_index()\n        stats.columns = [f'{feature}_bin'] + [f'{feature}_{agg}' for agg in aggs]\n\n        bin_edges = df[f'{feature}_bin'].cat.categories\n\n        for i, temp_df in enumerate(df_list):\n            temp_df[f'{feature}_bin'] = pd.cut(temp_df[feature], bins=bin_edges, include_lowest=True)\n            temp_df = temp_df.merge(stats, on=f'{feature}_bin', how='left')\n            temp_df.drop(columns=[f'{feature}_bin'], inplace=True)\n            df_list[i] = temp_df  # update in place\n\n    return df_list\n\n# KFold cross-validation\nprint('### Training LightGBM Model ###')\nfor fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n    print(f\"\\n### Training Fold {fold+1} ###\")\n\n    # Train-valid Split \n    X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n    X_valid, y_valid = X.iloc[valid_idx], y.iloc[valid_idx]  \n    \n    # Impute missing values separately for train/val\n    X_train = pd.DataFrame(preprocessor.fit_transform(X_train), columns = numerical + categorical)\n    X_valid = pd.DataFrame(preprocessor.transform(X_valid), columns = numerical + categorical) \n    X_test = pd.DataFrame(preprocessor.transform(X_test), columns = numerical + categorical) \n    \n    # All numeric features to remain numeric after imputation \n    X_train[numerical] = X_train[numerical].astype(float)\n    X_valid[numerical] = X_valid[numerical].astype(float)\n    X_test[numerical] = X_test[numerical].astype(float) \n    \n    # Reset index before further processing\n    X_train.reset_index(drop=True, inplace=True)\n    X_valid.reset_index(drop=True, inplace=True)\n    X_test.reset_index(drop=True, inplace=True)\n    \n    y_train = y_train.reset_index(drop=True)\n    y_valid = y_valid.reset_index(drop=True)\n    \n    # Add new features (Interactions & Ratio-based)\n    X_train = create_features(X_train)\n    X_valid = create_features(X_valid)  \n    X_test = create_features(X_test)\n\n    # Add group-wise features\n    X_train = group_feature(X_train)\n    X_valid = group_feature(X_valid)  \n    X_test = group_feature(X_test)\n    \n    # Credit_score\n    Xy_train = X_train.copy()\n    Xy_train['premium_amount'] = y_train        \n    aggs = ['mean', 'median', 'min', 'max', 'skew', 'std', 'nunique']   \n    credit_score_stats = Xy_train.groupby('credit_score')['premium_amount'].agg(aggs)\n    credit_score_stats.columns = [f'credit_score_{agg}' for agg in aggs]\n    credit_score_stats.reset_index(inplace=True)   \n    \n    # Merge Aggregated Features into Train, Validation, and Test\n    X_train = X_train.merge(credit_score_stats , on='credit_score', how='left')\n    X_valid = X_valid.merge(credit_score_stats , on='credit_score', how='left')\n    X_test = X_test.merge(credit_score_stats , on='credit_score', how='left')  \n\n    # Average Claims Per Policy Type \n    claims_per_policy_type = Xy_train.groupby('policy_type')['previous_claims'].agg('mean').reset_index()\n    claims_per_policy_type.columns = ['policy_type'] + ['claims_per_policy_type_mean']    \n\n    # Merge Aggregated Features into Train, Validation, and Test\n    X_train = X_train.merge(claims_per_policy_type, on='policy_type', how='left')\n    X_valid = X_valid.merge(claims_per_policy_type, on='policy_type', how='left')\n    X_test = X_test.merge(claims_per_policy_type, on='policy_type', how='left')\n\n    # Risk Profile Index \n    risk_profile_index = Xy_train.groupby(['education_level', 'occupation', 'property_type'])['premium_amount'].agg(aggs).reset_index()\n    risk_profile_index.columns = ['education_level', 'occupation', 'property_type'] + [f'risk_profile_index_{agg}' for agg in aggs]\n    \n    # Merge Aggregated Features into Train, Validation, and Test\n    X_train = X_train.merge(risk_profile_index, on=['education_level', 'occupation', 'property_type'], how='left')\n    X_valid = X_valid.merge(risk_profile_index, on=['education_level', 'occupation', 'property_type'], how='left')\n    X_test = X_test.merge(risk_profile_index, on=['education_level', 'occupation', 'property_type'], how='left')\n       \n    # annual_income & health_score target encoding\n    features = ['health_score', 'annual_income']\n    X_train, X_valid, X_test = add_binned_target_encoding(\n        df=Xy_train,\n        df_list=[X_train, X_valid, X_test],\n        target='premium_amount',\n        features=features)  \n      \n    # Categorical Target Encoding \n    enco_cat = categorical + categorical_2    \n    te = TargetEncoder(cols=enco_cat, smoothing=10)\n    X_train = te.fit_transform(X_train, y_train)\n    X_valid = te.transform(X_valid)\n    X_test = te.transform(X_test)   \n\n    # Fill NaNs in each of these newly aggregated columns with its median (based on train data only)\n    agg_cols = [col for col in X_train.columns if any(\n        stat in col for stat in ['mean', 'median', 'min', 'max', 'skew', 'std', 'nunique'])]  \n\n    # Fill NaNs in each of these columns with its median (based on train data only)\n    for col in agg_cols:  \n      median_val = X_train[col].median()\n      X_train[col].fillna(median_val, inplace=True)\n      X_valid[col].fillna(median_val, inplace=True)\n      X_test[col].fillna(median_val, inplace=True)\n    \n    # Training LightGBM Model\n    model = LGBMRegressor(**params, verbose=-1)\n    model.fit(\n    X_train, y_train,\n    eval_set=[(X_valid, y_valid)],\n    eval_metric='rmse',\n    callbacks=[early_stopping(stopping_rounds=50, verbose=False)])\n\n    # Predict on validation & test set (in log space)\n    y_valid_pred_log = model.predict(X_valid) \n    oof_preds[valid_idx] = y_valid_pred_log\n    test_preds += model.predict(X_test) / kf.n_splits\n\n    # Fold RMSLE\n    fold_rmsle = rmsle(y_valid, y_valid_pred_log)\n    print(f'LightGBM Fold {fold + 1} RMSLE: {fold_rmsle:.5f}')\n    rmsle_scores.append(fold_rmsle)\n\nfinal_oof_rmsle = rmsle(y, oof_preds) \nprint(f'\\nMean Fold-wise LightGBM RMSLE: {np.mean(rmsle_scores):.5f}') \nprint(f'LightGBM OOF RMSLE on full training set: {final_oof_rmsle:.5f}')   \n\nprint(f\"\\nTotal number of features used: {X_train.shape[1]}\")\nprint('Features used:\\n', X_train.columns)                       ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:29:06.864593Z","iopub.execute_input":"2025-09-09T05:29:06.864919Z","iopub.status.idle":"2025-09-09T05:38:57.366995Z","shell.execute_reply.started":"2025-09-09T05:29:06.864894Z","shell.execute_reply":"2025-09-09T05:38:57.366042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## **Feature Transformation and Encoding Strategy**\n\n### **Log Transformation for Skewed Distributions:**\n\nThe target variable, **`premium_amount`**, exhibited a **right-skewed distribution** with **outliers**, which could negatively impact the **Root Mean Squared Logarithmic Error (RMSLE)** metric, as **RMSLE** is not robust to extreme values. To mitigate this, I applied a **log transformation to `premium_amount`** to reduce skewness and stabilize variance.\n\nSimilarly, **`annual_income`** had a wide range with **significant outliers**. Since these values could be genuine, rather than removing them, I log-transformed **`annual_income`** to make the distribution more manageable while preserving all data points. \n\n### **Target Encoding of Categorical and Group-wise Features**\nThe boxplots showed consistent shapes and sizes, with similar or identical means, medians, and interquartile ranges (IQR) across categories, indicating limited variation in the target variable among the categorical groups. This suggests that individual categorical features **offer weak standalone predictive power.**\n\nTo address this, I applied target encoding to both **single categorical** variables and **engineered group-wise combinations**, such as **month + year** and **occupation + gender**. These combinations were created to introduce more granular patterns and uncover interactions that individual features may not capture, potentially reflecting **subtle socio-demographic** or **temporal effects** on premium pricing.\n\nAll target encoding was performed within each fold of cross-validation to prevent data leakage and preserve validation integrity. This approach might help the model learn **category-to-target relationships** more effectively.\n\n### **Target Encoding of Low-Signal Numeric Features:**\n\nWhile features like **`previous_claims`**, **`month`**, **`year`**, and **`number_of_dependents`** individually showed weak correlation with the target variable, **`previous_claims`** displayed a **clear upward trend** in **median premium amount** up to a certain range (specifically for **claims ≤ 4**).\n\nTo retain the original signal and extract additional patterns, I created duplicate columns for these variables and applied target encoding to the copies, preserving the raw versions as well.","metadata":{}},{"cell_type":"code","source":"# Check unique credit score\nprint(train_df['credit_score'].nunique())\n\ntrain_credit = train_df['credit_score'].dropna().unique().tolist()\ntest_credit = test_df['credit_score'].dropna().unique().tolist()\n\nset(train_credit) == set(test_credit)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:38:57.368003Z","iopub.execute_input":"2025-09-09T05:38:57.368267Z","iopub.status.idle":"2025-09-09T05:38:57.416344Z","shell.execute_reply.started":"2025-09-09T05:38:57.368241Z","shell.execute_reply":"2025-09-09T05:38:57.415561Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### **Aggregated Features from Credit Scores:**\nThe **`credit_score`** feature contained approximately **550** unique values, which were consistently present in both the training and test sets, ensuring stable representation across datasets. Despite its **high cardinality**, the large training set of **1.2 million** samples made it viable to leverage this feature for meaningful signal extraction.\n\nTo capture patterns linked to **`credit_score`**, I computed **statistical aggregations** (mean, median, min, max, standard deviation, etc.) of the log-transformed **`premium_amount`**, grouped by **`credit_score`**. These aggregated features were intended to enrich the model's understanding of **credit-related trends** in premium pricing. To prevent data leakage, all aggregations were computed within each fold during cross-validation.\n\n### **Encoding High-Cardinality Numerical Features:**\nFeatures like **`annual_income`** and **`health_score`** exhibited ***too high cardinality*** for direct target encoding to be effective, as this would result in sparse and potentially noisy representations. To overcome this, I **binned these features into 100 percentile-based groups** to reduce cardinality while retaining distributional context. Target encoding and statistical aggregations (mean, median, standard deviation, etc.) of the log-transformed target variable were then computed on these bins within each fold to prevent data leakage.","metadata":{}},{"cell_type":"markdown","source":"# **Feature Importance**","metadata":{}},{"cell_type":"code","source":"# Plot Feature Importance (gain)\nlgb.plot_importance(model.booster_, importance_type = 'gain', figsize=(12, 18))\nplt.title('Feature Importance by Gain');","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:38:57.417385Z","iopub.execute_input":"2025-09-09T05:38:57.417661Z","iopub.status.idle":"2025-09-09T05:38:58.558992Z","shell.execute_reply.started":"2025-09-09T05:38:57.417634Z","shell.execute_reply":"2025-09-09T05:38:58.558128Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **Predicting on the Test Dataset**","metadata":{}},{"cell_type":"code","source":"# Converting log predictions back to original values\ny_test_pred = np.expm1(test_preds)    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:38:58.560054Z","iopub.execute_input":"2025-09-09T05:38:58.560322Z","iopub.status.idle":"2025-09-09T05:38:58.565851Z","shell.execute_reply.started":"2025-09-09T05:38:58.560295Z","shell.execute_reply":"2025-09-09T05:38:58.564672Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ensure no negative predictions\ny_test_pred = np.maximum(y_test_pred, 0)\n\n# Create the submission DataFrame\nsubmission = pd.DataFrame({\n    'id': test_df['id'],\n    'Premium Amount': y_test_pred\n})\n\n# Save the submission file\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Final submission file created\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:38:58.566891Z","iopub.execute_input":"2025-09-09T05:38:58.567108Z","iopub.status.idle":"2025-09-09T05:38:59.917349Z","shell.execute_reply.started":"2025-09-09T05:38:58.567085Z","shell.execute_reply":"2025-09-09T05:38:59.916461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head()   ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:38:59.918448Z","iopub.execute_input":"2025-09-09T05:38:59.91879Z","iopub.status.idle":"2025-09-09T05:38:59.926449Z","shell.execute_reply.started":"2025-09-09T05:38:59.918749Z","shell.execute_reply":"2025-09-09T05:38:59.925547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"credit_counts = train_df['credit_score'].value_counts()\ncredit_counts.describe()  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:39:43.934616Z","iopub.execute_input":"2025-09-09T05:39:43.935452Z","iopub.status.idle":"2025-09-09T05:39:43.958056Z","shell.execute_reply.started":"2025-09-09T05:39:43.935417Z","shell.execute_reply":"2025-09-09T05:39:43.957193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['credit_score'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:40:56.080207Z","iopub.execute_input":"2025-09-09T05:40:56.08085Z","iopub.status.idle":"2025-09-09T05:40:56.098545Z","shell.execute_reply.started":"2025-09-09T05:40:56.080815Z","shell.execute_reply":"2025-09-09T05:40:56.097868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T05:42:10.560065Z","iopub.execute_input":"2025-09-09T05:42:10.560417Z","iopub.status.idle":"2025-09-09T05:42:10.566134Z","shell.execute_reply.started":"2025-09-09T05:42:10.560386Z","shell.execute_reply":"2025-09-09T05:42:10.565249Z"}},"outputs":[],"execution_count":null}]}