{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:26.84427Z","iopub.execute_input":"2024-12-09T17:23:26.844601Z","iopub.status.idle":"2024-12-09T17:23:27.185784Z","shell.execute_reply.started":"2024-12-09T17:23:26.84457Z","shell.execute_reply":"2024-12-09T17:23:27.184902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Installing XGBoost","metadata":{}},{"cell_type":"code","source":"!pip install xgbtune","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:27.187495Z","iopub.execute_input":"2024-12-09T17:23:27.187977Z","iopub.status.idle":"2024-12-09T17:23:38.60115Z","shell.execute_reply.started":"2024-12-09T17:23:27.187936Z","shell.execute_reply":"2024-12-09T17:23:38.599936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#Importing Necessary Libraries","metadata":{}},{"cell_type":"markdown","source":"# Importing Necessary Libraries","metadata":{}},{"cell_type":"code","source":"# Import core libraries\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport os\n\n# Import machine learning tools\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import mean_squared_log_error\n\n# Import XGBoost and optimization tools\nimport xgboost as xgb\nimport optuna\n\n# Ignore warnings for a cleaner output\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:38.602525Z","iopub.execute_input":"2024-12-09T17:23:38.602824Z","iopub.status.idle":"2024-12-09T17:23:39.834359Z","shell.execute_reply.started":"2024-12-09T17:23:38.602794Z","shell.execute_reply":"2024-12-09T17:23:39.833634Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"# Load datasets\ntrain = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\n# Drop the 'id' column as it is not required for training\ntrain.drop('id', axis=1, inplace=True)\ntest.drop('id', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:39.836115Z","iopub.execute_input":"2024-12-09T17:23:39.836507Z","iopub.status.idle":"2024-12-09T17:23:49.64926Z","shell.execute_reply.started":"2024-12-09T17:23:39.836478Z","shell.execute_reply":"2024-12-09T17:23:49.648223Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Explore and Understand the Data","metadata":{}},{"cell_type":"code","source":"# Display the first few rows of the training dataset\nprint(\"Training Data Overview:\")\ndisplay(train.head())\n\n# Display the first few rows of the test dataset\nprint(\"Test Data Overview:\")\ndisplay(test.head())\n\n# Summary of the training data\nprint(\"Training Data Summary:\")\ntrain.info()\n\n# Check for missing values\nprint(\"Missing Values in Training Data:\")\ntrain.isnull().sum()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:49.650665Z","iopub.execute_input":"2024-12-09T17:23:49.651039Z","iopub.status.idle":"2024-12-09T17:23:50.75912Z","shell.execute_reply.started":"2024-12-09T17:23:49.651Z","shell.execute_reply":"2024-12-09T17:23:50.758316Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering - Extract Date Features","metadata":{}},{"cell_type":"code","source":"# Function to extract date-related features\ndef date_features(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Quarter'] = df['Policy Start Date'].dt.quarter\n    df['Day of Week'] = df['Policy Start Date'].dt.dayofweek\n    df.drop('Policy Start Date', axis=1, inplace=True)\n    return df\n\n# Apply the function to train and test datasets\ntrain = date_features(train)\ntest = date_features(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:50.760256Z","iopub.execute_input":"2024-12-09T17:23:50.760632Z","iopub.status.idle":"2024-12-09T17:23:52.028522Z","shell.execute_reply.started":"2024-12-09T17:23:50.760604Z","shell.execute_reply":"2024-12-09T17:23:52.027759Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering - Create Interaction Features","metadata":{}},{"cell_type":"code","source":"# Feature Engineering\ntrain['Log_Annual_Income'] = np.log1p(train['Annual Income'])\ntest['Log_Annual_Income'] = np.log1p(test['Annual Income'])\n\ntrain['Health_Score_Age_Ratio'] = train['Health Score'] / train['Age']\ntest['Health_Score_Age_Ratio'] = test['Health Score'] / test['Age']\n\ntrain['Vehicle_Age_Per_Insurance'] = train['Vehicle Age'] / train['Insurance Duration']\ntest['Vehicle_Age_Per_Insurance'] = test['Vehicle Age'] / test['Insurance Duration']\n\ntrain['Income_Per_Credit'] = train['Annual Income'] / train['Credit Score']\ntest['Income_Per_Credit'] = test['Annual Income'] / test['Credit Score']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:52.029652Z","iopub.execute_input":"2024-12-09T17:23:52.029922Z","iopub.status.idle":"2024-12-09T17:23:52.081628Z","shell.execute_reply.started":"2024-12-09T17:23:52.029896Z","shell.execute_reply":"2024-12-09T17:23:52.080948Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering - Categorical Transformation","metadata":{}},{"cell_type":"code","source":"# Combine location and property type into a new feature\ntrain['Property Location Type'] = train['Location'] + '_' + train['Property Type']\ntest['Property Location Type'] = test['Location'] + '_' + test['Property Type']\n\n# Drop the original 'Property Type' feature\ntrain.drop('Property Type', axis=1, inplace=True)\ntest.drop('Property Type', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:52.082613Z","iopub.execute_input":"2024-12-09T17:23:52.082871Z","iopub.status.idle":"2024-12-09T17:23:52.79244Z","shell.execute_reply.started":"2024-12-09T17:23:52.082846Z","shell.execute_reply":"2024-12-09T17:23:52.791587Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Transformation - Log Transformation of Skewed Features","metadata":{}},{"cell_type":"code","source":"# Apply log transformation to skewed numerical features\nfor col in ['Annual Income', 'Vehicle Age', 'Health Score', 'Credit Score']:\n    train[col] = np.log1p(train[col])\n    test[col] = np.log1p(test[col])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:52.793453Z","iopub.execute_input":"2024-12-09T17:23:52.793736Z","iopub.status.idle":"2024-12-09T17:23:52.848185Z","shell.execute_reply.started":"2024-12-09T17:23:52.793709Z","shell.execute_reply":"2024-12-09T17:23:52.847219Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing - Memory Optimization","metadata":{}},{"cell_type":"code","source":"# Function to reduce memory usage\ndef reduce_memory_usage(df):\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type == 'float64':\n            df[col] = df[col].astype('float32')\n        elif col_type == 'int64':\n            df[col] = df[col].astype('int32')\n    return df\n\n# Apply the function to both train and test datasets\ntrain = reduce_memory_usage(train)\ntest = reduce_memory_usage(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:52.850744Z","iopub.execute_input":"2024-12-09T17:23:52.851025Z","iopub.status.idle":"2024-12-09T17:23:52.901308Z","shell.execute_reply.started":"2024-12-09T17:23:52.850997Z","shell.execute_reply":"2024-12-09T17:23:52.900602Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Preprocessing - Define Target and Preprocessing Pipeline","metadata":{}},{"cell_type":"code","source":"# Define the target variable\ntarget = 'Premium Amount'\n\n# Identify numerical and categorical columns\nnumerical_cols = train.select_dtypes(include=['float32', 'int32']).columns.tolist()\nif target in numerical_cols:\n    numerical_cols.remove(target)\n\ncategorical_cols = train.select_dtypes(include=['object']).columns.tolist()\n\n# Define a preprocessing pipeline\npreprocessing = ColumnTransformer([\n    ('num', make_pipeline(SimpleImputer(strategy='mean'), StandardScaler()), numerical_cols),\n    ('cat', make_pipeline(SimpleImputer(strategy='constant', fill_value='unknown'),\n                          OneHotEncoder(handle_unknown='ignore')), categorical_cols)\n], remainder='drop')\n\n# Prepare training and test data\nX_train = train.drop(columns=[target])\ny_train = np.log1p(train[target])  # Log transform target for RMSLE\nX_test = test.copy()\n\nX_train_preprocessed = preprocessing.fit_transform(X_train)\nX_test_preprocessed = preprocessing.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:23:52.902397Z","iopub.execute_input":"2024-12-09T17:23:52.902745Z","iopub.status.idle":"2024-12-09T17:24:02.058125Z","shell.execute_reply.started":"2024-12-09T17:23:52.90271Z","shell.execute_reply":"2024-12-09T17:24:02.0574Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hyperparameter Tuning using Optuna","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    params = {\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.2),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n        'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n        'lambda': trial.suggest_float('lambda', 0.01, 1.0),\n        'alpha': trial.suggest_float('alpha', 0.01, 1.0),\n        'eval_metric': 'rmsle',\n        'tree_method': 'hist'\n    }\n    \n    dtrain = xgb.DMatrix(X_train_preprocessed, label=y_train)\n    dval = xgb.DMatrix(X_val_preprocessed, label=y_val)\n    \n    results = {}\n    model = xgb.train(\n        params, dtrain, num_boost_round=500, evals=[(dval, 'validation')],\n        early_stopping_rounds=20, evals_result=results, verbose_eval=False\n    )\n    \n    y_pred_val = model.predict(dval)\n    score = mean_squared_log_error(np.expm1(y_val), np.expm1(y_pred_val), squared=False)\n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:24:02.059396Z","iopub.execute_input":"2024-12-09T17:24:02.059747Z","iopub.status.idle":"2024-12-09T17:24:02.067026Z","shell.execute_reply.started":"2024-12-09T17:24:02.05971Z","shell.execute_reply":"2024-12-09T17:24:02.066303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Train-Validation Split\nX = train.drop(columns=['Premium Amount'])\ny = np.log1p(train['Premium Amount'])\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\nX_train_preprocessed = preprocessing.fit_transform(X_train)\nX_val_preprocessed = preprocessing.transform(X_val)\nX_test_preprocessed = preprocessing.transform(test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:24:02.068268Z","iopub.execute_input":"2024-12-09T17:24:02.068601Z","iopub.status.idle":"2024-12-09T17:24:11.982531Z","shell.execute_reply.started":"2024-12-09T17:24:02.068563Z","shell.execute_reply":"2024-12-09T17:24:11.98164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Optuna Study\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:24:11.983546Z","iopub.execute_input":"2024-12-09T17:24:11.983801Z","iopub.status.idle":"2024-12-09T17:48:34.570669Z","shell.execute_reply.started":"2024-12-09T17:24:11.983778Z","shell.execute_reply":"2024-12-09T17:48:34.569829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\nprint(\"Best Parameters:\", best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:49:29.175116Z","iopub.execute_input":"2024-12-09T17:49:29.175494Z","iopub.status.idle":"2024-12-09T17:49:29.180637Z","shell.execute_reply.started":"2024-12-09T17:49:29.175462Z","shell.execute_reply":"2024-12-09T17:49:29.17979Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train the Final Model","metadata":{}},{"cell_type":"code","source":"# Train Final Model\ndtrain = xgb.DMatrix(X_train_preprocessed, label=y_train)\nfinal_model = xgb.train(best_params, dtrain, num_boost_round=study.best_trial.number)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:49:32.412336Z","iopub.execute_input":"2024-12-09T17:49:32.412685Z","iopub.status.idle":"2024-12-09T17:49:36.024961Z","shell.execute_reply.started":"2024-12-09T17:49:32.412654Z","shell.execute_reply":"2024-12-09T17:49:36.024263Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Make Predictions and Create Submission File","metadata":{}},{"cell_type":"code","source":"test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:50:26.978076Z","iopub.execute_input":"2024-12-09T17:50:26.978437Z","iopub.status.idle":"2024-12-09T17:50:27.337735Z","shell.execute_reply.started":"2024-12-09T17:50:26.978406Z","shell.execute_reply":"2024-12-09T17:50:27.336827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict on Test Set\ndtest = xgb.DMatrix(X_test_preprocessed)\ny_pred_test = final_model.predict(dtest)\ny_pred_final = np.expm1(y_pred_test)\n\n# Prepare the submission file\nsub = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\noutput = pd.DataFrame({\"id\": sub.id, \"Premium Amount\": y_pred_final})\noutput.to_csv('improved_submission.csv', index=False)\n\n# Display the first few rows of the submission file\noutput.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T17:52:01.23779Z","iopub.execute_input":"2024-12-09T17:52:01.238149Z","iopub.status.idle":"2024-12-09T17:52:03.220094Z","shell.execute_reply.started":"2024-12-09T17:52:01.238118Z","shell.execute_reply":"2024-12-09T17:52:03.219197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}