{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"colab":{"provenance":[]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Regression with an Insurance Dataset (Regression)

\n\n\n\n---\n\n## Problem Definition\n\n- Domain\n\n * Insurance | Finance\n\n
\n\n- Dataset\n * [Regression with an Insurance Dataset](https://www.kaggle.com/competitions/playground-series-s4e12/data) dataset from Kaggle which contains 19 features explaining an individaul's demographics, professional and educational background and insurance payments related information. Train dataset contains 1,200,0000 samples and each instance represents one person.\n\n
\n\n- Objective\n * The goal of this project is to predict insurance premiums based on various factors.\n\n
\n\n- Algorithms\n * Following regressiion algorithms are used to train models on the train dataset. The models are evaluated using the [Root Mean Squared Logarithmic Error (RMSLE)](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.root_mean_squared_log_error.html) metric.\n\n 1. [XGBoost (Extreme Gradient Boosting)Regressor](https://xgboost.readthedocs.io/en/stable/python/python_api.html#xgboost.XGBRegressor)\n 2. [LightGBMRegressor](https://lightgbm.readthedocs.io/en/latest/pythonapi/lightgbm.LGBMRegressor.html)\n 3. [CatBoostRegressor](https://catboost.ai/docs/en/concepts/python-reference_catboostregressor)\n\n
\n","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom statsmodels.graphics.mosaicplot import mosaic\n%config InlineBackend.figure_format=\"svg\"\n\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline, make_pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\n\nimport optuna\n\nRSEED = 42\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"id":"7V3pNi21BIh0","executionInfo":{"status":"ok","timestamp":1732784962394,"user_tz":-330,"elapsed":304,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:01:39.276896Z","iopub.execute_input":"2024-12-31T20:01:39.277231Z","iopub.status.idle":"2024-12-31T20:01:44.128939Z","shell.execute_reply.started":"2024-12-31T20:01:39.277198Z","shell.execute_reply":"2024-12-31T20:01:44.128026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n","metadata":{"trusted":true,"id":"AD-WZSYxBIh3","executionInfo":{"status":"ok","timestamp":1732784963552,"user_tz":-330,"elapsed":832,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:01:44.131162Z","iopub.execute_input":"2024-12-31T20:01:44.131698Z","iopub.status.idle":"2024-12-31T20:01:55.188736Z","shell.execute_reply.started":"2024-12-31T20:01:44.131663Z","shell.execute_reply":"2024-12-31T20:01:55.187749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()\n","metadata":{"trusted":true,"id":"FYyUhlRoBIh5","executionInfo":{"status":"ok","timestamp":1732784963553,"user_tz":-330,"elapsed":13,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"c996aeb1-b39e-4384-a65e-a3e107400809","execution":{"iopub.status.busy":"2024-12-31T20:01:55.190071Z","iopub.execute_input":"2024-12-31T20:01:55.190401Z","iopub.status.idle":"2024-12-31T20:01:55.232376Z","shell.execute_reply.started":"2024-12-31T20:01:55.19037Z","shell.execute_reply":"2024-12-31T20:01:55.231196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.head()\n","metadata":{"trusted":true,"id":"lL7LZ4gxBIh8","executionInfo":{"status":"ok","timestamp":1732784963906,"user_tz":-330,"elapsed":361,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"f1cc5d94-dc4d-4b9b-e2f6-b22eeb0d521d","execution":{"iopub.status.busy":"2024-12-31T20:01:55.233588Z","iopub.execute_input":"2024-12-31T20:01:55.233899Z","iopub.status.idle":"2024-12-31T20:01:55.255776Z","shell.execute_reply.started":"2024-12-31T20:01:55.233869Z","shell.execute_reply":"2024-12-31T20:01:55.254558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.columns\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:01:55.257161Z","iopub.execute_input":"2024-12-31T20:01:55.25751Z","iopub.status.idle":"2024-12-31T20:01:55.271178Z","shell.execute_reply.started":"2024-12-31T20:01:55.257477Z","shell.execute_reply":"2024-12-31T20:01:55.270082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.drop(columns=[\"id\"], inplace=True)\ntest_data.drop(columns=[\"id\"], inplace=True)\n","metadata":{"trusted":true,"id":"f_x6nvKfBIh9","executionInfo":{"status":"ok","timestamp":1732784963907,"user_tz":-330,"elapsed":18,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:01:55.272319Z","iopub.execute_input":"2024-12-31T20:01:55.272631Z","iopub.status.idle":"2024-12-31T20:01:55.565897Z","shell.execute_reply.started":"2024-12-31T20:01:55.272602Z","shell.execute_reply":"2024-12-31T20:01:55.564676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n# Step 1: Exploratory Data Analysis (EDA)\n\n---","metadata":{"id":"QhL78ojtBIh-"}},{"cell_type":"code","source":"print(f\"Train Data\\nInstance Count: {train_data.shape[0]} \\nFeature Count: {train_data.shape[1]-1}\")\n\ntrain_data.head().style.set_table_attributes(\n 'style=\"overflow-x: auto; display: inline-block;\"'\n ).set_properties(**{'min-width': '50px'})\n","metadata":{"trusted":true,"id":"0BfzJvAdBIh_","executionInfo":{"status":"ok","timestamp":1732784963907,"user_tz":-330,"elapsed":17,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"037aedfe-b7c2-4ec0-c88d-7a22abee292c","execution":{"iopub.status.busy":"2024-12-31T20:01:55.569061Z","iopub.execute_input":"2024-12-31T20:01:55.569478Z","iopub.status.idle":"2024-12-31T20:01:55.629122Z","shell.execute_reply.started":"2024-12-31T20:01:55.569442Z","shell.execute_reply":"2024-12-31T20:01:55.628065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Test Data\\nInstance Count: {test_data.shape[0]} \\nFeature Count: {test_data.shape[1]}\")\n\ntest_data.head().style.set_table_attributes(\n 'style=\"overflow-x: auto; display: inline-block;\"'\n ).set_properties(**{'min-width': '50px'})\n","metadata":{"trusted":true,"id":"oVBD8F-aBIiA","executionInfo":{"status":"ok","timestamp":1732784964260,"user_tz":-330,"elapsed":364,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"bf9c1e2c-a954-40d3-81f2-b32325fb8fb4","execution":{"iopub.status.busy":"2024-12-31T20:01:55.630653Z","iopub.execute_input":"2024-12-31T20:01:55.631467Z","iopub.status.idle":"2024-12-31T20:01:55.647466Z","shell.execute_reply.started":"2024-12-31T20:01:55.631416Z","shell.execute_reply":"2024-12-31T20:01:55.646381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.info()\n","metadata":{"trusted":true,"id":"Q-7-shwZBIiB","executionInfo":{"status":"ok","timestamp":1732784964260,"user_tz":-330,"elapsed":22,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"4da9ff03-f154-43ee-ab71-fd16ca5e2d41","execution":{"iopub.status.busy":"2024-12-31T20:01:55.649068Z","iopub.execute_input":"2024-12-31T20:01:55.64961Z","iopub.status.idle":"2024-12-31T20:01:56.309952Z","shell.execute_reply.started":"2024-12-31T20:01:55.649564Z","shell.execute_reply":"2024-12-31T20:01:56.308824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"missing_values = train_data.isnull().sum().sort_values(ascending=False)\nmissing_values = missing_values[missing_values > 0]\nprint(missing_values)\n","metadata":{"trusted":true,"id":"eCYBg-F1BIiC","executionInfo":{"status":"ok","timestamp":1732784964261,"user_tz":-330,"elapsed":19,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"1436953e-6c7e-49ee-9a6b-2f6aa93624bf","execution":{"iopub.status.busy":"2024-12-31T20:01:56.311401Z","iopub.execute_input":"2024-12-31T20:01:56.311744Z","iopub.status.idle":"2024-12-31T20:01:56.953798Z","shell.execute_reply.started":"2024-12-31T20:01:56.311703Z","shell.execute_reply":"2024-12-31T20:01:56.952504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"duplicates = train_data[train_data.duplicated(keep=False)]\nprint(len(duplicates))\n# print(duplicates)\n","metadata":{"trusted":true,"id":"wlw3VbMKBIiD","executionInfo":{"status":"ok","timestamp":1732784964261,"user_tz":-330,"elapsed":14,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"8b2db0f1-351a-4132-891c-eb94b707f0f2","execution":{"iopub.status.busy":"2024-12-31T20:01:56.955691Z","iopub.execute_input":"2024-12-31T20:01:56.956092Z","iopub.status.idle":"2024-12-31T20:01:58.960763Z","shell.execute_reply.started":"2024-12-31T20:01:56.956057Z","shell.execute_reply":"2024-12-31T20:01:58.95961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.nunique().sort_values(ascending=False)\n","metadata":{"trusted":true,"id":"MVQX9yCCBIiE","executionInfo":{"status":"ok","timestamp":1732784964646,"user_tz":-330,"elapsed":395,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"13790e49-20b0-42e7-aa2e-374a4abecf0a","execution":{"iopub.status.busy":"2024-12-31T20:01:58.962059Z","iopub.execute_input":"2024-12-31T20:01:58.962393Z","iopub.status.idle":"2024-12-31T20:02:00.144178Z","shell.execute_reply.started":"2024-12-31T20:01:58.962361Z","shell.execute_reply":"2024-12-31T20:02:00.14301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = train_data.select_dtypes(include=[\"object\"]).columns.tolist()\nnum_features = train_data.select_dtypes(include=[np.number]).columns.tolist()\n\nprint(f\"Categorical columns:\\n{cat_features}\")\nprint(f\"\\nNumerical columns:\\n{num_features}\")\n","metadata":{"trusted":true,"id":"-fl_l7YUBIiF","executionInfo":{"status":"ok","timestamp":1732784964646,"user_tz":-330,"elapsed":22,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"0bcfba9a-548c-4527-c3a2-52f0dde30933","execution":{"iopub.status.busy":"2024-12-31T20:02:00.145813Z","iopub.execute_input":"2024-12-31T20:02:00.146173Z","iopub.status.idle":"2024-12-31T20:02:00.355298Z","shell.execute_reply.started":"2024-12-31T20:02:00.146138Z","shell.execute_reply":"2024-12-31T20:02:00.354118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_features.remove('Premium Amount')\ntarget = 'Premium Amount'\n","metadata":{"trusted":true,"id":"Xf8TzCOeBIiG","executionInfo":{"status":"ok","timestamp":1732784964647,"user_tz":-330,"elapsed":18,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:02:00.356595Z","iopub.execute_input":"2024-12-31T20:02:00.356937Z","iopub.status.idle":"2024-12-31T20:02:00.361895Z","shell.execute_reply.started":"2024-12-31T20:02:00.356905Z","shell.execute_reply":"2024-12-31T20:02:00.36079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features.remove('Policy Start Date')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:02:00.363104Z","iopub.execute_input":"2024-12-31T20:02:00.363555Z","iopub.status.idle":"2024-12-31T20:02:00.376783Z","shell.execute_reply.started":"2024-12-31T20:02:00.363506Z","shell.execute_reply":"2024-12-31T20:02:00.37544Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in cat_features:\n print(\"Feature:\", col)\n print(\"Missing Value Count:\", train_data[col].isnull().sum())\n print(dict(train_data[col].value_counts()), end='\\n\\n')\n","metadata":{"trusted":true,"id":"4JRq7R8gBIiH","executionInfo":{"status":"ok","timestamp":1732784964647,"user_tz":-330,"elapsed":17,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"3170fa69-a5e1-435a-8033-de4b47a59195","execution":{"iopub.status.busy":"2024-12-31T20:02:00.378109Z","iopub.execute_input":"2024-12-31T20:02:00.378433Z","iopub.status.idle":"2024-12-31T20:02:01.840567Z","shell.execute_reply.started":"2024-12-31T20:02:00.378397Z","shell.execute_reply":"2024-12-31T20:02:01.839352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data[target].describe()\n","metadata":{"trusted":true,"id":"gbHP9--oBIiJ","executionInfo":{"status":"ok","timestamp":1732784965274,"user_tz":-330,"elapsed":40,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"61149ea9-1318-4ce0-e00a-69dac7b24d47","execution":{"iopub.status.busy":"2024-12-31T20:02:01.841955Z","iopub.execute_input":"2024-12-31T20:02:01.842349Z","iopub.status.idle":"2024-12-31T20:02:01.908684Z","shell.execute_reply.started":"2024-12-31T20:02:01.842314Z","shell.execute_reply":"2024-12-31T20:02:01.907557Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 1.1 Statistical Analysis\n\n---","metadata":{"id":"5KlrLPB6BIiK"}},{"cell_type":"code","source":"train_data[num_features].describe().style.set_table_attributes(\n 'style=\"overflow-x: auto; display: inline-block;\"').set_properties(**{'min-width': '100px'})\n","metadata":{"trusted":true,"id":"c4N-zMCcBIiL","executionInfo":{"status":"ok","timestamp":1732784965275,"user_tz":-330,"elapsed":37,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"d1a0c144-1a8a-4857-ce16-9a7784bd9439","execution":{"iopub.status.busy":"2024-12-31T20:02:01.91013Z","iopub.execute_input":"2024-12-31T20:02:01.910555Z","iopub.status.idle":"2024-12-31T20:02:02.557797Z","shell.execute_reply.started":"2024-12-31T20:02:01.910503Z","shell.execute_reply":"2024-12-31T20:02:02.556692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Q1 = train_data[num_features].quantile(0.25)\nQ3 = train_data[num_features].quantile(0.75)\nIQR = Q3 - Q1\n\nlower_bound = Q1 - 1.5 * IQR\nupper_bound = Q3 + 1.5 * IQR\n\noutliers_iqr = ((train_data[num_features] < lower_bound) | (train_data[num_features] > upper_bound))\n\noutliers_count = outliers_iqr.sum()\noutliers_count = outliers_count[outliers_count > 0].sort_values(ascending=False)\n\nprint(f\"Outliers Count: \\n{outliers_count}\")\n","metadata":{"trusted":true,"id":"wtKQxq_oBIiL","executionInfo":{"status":"ok","timestamp":1732784965275,"user_tz":-330,"elapsed":33,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"de6562f3-c058-48af-e561-33e49e15f9d2","execution":{"iopub.status.busy":"2024-12-31T20:02:02.559004Z","iopub.execute_input":"2024-12-31T20:02:02.559308Z","iopub.status.idle":"2024-12-31T20:02:03.090048Z","shell.execute_reply.started":"2024-12-31T20:02:02.559278Z","shell.execute_reply":"2024-12-31T20:02:03.087566Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 1.2 Data Visualization\n\n---","metadata":{"id":"Kq4tQceMBIiM"}},{"cell_type":"markdown","source":"---\n\n## 1.2.1 Univariate Analysis¶\n\n---","metadata":{"id":"j1w1GjvRBIiM"}},{"cell_type":"code","source":"len(cat_features)\n","metadata":{"trusted":true,"id":"s3uo38zPBIiO","executionInfo":{"status":"ok","timestamp":1732784965699,"user_tz":-330,"elapsed":10,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"28f34cab-de52-4384-ce86-50edeac1d949","execution":{"iopub.status.busy":"2024-12-31T20:02:03.091949Z","iopub.execute_input":"2024-12-31T20:02:03.092527Z","iopub.status.idle":"2024-12-31T20:02:03.09971Z","shell.execute_reply.started":"2024-12-31T20:02:03.092475Z","shell.execute_reply":"2024-12-31T20:02:03.098466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_cols_cat = 2\nn_rows_cat = 5\n","metadata":{"trusted":true,"id":"N3i_QKbhBIiP","executionInfo":{"status":"ok","timestamp":1732784965699,"user_tz":-330,"elapsed":8,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:02:03.101439Z","iopub.execute_input":"2024-12-31T20:02:03.10184Z","iopub.status.idle":"2024-12-31T20:02:03.110578Z","shell.execute_reply.started":"2024-12-31T20:02:03.101805Z","shell.execute_reply":"2024-12-31T20:02:03.10925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(n_rows_cat, n_cols_cat, figsize=(10, 4*n_rows_cat))\naxes = axes.flatten()\n\nfor i, column in enumerate(train_data[cat_features].columns):\n ax = axes[i]\n category_counts = train_data[cat_features][column].value_counts()\n sns.barplot(x=category_counts.index, y=category_counts.values, ax=ax, palette=\"crest\")\n ax.set_xticklabels(category_counts.index, rotation=45)\n\nplt.suptitle(\"Bar Charts - Insurance Dataset\", fontsize=20, y=1.0)\n\nplt.tight_layout()\nplt.savefig(\"Categorical Feature Analysis - Bar Charts.svg\")\nplt.show()\n","metadata":{"trusted":true,"id":"KGA1u8vtBIiP","executionInfo":{"status":"ok","timestamp":1732784968580,"user_tz":-330,"elapsed":2888,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"b0ec3700-9aad-432e-fe1f-deeb766867be","execution":{"iopub.status.busy":"2024-12-31T20:02:03.116094Z","iopub.execute_input":"2024-12-31T20:02:03.116548Z","iopub.status.idle":"2024-12-31T20:02:08.150391Z","shell.execute_reply.started":"2024-12-31T20:02:03.116513Z","shell.execute_reply":"2024-12-31T20:02:08.149163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(n_rows_cat, n_cols_cat, figsize=(10, 5*n_rows_cat))\naxes = axes.flatten()\n\nfor i, column in enumerate(train_data[cat_features].columns):\n ax = axes[i]\n category_counts = train_data[cat_features][column].value_counts()\n ax.pie(\n category_counts.values,\n labels=category_counts.index,\n autopct='%1.1f%%',\n startangle=45,\n colors=sns.color_palette('crest'),\n labeldistance=1.1,\n pctdistance=0.80\n )\n ax.set_title(column)\n\nplt.suptitle('Pie Charts - Insurance Dataset', fontsize=20, y=1.0)\n\nplt.tight_layout()\nplt.savefig(\"Categorical Feature Analysis - Pie Charts.svg\")\nplt.show()\n","metadata":{"trusted":true,"id":"V5bcIDGdBIiQ","executionInfo":{"status":"ok","timestamp":1732784970065,"user_tz":-330,"elapsed":1494,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"c10bbbb8-90e7-4f8f-ecfb-b776cecc97f5","execution":{"iopub.status.busy":"2024-12-31T20:02:08.151715Z","iopub.execute_input":"2024-12-31T20:02:08.152062Z","iopub.status.idle":"2024-12-31T20:02:11.916348Z","shell.execute_reply.started":"2024-12-31T20:02:08.152029Z","shell.execute_reply":"2024-12-31T20:02:11.915336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(num_features)\n","metadata":{"trusted":true,"id":"aHFt5XaKBIiR","executionInfo":{"status":"ok","timestamp":1732784970065,"user_tz":-330,"elapsed":23,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"949089df-f593-4684-c992-fcc8ad5af355","execution":{"iopub.status.busy":"2024-12-31T20:02:11.917553Z","iopub.execute_input":"2024-12-31T20:02:11.918023Z","iopub.status.idle":"2024-12-31T20:02:11.926326Z","shell.execute_reply.started":"2024-12-31T20:02:11.917951Z","shell.execute_reply":"2024-12-31T20:02:11.925126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_cols_num = 2\nn_rows_num = 4\n","metadata":{"trusted":true,"id":"vQ27HLa-BIiR","executionInfo":{"status":"ok","timestamp":1732784970066,"user_tz":-330,"elapsed":17,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:02:11.927753Z","iopub.execute_input":"2024-12-31T20:02:11.928206Z","iopub.status.idle":"2024-12-31T20:02:11.937748Z","shell.execute_reply.started":"2024-12-31T20:02:11.928158Z","shell.execute_reply":"2024-12-31T20:02:11.936737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\nfig, axes = plt.subplots(n_rows_num, n_cols_num, figsize=(10, 3*n_rows_num))\naxes = axes.flatten()\n\nfor i, col in enumerate(train_data[num_features].columns):\n ax = axes[i]\n sns.histplot(train_data[num_features][col], ax=ax, color='seagreen', stat='frequency', bins=20, kde=True)\n ax.set_ylabel('Frequency')\n\nfor j in range(i + 1, len(axes)):\n fig.delaxes(axes[j])\n\nplt.suptitle('Histograms - Insurance Dataset', fontsize=20, y=1.0)\n\nplt.tight_layout()\nplt.savefig('Numerical Feature Analysis - Histograms.svg')\nplt.show()\n","metadata":{"trusted":true,"id":"ps3StJq8BIiS","executionInfo":{"status":"ok","timestamp":1732784972473,"user_tz":-330,"elapsed":2422,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"6c90eea0-b845-4265-f56e-cb98a5699184","execution":{"iopub.status.busy":"2024-12-31T20:02:11.939374Z","iopub.execute_input":"2024-12-31T20:02:11.939829Z","iopub.status.idle":"2024-12-31T20:02:53.432205Z","shell.execute_reply.started":"2024-12-31T20:02:11.939781Z","shell.execute_reply":"2024-12-31T20:02:53.43106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, axes = plt.subplots(n_rows_num, n_cols_num, figsize=(10, 2*n_rows_num))\naxes = axes.flatten()\n\nfor i, col in enumerate(train_data[num_features].columns):\n ax = axes[i]\n sns.boxplot(x=train_data[num_features][col], ax=ax, color='seagreen')\n ax.set_xlabel(col)\n ax.set_ylabel('Value')\n\nfor j in range(i + 1, len(axes)):\n fig.delaxes(axes[j])\n\nplt.suptitle('Box Plots - Insurance Dataset', fontsize=20, y=1.0)\n\nplt.tight_layout()\nplt.savefig('Numerical Feature Analysis - Box Plots.svg')\nplt.show()\n","metadata":{"trusted":true,"id":"xijwjas7BIiT","executionInfo":{"status":"ok","timestamp":1732784974061,"user_tz":-330,"elapsed":1602,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"875bc751-997a-4d9b-b961-804e44138ad5","execution":{"iopub.status.busy":"2024-12-31T20:02:53.433527Z","iopub.execute_input":"2024-12-31T20:02:53.433861Z","iopub.status.idle":"2024-12-31T20:02:59.134013Z","shell.execute_reply.started":"2024-12-31T20:02:53.433829Z","shell.execute_reply":"2024-12-31T20:02:59.132853Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n## 1.2.2 Bivariate Analysis\n\n---","metadata":{"id":"CbQC4QfIBIiU"}},{"cell_type":"code","source":"corr = train_data[num_features].corr()\ncmap = sns.light_palette(\"darkgreen\", as_cmap=True)\n\nplt.figure(figsize=(8,6))\nsns.heatmap(corr, annot=True, cmap=cmap, linewidths=0.2)\n\nplt.title('Correlation Matrix - Insurance Dataset', fontsize=20, y=1.1)\n\nplt.tight_layout()\nplt.savefig('Numerical Features Correlation Analysis - Corr Matrix.svg')\nplt.show()\n","metadata":{"trusted":true,"id":"2cJm8kvJBIiW","executionInfo":{"status":"ok","timestamp":1732784983298,"user_tz":-330,"elapsed":443,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"036e7427-ef86-4fb6-82e8-82c171142985","execution":{"iopub.status.busy":"2024-12-31T20:02:59.135277Z","iopub.execute_input":"2024-12-31T20:02:59.135615Z","iopub.status.idle":"2024-12-31T20:03:00.517341Z","shell.execute_reply.started":"2024-12-31T20:02:59.135581Z","shell.execute_reply":"2024-12-31T20:03:00.51572Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n# Step 2: Data Preprocessing\n\n---","metadata":{"id":"3uqAhZP5BIiY"}},{"cell_type":"code","source":"train_data['Policy Start Date'] = pd.to_datetime(train_data['Policy Start Date'])\ntrain_data['Policy Start Date']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:00.519455Z","iopub.execute_input":"2024-12-31T20:03:00.520544Z","iopub.status.idle":"2024-12-31T20:03:00.988486Z","shell.execute_reply.started":"2024-12-31T20:03:00.520458Z","shell.execute_reply":"2024-12-31T20:03:00.987376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# cat_features\n# num_features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:00.989934Z","iopub.execute_input":"2024-12-31T20:03:00.990394Z","iopub.status.idle":"2024-12-31T20:03:00.995519Z","shell.execute_reply.started":"2024-12-31T20:03:00.990339Z","shell.execute_reply":"2024-12-31T20:03:00.994178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessor = ColumnTransformer(\n transformers=[\n ('numerical', Pipeline(steps=[\n ('imputer', SimpleImputer(strategy='median')),\n ('scaler', StandardScaler())\n ]), num_features),\n\n ('categorical', Pipeline(steps=[\n ('imputer', SimpleImputer(strategy='most_frequent')),\n ('encoder', OneHotEncoder(drop='first', handle_unknown='ignore')),\n ]), cat_features)\n ],\n remainder='passthrough'\n)\n\npreprocessor\n","metadata":{"trusted":true,"id":"ai-kkBcWBIib","executionInfo":{"status":"ok","timestamp":1732788707728,"user_tz":-330,"elapsed":350,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"outputId":"729f1073-bb21-4350-f794-692a2df0376b","execution":{"iopub.status.busy":"2024-12-31T20:03:00.996787Z","iopub.execute_input":"2024-12-31T20:03:00.997129Z","iopub.status.idle":"2024-12-31T20:03:01.037098Z","shell.execute_reply.started":"2024-12-31T20:03:00.997096Z","shell.execute_reply":"2024-12-31T20:03:01.035844Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train_data.drop(columns=[target, 'Policy Start Date'])\ny = train_data[target]\nX.shape, y.shape\n","metadata":{"trusted":true,"id":"hecUxQjWBIid","executionInfo":{"status":"ok","timestamp":1732788995162,"user_tz":-330,"elapsed":425,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:03:01.038482Z","iopub.execute_input":"2024-12-31T20:03:01.03884Z","iopub.status.idle":"2024-12-31T20:03:01.129258Z","shell.execute_reply.started":"2024-12-31T20:03:01.038807Z","shell.execute_reply":"2024-12-31T20:03:01.128153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:01.130566Z","iopub.execute_input":"2024-12-31T20:03:01.130894Z","iopub.status.idle":"2024-12-31T20:03:01.151519Z","shell.execute_reply.started":"2024-12-31T20:03:01.130855Z","shell.execute_reply":"2024-12-31T20:03:01.150528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=RSEED)\n","metadata":{"trusted":true,"id":"aW-sbx2cBIid","executionInfo":{"status":"ok","timestamp":1732789014207,"user_tz":-330,"elapsed":638,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:03:01.1528Z","iopub.execute_input":"2024-12-31T20:03:01.153144Z","iopub.status.idle":"2024-12-31T20:03:01.747156Z","shell.execute_reply.started":"2024-12-31T20:03:01.153112Z","shell.execute_reply":"2024-12-31T20:03:01.745951Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n# Step 3: Model Training & Evaluation\n\n---","metadata":{"id":"5MeXnWfSBIie"}},{"cell_type":"code","source":"\ndef root_mean_squared_log_error(y_true, y_pred):\n \n log_true = np.log1p(y_true)\n log_pred = np.log1p(y_pred)\n \n squared_diff = (log_true - log_pred) ** 2\n \n return np.sqrt(np.mean(squared_diff))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:01.748614Z","iopub.execute_input":"2024-12-31T20:03:01.74909Z","iopub.status.idle":"2024-12-31T20:03:01.755805Z","shell.execute_reply.started":"2024-12-31T20:03:01.74904Z","shell.execute_reply":"2024-12-31T20:03:01.754419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# models = {\n# 'regression_model': LinearRegression(), # 1.1695\n# 'XGBoost': XGBRegressor(random_state=RSEED), # 1.1495\n# 'LGBM': LGBMRegressor(random_state=RSEED), # 1.1495\n# 'CatBoost': CatBoostRegressor(random_state=RSEED) # 1.1487\n# }\n","metadata":{"trusted":true,"id":"KGfbjo4NBIif","executionInfo":{"status":"ok","timestamp":1732789422323,"user_tz":-330,"elapsed":388,"user":{"displayName":"Danushika Herath","userId":"13496251029135326221"}},"execution":{"iopub.status.busy":"2024-12-31T20:03:01.757344Z","iopub.execute_input":"2024-12-31T20:03:01.7578Z","iopub.status.idle":"2024-12-31T20:03:01.768413Z","shell.execute_reply.started":"2024-12-31T20:03:01.757752Z","shell.execute_reply":"2024-12-31T20:03:01.767144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# def train_models(X_train, X_val, y_train, y_val, models, preprocessor):\n# results = {}\n# pipelines = {}\n \n# for name, model in models.items():\n# pipeline = make_pipeline(preprocessor, model)\n\n# pipeline.fit(X_train, y_train)\n \n# y_pred = pipeline.predict(X_val)\n \n# score = root_mean_squared_log_error(y_val, y_pred)\n \n# print(name, score)\n \n# results[name] = score\n# pipelines[name] = pipeline\n \n# return pipelines, results\n\n\n# pipelines, scores = train_models(X_train, X_val, y_train, y_val, models, preprocessor)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:01.77026Z","iopub.execute_input":"2024-12-31T20:03:01.770595Z","iopub.status.idle":"2024-12-31T20:03:01.782532Z","shell.execute_reply.started":"2024-12-31T20:03:01.77056Z","shell.execute_reply":"2024-12-31T20:03:01.781309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# def objective(trial, X_train, X_val, y_train, y_val, preprocessor):\n# param = {\n# 'objective': 'RMSE',\n# 'eval_metric': 'RMSE',\n# 'max_depth': trial.suggest_int('max_depth', 3, 15),\n# 'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),\n# 'colsample_bylevel': trial.suggest_uniform('colsample_bylevel', 0.5, 1.0),\n# 'min_child_samples': trial.suggest_int('min_child_samples', 1, 10),\n# 'learning_rate': trial.suggest_loguniform('learning_rate', 1e-4, 0.1),\n# 'n_estimators': 300,\n# 'random_state': 42,\n# 'thread_count': -1\n# }\n \n# model = CatBoostRegressor(**param)\n# pipeline = make_pipeline(preprocessor, model)\n# pipeline.fit(X_train, y_train)\n \n# y_pred = pipeline.predict(X_val)\n# score = mean_squared_error(y_val, y_pred)\n \n# return score\n\n\n# def tune_catboost(X_train, X_val, y_train, y_val, preprocessor, n_trials=30):\n# study = optuna.create_study(direction='minimize')\n# study.optimize(lambda trial: objective(trial, X_train, X_val, y_train, y_val, preprocessor), n_trials=n_trials)\n \n# print(f'Best trial: {study.best_trial.params}')\n\n# best_model_params = study.best_trial.params\n\n# return best_model_params\n\n\n# best_model_params = tune_catboost(X_train, X_val, y_train, y_val, preprocessor)\n# best_model_params\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:01.784303Z","iopub.execute_input":"2024-12-31T20:03:01.784763Z","iopub.status.idle":"2024-12-31T20:03:01.801324Z","shell.execute_reply.started":"2024-12-31T20:03:01.784715Z","shell.execute_reply":"2024-12-31T20:03:01.800067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model_params = {\n 'objective': 'RMSE',\n 'eval_metric': 'RMSE',\n 'max_depth': 10,\n 'subsample': 0.9634787862433333,\n 'colsample_bylevel': 0.9158885547943915,\n 'min_child_samples': 4,\n 'learning_rate': 0.05713686350188332,\n 'n_estimators': 300,\n 'random_state': 42,\n 'thread_count': -1\n}\n\nbest_model = CatBoostRegressor(**best_model_params)\n\nbest_catboost_model = make_pipeline(preprocessor, best_model)\nbest_catboost_model.fit(X_train, y_train)\n\ny_pred = best_catboost_model.predict(X_val)\nscore = root_mean_squared_log_error(y_val, y_pred)\nscore\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:03:01.802735Z","iopub.execute_input":"2024-12-31T20:03:01.803196Z","iopub.status.idle":"2024-12-31T20:04:04.545229Z","shell.execute_reply.started":"2024-12-31T20:03:01.803149Z","shell.execute_reply":"2024-12-31T20:04:04.543847Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"---\n\n# Step 4: Test Data Prediction\n\n---","metadata":{"id":"703SiBxhBIig"}},{"cell_type":"code","source":"test_data_predictions = best_catboost_model.predict(test_data)\ntest_data_predictions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:04:04.546578Z","iopub.execute_input":"2024-12-31T20:04:04.547048Z","iopub.status.idle":"2024-12-31T20:04:14.789621Z","shell.execute_reply.started":"2024-12-31T20:04:04.546998Z","shell.execute_reply":"2024-12-31T20:04:14.788542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsubmission[target] = test_data_predictions\n\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:04:14.791046Z","iopub.execute_input":"2024-12-31T20:04:14.791392Z","iopub.status.idle":"2024-12-31T20:04:16.792178Z","shell.execute_reply.started":"2024-12-31T20:04:14.791358Z","shell.execute_reply":"2024-12-31T20:04:16.791099Z"}},"outputs":[],"execution_count":null}]}