{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.14"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":649.226298,"end_time":"2024-12-08T17:44:17.194139","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-12-08T17:33:27.967841","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"66595eb8","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 1. Import Libraries </b></div>","metadata":{"papermill":{"duration":0.009189,"end_time":"2024-12-08T17:33:30.57313","exception":false,"start_time":"2024-12-08T17:33:30.563941","status":"completed"},"tags":[]}},{"id":"8cd7d7dc","cell_type":"code","source":"import numpy as np, pandas as pd\nimport category_encoders as ce\nimport lightgbm as lgb, xgboost as xgb\nfrom sklearn.model_selection import cross_val_score, KFold\nfrom sklearn.metrics import mean_squared_log_error, make_scorer, mean_squared_error\nimport matplotlib.pyplot as plt, seaborn as sns\nfrom scipy.signal import find_peaks\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import HuberRegressor\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:23.089973Z","iopub.execute_input":"2024-12-09T14:27:23.09039Z","iopub.status.idle":"2024-12-09T14:27:27.549957Z","shell.execute_reply.started":"2024-12-09T14:27:23.090356Z","shell.execute_reply":"2024-12-09T14:27:27.549273Z"},"papermill":{"duration":5.855321,"end_time":"2024-12-08T17:33:36.437158","exception":false,"start_time":"2024-12-08T17:33:30.581837","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"78130472","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 2. Import and Examine Data </b></div>","metadata":{"papermill":{"duration":0.00957,"end_time":"2024-12-08T17:33:36.464655","exception":false,"start_time":"2024-12-08T17:33:36.455085","status":"completed"},"tags":[]}},{"id":"11f18884","cell_type":"code","source":"base_path = '../input/playground-series-s4e12/'\n\ntrain = pd.read_csv(base_path + 'train.csv', index_col = 'id')\ntest = pd.read_csv(base_path + 'test.csv', index_col = 'id')\nsubmission = pd.read_csv(base_path + 'sample_submission.csv', index_col = 'id')","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:27.551414Z","iopub.execute_input":"2024-12-09T14:27:27.55184Z","iopub.status.idle":"2024-12-09T14:27:36.243679Z","shell.execute_reply.started":"2024-12-09T14:27:27.551813Z","shell.execute_reply":"2024-12-09T14:27:36.24288Z"},"papermill":{"duration":8.992622,"end_time":"2024-12-08T17:33:45.466464","exception":false,"start_time":"2024-12-08T17:33:36.473842","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8636ef41","cell_type":"code","source":"print(f\"There are {train.shape[1]} columns and {train.shape[0]} rows in the train dataset.\")\nprint(f\"There are {test.shape[1]} columns and {test.shape[0]} rows in the test dataset.\")\nprint(f\"There are {submission.shape[0]} rows in the submission file.\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:36.244986Z","iopub.execute_input":"2024-12-09T14:27:36.245384Z","iopub.status.idle":"2024-12-09T14:27:36.251002Z","shell.execute_reply.started":"2024-12-09T14:27:36.245346Z","shell.execute_reply":"2024-12-09T14:27:36.250133Z"},"papermill":{"duration":0.017485,"end_time":"2024-12-08T17:33:45.494275","exception":false,"start_time":"2024-12-08T17:33:45.47679","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"cd2b97dc","cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:36.253378Z","iopub.execute_input":"2024-12-09T14:27:36.254099Z","iopub.status.idle":"2024-12-09T14:27:36.810499Z","shell.execute_reply.started":"2024-12-09T14:27:36.254036Z","shell.execute_reply":"2024-12-09T14:27:36.809577Z"},"papermill":{"duration":0.615993,"end_time":"2024-12-08T17:33:46.120021","exception":false,"start_time":"2024-12-08T17:33:45.504028","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b2f5a2ec","cell_type":"markdown","source":"Data types look okay apart from Policy Start Date which should be datetime.","metadata":{"execution":{"iopub.execute_input":"2024-12-01T14:07:32.124286Z","iopub.status.busy":"2024-12-01T14:07:32.123886Z","iopub.status.idle":"2024-12-01T14:07:32.132207Z","shell.execute_reply":"2024-12-01T14:07:32.130361Z","shell.execute_reply.started":"2024-12-01T14:07:32.12425Z"},"papermill":{"duration":0.009405,"end_time":"2024-12-08T17:33:46.139209","exception":false,"start_time":"2024-12-08T17:33:46.129804","status":"completed"},"tags":[]}},{"id":"30210df7","cell_type":"code","source":"for df in [train, test]:\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    \nprint(\"Policy Start Date converted to Datetime in train and test data\")","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:36.811674Z","iopub.execute_input":"2024-12-09T14:27:36.812031Z","iopub.status.idle":"2024-12-09T14:27:37.36588Z","shell.execute_reply.started":"2024-12-09T14:27:36.811992Z","shell.execute_reply":"2024-12-09T14:27:37.364892Z"},"papermill":{"duration":0.594644,"end_time":"2024-12-08T17:33:46.743208","exception":false,"start_time":"2024-12-08T17:33:46.148564","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"f3c1cd04","cell_type":"code","source":"print(\"First 5 values in the dataset:\")\nprint(\"\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:37.367072Z","iopub.execute_input":"2024-12-09T14:27:37.367391Z","iopub.status.idle":"2024-12-09T14:27:37.394039Z","shell.execute_reply.started":"2024-12-09T14:27:37.367363Z","shell.execute_reply":"2024-12-09T14:27:37.393161Z"},"papermill":{"duration":0.038975,"end_time":"2024-12-08T17:33:46.79182","exception":false,"start_time":"2024-12-08T17:33:46.752845","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"efbbfe2a","cell_type":"code","source":"print(\"Last 5 values in the dataset:\")\nprint(\"\")\ntrain.tail()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:37.395039Z","iopub.execute_input":"2024-12-09T14:27:37.395298Z","iopub.status.idle":"2024-12-09T14:27:37.414101Z","shell.execute_reply.started":"2024-12-09T14:27:37.395274Z","shell.execute_reply":"2024-12-09T14:27:37.413204Z"},"papermill":{"duration":0.032098,"end_time":"2024-12-08T17:33:46.834404","exception":false,"start_time":"2024-12-08T17:33:46.802306","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0dc2c672","cell_type":"code","source":"print(\"Descriptive statistics for object-type columns:\")\nprint(\"\")\nprint(train.describe(include = 'object').T)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:37.415299Z","iopub.execute_input":"2024-12-09T14:27:37.415615Z","iopub.status.idle":"2024-12-09T14:27:39.065349Z","shell.execute_reply.started":"2024-12-09T14:27:37.415588Z","shell.execute_reply":"2024-12-09T14:27:39.06446Z"},"papermill":{"duration":1.786166,"end_time":"2024-12-08T17:33:48.630359","exception":false,"start_time":"2024-12-08T17:33:46.844193","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"d1a85033","cell_type":"code","source":"print(\"Descriptive statistics for numerical columns:\")\nprint(\"\")\nprint(train.describe(include = 'object').T)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:39.066377Z","iopub.execute_input":"2024-12-09T14:27:39.066627Z","iopub.status.idle":"2024-12-09T14:27:40.709105Z","shell.execute_reply.started":"2024-12-09T14:27:39.066605Z","shell.execute_reply":"2024-12-09T14:27:40.708208Z"},"papermill":{"duration":1.828467,"end_time":"2024-12-08T17:33:50.46991","exception":false,"start_time":"2024-12-08T17:33:48.641443","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"aee2abca","cell_type":"code","source":"print(\"Columns with missing data:\")\nprint(\"\")\ntrain.tail()\ntrain.isnull().sum().sort_values(ascending = False).head(11)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:40.712741Z","iopub.execute_input":"2024-12-09T14:27:40.713245Z","iopub.status.idle":"2024-12-09T14:27:41.198932Z","shell.execute_reply.started":"2024-12-09T14:27:40.713216Z","shell.execute_reply":"2024-12-09T14:27:41.198119Z"},"papermill":{"duration":0.532955,"end_time":"2024-12-08T17:33:51.014807","exception":false,"start_time":"2024-12-08T17:33:50.481852","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"1c1e00df","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 3. EDA</b></div>","metadata":{"papermill":{"duration":0.010257,"end_time":"2024-12-08T17:33:51.035748","exception":false,"start_time":"2024-12-08T17:33:51.025491","status":"completed"},"tags":[]}},{"id":"d208de6e","cell_type":"markdown","source":"Chart modified from [here](https://www.kaggle.com/code/oscarm524/ps-s4-ep12-eda-modeling-submission).","metadata":{"papermill":{"duration":0.01108,"end_time":"2024-12-08T17:33:51.057399","exception":false,"start_time":"2024-12-08T17:33:51.046319","status":"completed"},"tags":[]}},{"id":"bb585f9d","cell_type":"code","source":"kde = sns.kdeplot(data = train, x = 'Premium Amount').get_lines()[0].get_data()\nx = kde[0]\ny = kde[1]\npeaks, _ = find_peaks(y)\n\n# Highlight the peaks\nplt.plot(x[peaks], y[peaks], 'ro')  # 'ro' means red color, circle marker\nplt.fill_between(x, y, color='blue', alpha = 0.5)\nplt.title('Premium Amount: KDE Plot with Highlighted Peaks')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:41.200002Z","iopub.execute_input":"2024-12-09T14:27:41.200329Z","iopub.status.idle":"2024-12-09T14:27:45.672046Z","shell.execute_reply.started":"2024-12-09T14:27:41.200303Z","shell.execute_reply":"2024-12-09T14:27:45.671241Z"},"papermill":{"duration":4.823261,"end_time":"2024-12-08T17:33:55.89123","exception":false,"start_time":"2024-12-08T17:33:51.067969","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"77dc78ac","cell_type":"code","source":"num_cols= train.select_dtypes(include='number').columns\nnum_cols = [val for val in num_cols]\ncat_cols = train.select_dtypes(exclude='number').columns\ndatetime_cols = ['Policy Start Date']\ncat_cols = [col for col in cat_cols if col not in datetime_cols]\n\nprint(\"Numerical columns:\",)\nprint(num_cols)\nprint(\"\")\nprint(\"Categorical columns excluding datetime columns:\")\nprint(cat_cols)\nprint(\"\")\nprint(\"Datetime column:\")\ndatetime_cols","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:45.67332Z","iopub.execute_input":"2024-12-09T14:27:45.673971Z","iopub.status.idle":"2024-12-09T14:27:46.187097Z","shell.execute_reply.started":"2024-12-09T14:27:45.673927Z","shell.execute_reply":"2024-12-09T14:27:46.186255Z"},"papermill":{"duration":0.543844,"end_time":"2024-12-08T17:33:56.446743","exception":false,"start_time":"2024-12-08T17:33:55.902899","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"bf322f53","cell_type":"code","source":"train[num_cols].hist(bins=30, figsize=(12, 15))\nplt.suptitle('Histograms of Numerical Columns')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:46.188369Z","iopub.execute_input":"2024-12-09T14:27:46.188992Z","iopub.status.idle":"2024-12-09T14:27:47.937862Z","shell.execute_reply.started":"2024-12-09T14:27:46.188952Z","shell.execute_reply":"2024-12-09T14:27:47.936949Z"},"papermill":{"duration":2.02445,"end_time":"2024-12-08T17:33:58.483216","exception":false,"start_time":"2024-12-08T17:33:56.458766","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"9c0b365b","cell_type":"code","source":"plt.figure(figsize=(12,15))\nfor i, col in enumerate(num_cols):\n    plt.subplot(4,3, i+1)\n    sns.boxplot(y=train[col])\n    plt.title(f'Box Plot of {col}')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:47.938875Z","iopub.execute_input":"2024-12-09T14:27:47.939132Z","iopub.status.idle":"2024-12-09T14:27:49.738504Z","shell.execute_reply.started":"2024-12-09T14:27:47.939108Z","shell.execute_reply":"2024-12-09T14:27:49.737713Z"},"papermill":{"duration":1.837533,"end_time":"2024-12-08T17:34:08.180457","exception":false,"start_time":"2024-12-08T17:34:06.342924","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"12f00a4c","cell_type":"code","source":"corr_matrix = train[num_cols].corr()\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, cmap = 'Blues')\nplt.title('Correlation Heatmap of Numerical Columns')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:49.739635Z","iopub.execute_input":"2024-12-09T14:27:49.739988Z","iopub.status.idle":"2024-12-09T14:27:50.291391Z","shell.execute_reply.started":"2024-12-09T14:27:49.739949Z","shell.execute_reply":"2024-12-09T14:27:50.29054Z"},"papermill":{"duration":0.6483,"end_time":"2024-12-08T17:34:08.846487","exception":false,"start_time":"2024-12-08T17:34:08.198187","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5aac2512","cell_type":"code","source":"corr = train[num_cols].corr()['Premium Amount'].sort_values(ascending = False)\npd.DataFrame(corr).style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:50.292723Z","iopub.execute_input":"2024-12-09T14:27:50.293366Z","iopub.status.idle":"2024-12-09T14:27:50.629089Z","shell.execute_reply.started":"2024-12-09T14:27:50.293325Z","shell.execute_reply":"2024-12-09T14:27:50.62827Z"},"papermill":{"duration":0.380245,"end_time":"2024-12-08T17:34:09.245088","exception":false,"start_time":"2024-12-08T17:34:08.864843","status":"completed"},"scrolled":true,"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"b7cabb7d","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 4. Preprocess</b></div>","metadata":{"papermill":{"duration":0.017948,"end_time":"2024-12-08T17:34:09.282615","exception":false,"start_time":"2024-12-08T17:34:09.264667","status":"completed"},"tags":[]}},{"id":"02424783","cell_type":"code","source":"for df in [train, test]:\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Year'] = df['Policy Start Date'].dt.year\n    df = df.drop(columns = 'Policy Start Date', inplace = True)\n\nprint(\"New columns created: Day, Month, Year\")\n\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:50.63014Z","iopub.execute_input":"2024-12-09T14:27:50.630393Z","iopub.status.idle":"2024-12-09T14:27:51.169788Z","shell.execute_reply.started":"2024-12-09T14:27:50.63037Z","shell.execute_reply":"2024-12-09T14:27:51.168864Z"},"papermill":{"duration":0.570591,"end_time":"2024-12-08T17:34:09.870685","exception":false,"start_time":"2024-12-08T17:34:09.300094","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"53235cdd","cell_type":"code","source":"encoder = ce.TargetEncoder()\n\nfor feature in cat_cols:\n    train[feature] = encoder.fit_transform(train[feature], train['Premium Amount'])\n    test[feature] = encoder.transform(test[feature])\n\nprint(\"Categorical columns transformed with target encoder in train and test data: \")\ncat_cols","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:27:51.17119Z","iopub.execute_input":"2024-12-09T14:27:51.171559Z","iopub.status.idle":"2024-12-09T14:28:00.959463Z","shell.execute_reply.started":"2024-12-09T14:27:51.171518Z","shell.execute_reply":"2024-12-09T14:28:00.958477Z"},"papermill":{"duration":10.027597,"end_time":"2024-12-08T17:34:19.916388","exception":false,"start_time":"2024-12-08T17:34:09.888791","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"bb9c1b90","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 5. Days, Months & Years</b></div>","metadata":{"papermill":{"duration":0.018547,"end_time":"2024-12-08T17:34:19.953495","exception":false,"start_time":"2024-12-08T17:34:19.934948","status":"completed"},"tags":[]}},{"id":"2f067e6e","cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.lineplot(x='Day', y='Premium Amount', data=train)\nplt.xlabel('Day')\nplt.ylabel('Target Amount')\nplt.title('Daily Patterns of Premium Amount')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:00.960483Z","iopub.execute_input":"2024-12-09T14:28:00.960743Z","iopub.status.idle":"2024-12-09T14:28:08.551283Z","shell.execute_reply.started":"2024-12-09T14:28:00.96072Z","shell.execute_reply":"2024-12-09T14:28:08.550375Z"},"papermill":{"duration":8.772864,"end_time":"2024-12-08T17:34:28.744723","exception":false,"start_time":"2024-12-08T17:34:19.971859","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"0bf95b74","cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.boxplot(x='Month', y='Premium Amount', data=train)\nplt.xlabel('Month')\nplt.ylabel('Target Amount')\nplt.title('Seasonality of Target Amount by Month')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:08.552277Z","iopub.execute_input":"2024-12-09T14:28:08.552617Z","iopub.status.idle":"2024-12-09T14:28:08.966783Z","shell.execute_reply.started":"2024-12-09T14:28:08.552581Z","shell.execute_reply":"2024-12-09T14:28:08.965884Z"},"papermill":{"duration":0.563623,"end_time":"2024-12-08T17:34:29.32779","exception":false,"start_time":"2024-12-08T17:34:28.764167","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"12576b23","cell_type":"code","source":"plt.figure(figsize=(10, 6))\nsns.lineplot(x='Year', y='Premium Amount', data=train)\nplt.xlabel('Year')\nplt.ylabel('Premium Amount')\nplt.title('Trend of Premium Amount Over Years')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:08.967885Z","iopub.execute_input":"2024-12-09T14:28:08.968172Z","iopub.status.idle":"2024-12-09T14:28:17.119013Z","shell.execute_reply.started":"2024-12-09T14:28:08.968146Z","shell.execute_reply":"2024-12-09T14:28:17.118173Z"},"papermill":{"duration":8.747538,"end_time":"2024-12-08T17:34:38.095402","exception":false,"start_time":"2024-12-08T17:34:29.347864","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"527627fc","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 6. Scoring</b></div>","metadata":{"papermill":{"duration":0.019318,"end_time":"2024-12-08T17:34:38.13487","exception":false,"start_time":"2024-12-08T17:34:38.115552","status":"completed"},"tags":[]}},{"id":"7ef2992d","cell_type":"code","source":"# I have set a floor value of 20 in line with training data\n\ndef rmsle(y_true, y_pred):\n    y_pred = np.maximum(y_pred, 20)\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\nRMSLE = make_scorer(rmsle, greater_is_better=False)\n\nprint('Submissions are evaluated based on Root Mean Squared Logarithmic Error (RMSLE)')","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:17.120283Z","iopub.execute_input":"2024-12-09T14:28:17.120966Z","iopub.status.idle":"2024-12-09T14:28:17.126759Z","shell.execute_reply.started":"2024-12-09T14:28:17.120925Z","shell.execute_reply":"2024-12-09T14:28:17.125873Z"},"papermill":{"duration":0.028962,"end_time":"2024-12-08T17:34:38.183334","exception":false,"start_time":"2024-12-08T17:34:38.154372","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a4c4f22a","cell_type":"code","source":"def cross_val_score_log(model, X, y, n_splits=5, random_state=42):\n    \n    \"\"\"\"\n    Perform cross-validation and return scores using log-transformed y.\n    \"\"\"\n\n    kf = KFold(n_splits = 5, shuffle=True, random_state=random_state)\n    scores = []\n\n    for train_index, test_index in kf.split(X):\n        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n        y_train, y_test = y_log.iloc[train_index], y_log.iloc[test_index]\n        \n        model.fit(X_train, y_train)\n        preds = model.predict(X_test)\n        \n        # Calculate the score directly in the logarithmic scale\n        score = np.sqrt(mean_squared_error(y_test, preds))\n        scores.append(score)\n\n    mean_score = np.mean(scores)\n    return mean_score","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:17.127797Z","iopub.execute_input":"2024-12-09T14:28:17.128077Z","iopub.status.idle":"2024-12-09T14:28:17.141511Z","shell.execute_reply.started":"2024-12-09T14:28:17.128033Z","shell.execute_reply":"2024-12-09T14:28:17.140822Z"},"papermill":{"duration":0.029841,"end_time":"2024-12-08T17:34:38.234433","exception":false,"start_time":"2024-12-08T17:34:38.204592","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"02fa7592","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 7. Feature selection</b></div>","metadata":{"papermill":{"duration":0.020592,"end_time":"2024-12-08T17:34:38.275499","exception":false,"start_time":"2024-12-08T17:34:38.254907","status":"completed"},"tags":[]}},{"id":"1e4ace45","cell_type":"code","source":"y = train['Premium Amount']\ny_log = np.log1p(y)\nX = train.drop(columns = ['Premium Amount'])","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:17.142375Z","iopub.execute_input":"2024-12-09T14:28:17.142647Z","iopub.status.idle":"2024-12-09T14:28:17.234566Z","shell.execute_reply.started":"2024-12-09T14:28:17.142622Z","shell.execute_reply":"2024-12-09T14:28:17.233846Z"},"papermill":{"duration":0.112147,"end_time":"2024-12-08T17:34:38.408458","exception":false,"start_time":"2024-12-08T17:34:38.296311","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"5e028045","cell_type":"code","source":"# Initialize the model\nmodel = xgb.XGBRegressor(random_state=37, \n                         tree_method = 'hist', \n                         device = 'cuda', \n                         n_jobs = -1)\n\n# Calculate the CV score with all features\ncv_score = -cross_val_score(model, X, y, cv=5, scoring = RMSLE).mean()\nprint(f'CV score with all features: {cv_score}')\n\n# Store the results in a list\nresults = []\n\n# Loop through each feature and calculate the CV score without that feature\nfor feature in X.columns:\n    X_temp = X.drop(feature, axis=1)\n    cv_score_temp = -cross_val_score(model, X_temp, y, cv=5, scoring = RMSLE).mean()\n    print(f'CV score without {feature}: {cv_score_temp}')\n    results.append((feature, cv_score_temp))\n\n# Sort the results in ascending order of CV scores\nresults.sort(key=lambda x: x[1])\n\n# Store features whose removal results in a lower CV score\nlower_cv_features = [feature for feature, score in results if score < cv_score]\n\n# Print the sorted results and the features with lower CV scores\nprint(\"\")\nprint(\"Features whose removal results in a lower CV score:\")\nprint(\"\")\nfor feature, score in results:\n    if score < cv_score:\n        print(f'- {feature}')","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:28:17.235514Z","iopub.execute_input":"2024-12-09T14:28:17.235943Z","iopub.status.idle":"2024-12-09T14:32:25.549897Z","shell.execute_reply.started":"2024-12-09T14:28:17.235912Z","shell.execute_reply":"2024-12-09T14:32:25.548981Z"},"papermill":{"duration":262.535457,"end_time":"2024-12-08T17:39:00.964206","exception":false,"start_time":"2024-12-08T17:34:38.428749","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"da59fe95","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 8. Base Model</b></div>","metadata":{"execution":{"iopub.execute_input":"2024-12-01T18:26:51.869955Z","iopub.status.busy":"2024-12-01T18:26:51.867724Z","iopub.status.idle":"2024-12-01T18:26:51.882484Z","shell.execute_reply":"2024-12-01T18:26:51.881025Z","shell.execute_reply.started":"2024-12-01T18:26:51.869877Z"},"papermill":{"duration":0.020901,"end_time":"2024-12-08T17:39:01.005926","exception":false,"start_time":"2024-12-08T17:39:00.985025","status":"completed"},"tags":[]}},{"id":"93689be7","cell_type":"code","source":"X_fewer = X.drop(columns = lower_cv_features)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:32:25.551211Z","iopub.execute_input":"2024-12-09T14:32:25.55157Z","iopub.status.idle":"2024-12-09T14:32:25.580455Z","shell.execute_reply.started":"2024-12-09T14:32:25.551532Z","shell.execute_reply":"2024-12-09T14:32:25.579854Z"},"papermill":{"duration":0.052747,"end_time":"2024-12-08T17:39:01.078652","exception":false,"start_time":"2024-12-08T17:39:01.025905","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"72a048c6","cell_type":"markdown","source":"Params modified from [here](https://www.kaggle.com/code/oscarm524/ps-s4-ep12-eda-modeling-submission).","metadata":{"papermill":{"duration":0.020707,"end_time":"2024-12-08T17:39:01.1195","exception":false,"start_time":"2024-12-08T17:39:01.098793","status":"completed"},"tags":[]}},{"id":"c38112fc","cell_type":"code","source":"params = {\n    'num_leaves': 71,\n    'learning_rate': 0.05412467152424433,\n    'n_estimators': 595,\n    'max_depth': 12,\n    'min_data_in_leaf': 97,\n    'bagging_fraction': 0.5200288825838669,\n    'feature_fraction': 0.9881738491942492,\n    'verbose': -1, \n    'n_jobs': -1, \n    'device': 'gpu'\n}\n\nmodel1 = lgb.LGBMRegressor(**params)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:32:25.581415Z","iopub.execute_input":"2024-12-09T14:32:25.581736Z","iopub.status.idle":"2024-12-09T14:32:25.58617Z","shell.execute_reply.started":"2024-12-09T14:32:25.58171Z","shell.execute_reply":"2024-12-09T14:32:25.585262Z"},"papermill":{"duration":0.027673,"end_time":"2024-12-08T17:39:01.167257","exception":false,"start_time":"2024-12-08T17:39:01.139584","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"a8bfdf9f","cell_type":"code","source":"print(- cross_val_score(model1, X, y, cv = 5, scoring = RMSLE).mean())\nprint('LightBoost cross validation score with all features')\nprint(\"\")\nprint(- cross_val_score(model1, X_fewer, y, cv = 5, scoring = RMSLE).mean())\nprint('Lightboost cross validation score with fewer features')\nprint(\"\")\nprint(cross_val_score_log(model1, X, y))\nprint('LightBoost y_log cross validation score with all features')\nprint(\"\")\nprint(cross_val_score_log(model1, X_fewer, y))\nprint('Lightboost y_log cross validation score with fewer features')","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:32:25.590766Z","iopub.execute_input":"2024-12-09T14:32:25.591005Z","iopub.status.idle":"2024-12-09T14:37:00.511549Z","shell.execute_reply.started":"2024-12-09T14:32:25.590982Z","shell.execute_reply":"2024-12-09T14:37:00.510214Z"},"papermill":{"duration":290.77298,"end_time":"2024-12-08T17:43:51.960107","exception":false,"start_time":"2024-12-08T17:39:01.187127","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"8f628dbc","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 9. Prediction</b></div>","metadata":{"papermill":{"duration":0.029132,"end_time":"2024-12-08T17:43:52.018911","exception":false,"start_time":"2024-12-08T17:43:51.989779","status":"completed"},"tags":[]}},{"id":"7ffa4f74","cell_type":"code","source":"test_fewer = test.drop(columns = lower_cv_features)\nprint(\"Predict using the following test columns to match the best model training:\")\n[ val for val in test_fewer.columns]","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:37:00.51258Z","iopub.execute_input":"2024-12-09T14:37:00.512897Z","iopub.status.idle":"2024-12-09T14:37:00.543096Z","shell.execute_reply.started":"2024-12-09T14:37:00.512866Z","shell.execute_reply":"2024-12-09T14:37:00.541788Z"},"papermill":{"duration":0.06265,"end_time":"2024-12-08T17:43:52.110768","exception":false,"start_time":"2024-12-08T17:43:52.048118","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"4757f2eb","cell_type":"code","source":"log_preds = model1.fit(X_fewer, y_log).predict(test_fewer)\npreds = np.expm1(log_preds)","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:37:00.544135Z","iopub.execute_input":"2024-12-09T14:37:00.544473Z","iopub.status.idle":"2024-12-09T14:37:22.759417Z","shell.execute_reply.started":"2024-12-09T14:37:00.544436Z","shell.execute_reply":"2024-12-09T14:37:22.758523Z"},"papermill":{"duration":22.561146,"end_time":"2024-12-08T17:44:14.698833","exception":false,"start_time":"2024-12-08T17:43:52.137687","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"id":"85075871","cell_type":"markdown","source":"# <div style=\"text-align:left; border-radius:15px; padding:15px; margin:0; font-size:100%; overflow:hidden; box-shadow:0 3px 6px rgba(0, 0, 0, 0.3);\"><b> 10. Submission</b></div>","metadata":{"papermill":{"duration":0.023068,"end_time":"2024-12-08T17:44:14.753612","exception":false,"start_time":"2024-12-08T17:44:14.730544","status":"completed"},"tags":[]}},{"id":"c55f9acc","cell_type":"code","source":"submission['Premium Amount'] = preds\nsubmission.to_csv('submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2024-12-09T14:37:22.76044Z","iopub.execute_input":"2024-12-09T14:37:22.761078Z","iopub.status.idle":"2024-12-09T14:37:24.123192Z","shell.execute_reply.started":"2024-12-09T14:37:22.76103Z","shell.execute_reply":"2024-12-09T14:37:24.122458Z"},"papermill":{"duration":1.476187,"end_time":"2024-12-08T17:44:16.252064","exception":false,"start_time":"2024-12-08T17:44:14.775877","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}