{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Importing libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:12.425119Z","iopub.execute_input":"2024-12-09T04:43:12.425483Z","iopub.status.idle":"2024-12-09T04:43:15.291729Z","shell.execute_reply.started":"2024-12-09T04:43:12.42545Z","shell.execute_reply":"2024-12-09T04:43:15.290477Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## loading train and test data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:15.294224Z","iopub.execute_input":"2024-12-09T04:43:15.295001Z","iopub.status.idle":"2024-12-09T04:43:21.952239Z","shell.execute_reply.started":"2024-12-09T04:43:15.294947Z","shell.execute_reply":"2024-12-09T04:43:21.951101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:21.953521Z","iopub.execute_input":"2024-12-09T04:43:21.953857Z","iopub.status.idle":"2024-12-09T04:43:26.030629Z","shell.execute_reply.started":"2024-12-09T04:43:21.953824Z","shell.execute_reply":"2024-12-09T04:43:26.029604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Taking a look at the data","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:26.032958Z","iopub.execute_input":"2024-12-09T04:43:26.03335Z","iopub.status.idle":"2024-12-09T04:43:26.701608Z","shell.execute_reply.started":"2024-12-09T04:43:26.033316Z","shell.execute_reply":"2024-12-09T04:43:26.700479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:26.702699Z","iopub.execute_input":"2024-12-09T04:43:26.703042Z","iopub.status.idle":"2024-12-09T04:43:26.750458Z","shell.execute_reply.started":"2024-12-09T04:43:26.702996Z","shell.execute_reply":"2024-12-09T04:43:26.749295Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:26.752227Z","iopub.execute_input":"2024-12-09T04:43:26.75265Z","iopub.status.idle":"2024-12-09T04:43:26.760163Z","shell.execute_reply.started":"2024-12-09T04:43:26.752602Z","shell.execute_reply":"2024-12-09T04:43:26.758972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:26.761396Z","iopub.execute_input":"2024-12-09T04:43:26.761768Z","iopub.status.idle":"2024-12-09T04:43:27.211317Z","shell.execute_reply.started":"2024-12-09T04:43:26.761701Z","shell.execute_reply":"2024-12-09T04:43:27.210245Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### deleting id column from both the train and the test data","metadata":{}},{"cell_type":"code","source":"del train['id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:27.212684Z","iopub.execute_input":"2024-12-09T04:43:27.213001Z","iopub.status.idle":"2024-12-09T04:43:27.218241Z","shell.execute_reply.started":"2024-12-09T04:43:27.212971Z","shell.execute_reply":"2024-12-09T04:43:27.217117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:27.21963Z","iopub.execute_input":"2024-12-09T04:43:27.220103Z","iopub.status.idle":"2024-12-09T04:43:27.875684Z","shell.execute_reply.started":"2024-12-09T04:43:27.220055Z","shell.execute_reply":"2024-12-09T04:43:27.874483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# del test['id']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:27.879513Z","iopub.execute_input":"2024-12-09T04:43:27.879891Z","iopub.status.idle":"2024-12-09T04:43:27.884393Z","shell.execute_reply.started":"2024-12-09T04:43:27.879857Z","shell.execute_reply":"2024-12-09T04:43:27.883126Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exploring data","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:27.885816Z","iopub.execute_input":"2024-12-09T04:43:27.8862Z","iopub.status.idle":"2024-12-09T04:43:28.621073Z","shell.execute_reply.started":"2024-12-09T04:43:27.886167Z","shell.execute_reply":"2024-12-09T04:43:28.619957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in list(train.columns):\n    print(f\"Total missing values in {col} is:\", train[col].isna().sum(), end='\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:28.622394Z","iopub.execute_input":"2024-12-09T04:43:28.622678Z","iopub.status.idle":"2024-12-09T04:43:29.233171Z","shell.execute_reply.started":"2024-12-09T04:43:28.622649Z","shell.execute_reply":"2024-12-09T04:43:29.231984Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are a lot of missing values in the dataset.","metadata":{}},{"cell_type":"markdown","source":"Let's see the non-numerical cols","metadata":{}},{"cell_type":"code","source":"train.describe(include='object')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:29.234473Z","iopub.execute_input":"2024-12-09T04:43:29.234791Z","iopub.status.idle":"2024-12-09T04:43:31.944025Z","shell.execute_reply.started":"2024-12-09T04:43:29.234752Z","shell.execute_reply":"2024-12-09T04:43:31.942964Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### defining non-numeric cols","metadata":{}},{"cell_type":"code","source":"non_numerical_cols = [col for col in list(train.columns) if train[col].dtype == 'object']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:31.945294Z","iopub.execute_input":"2024-12-09T04:43:31.945576Z","iopub.status.idle":"2024-12-09T04:43:31.95034Z","shell.execute_reply.started":"2024-12-09T04:43:31.945547Z","shell.execute_reply":"2024-12-09T04:43:31.949342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"non_numerical_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:31.951863Z","iopub.execute_input":"2024-12-09T04:43:31.952585Z","iopub.status.idle":"2024-12-09T04:43:31.967908Z","shell.execute_reply.started":"2024-12-09T04:43:31.952538Z","shell.execute_reply":"2024-12-09T04:43:31.966919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in non_numerical_cols:\n    print(train[col].value_counts(), end='\\n\\n\\n')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:31.969209Z","iopub.execute_input":"2024-12-09T04:43:31.969603Z","iopub.status.idle":"2024-12-09T04:43:33.369717Z","shell.execute_reply.started":"2024-12-09T04:43:31.969565Z","shell.execute_reply":"2024-12-09T04:43:33.368619Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### defining numerical cols","metadata":{}},{"cell_type":"code","source":"numerical_cols = [col for col in list(train.columns) if train[col].dtype == 'float']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:33.371014Z","iopub.execute_input":"2024-12-09T04:43:33.371441Z","iopub.status.idle":"2024-12-09T04:43:33.37724Z","shell.execute_reply.started":"2024-12-09T04:43:33.371392Z","shell.execute_reply":"2024-12-09T04:43:33.376128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numerical_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:33.378861Z","iopub.execute_input":"2024-12-09T04:43:33.379434Z","iopub.status.idle":"2024-12-09T04:43:33.394095Z","shell.execute_reply.started":"2024-12-09T04:43:33.379385Z","shell.execute_reply":"2024-12-09T04:43:33.393001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(numerical_cols + non_numerical_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:33.395424Z","iopub.execute_input":"2024-12-09T04:43:33.395769Z","iopub.status.idle":"2024-12-09T04:43:33.408481Z","shell.execute_reply.started":"2024-12-09T04:43:33.395739Z","shell.execute_reply":"2024-12-09T04:43:33.407418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(list(train.columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:33.409769Z","iopub.execute_input":"2024-12-09T04:43:33.410094Z","iopub.status.idle":"2024-12-09T04:43:33.421732Z","shell.execute_reply.started":"2024-12-09T04:43:33.410024Z","shell.execute_reply":"2024-12-09T04:43:33.420666Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"numerical + non numerical cols equals the number of total cols","metadata":{}},{"cell_type":"markdown","source":"## Preparing data for model building","metadata":{}},{"cell_type":"code","source":"for col in list(train.columns):\n    if train[col].isna().sum() > 0:\n        print(f\"{col}:\", train[col].isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:33.423003Z","iopub.execute_input":"2024-12-09T04:43:33.423313Z","iopub.status.idle":"2024-12-09T04:43:34.223144Z","shell.execute_reply.started":"2024-12-09T04:43:33.423284Z","shell.execute_reply":"2024-12-09T04:43:34.221912Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"These are all the cols with missing values.","metadata":{}},{"cell_type":"code","source":"# A better way to look at missing values in the dataset.\ntrain.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:34.224464Z","iopub.execute_input":"2024-12-09T04:43:34.224795Z","iopub.status.idle":"2024-12-09T04:43:34.862358Z","shell.execute_reply.started":"2024-12-09T04:43:34.224757Z","shell.execute_reply":"2024-12-09T04:43:34.861249Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Which row has missing Insurance Duration??","metadata":{}},{"cell_type":"code","source":"train[train['Insurance Duration'].isna() == True]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:34.863805Z","iopub.execute_input":"2024-12-09T04:43:34.864769Z","iopub.status.idle":"2024-12-09T04:43:34.887614Z","shell.execute_reply.started":"2024-12-09T04:43:34.864716Z","shell.execute_reply":"2024-12-09T04:43:34.886595Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There is no insurance end date, so I can't exactly predict what insurance duration could be. So let's just drop this entire row.","metadata":{}},{"cell_type":"code","source":"train = train.dropna(subset=['Insurance Duration'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:34.889064Z","iopub.execute_input":"2024-12-09T04:43:34.88948Z","iopub.status.idle":"2024-12-09T04:43:35.110148Z","shell.execute_reply.started":"2024-12-09T04:43:34.889433Z","shell.execute_reply":"2024-12-09T04:43:35.109094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# new_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.11138Z","iopub.execute_input":"2024-12-09T04:43:35.111687Z","iopub.status.idle":"2024-12-09T04:43:35.115957Z","shell.execute_reply.started":"2024-12-09T04:43:35.111656Z","shell.execute_reply":"2024-12-09T04:43:35.114851Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, let's see vehicle age","metadata":{}},{"cell_type":"code","source":"train[train['Vehicle Age'].isna() == True]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.117394Z","iopub.execute_input":"2024-12-09T04:43:35.117708Z","iopub.status.idle":"2024-12-09T04:43:35.149203Z","shell.execute_reply.started":"2024-12-09T04:43:35.117678Z","shell.execute_reply":"2024-12-09T04:43:35.148119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Vehicle Age'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.151017Z","iopub.execute_input":"2024-12-09T04:43:35.151457Z","iopub.status.idle":"2024-12-09T04:43:35.228826Z","shell.execute_reply.started":"2024-12-09T04:43:35.151409Z","shell.execute_reply":"2024-12-09T04:43:35.227853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.hist(train['Vehicle Age'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.236497Z","iopub.execute_input":"2024-12-09T04:43:35.236831Z","iopub.status.idle":"2024-12-09T04:43:35.554046Z","shell.execute_reply.started":"2024-12-09T04:43:35.236793Z","shell.execute_reply":"2024-12-09T04:43:35.552917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There is no normal distribution, so lets just remove those rows","metadata":{}},{"cell_type":"code","source":"train = train.drop(index = [15629, 53843, 134847, 412847, 595207, 1068825])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.555319Z","iopub.execute_input":"2024-12-09T04:43:35.555651Z","iopub.status.idle":"2024-12-09T04:43:35.882521Z","shell.execute_reply.started":"2024-12-09T04:43:35.555617Z","shell.execute_reply":"2024-12-09T04:43:35.881428Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We have deleted all rows that had missing values in vehicle age col","metadata":{}},{"cell_type":"code","source":"train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:35.883871Z","iopub.execute_input":"2024-12-09T04:43:35.884217Z","iopub.status.idle":"2024-12-09T04:43:36.523913Z","shell.execute_reply.started":"2024-12-09T04:43:35.884184Z","shell.execute_reply":"2024-12-09T04:43:36.522915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now, all cols have a lot of missing values, we can't simply delete those","metadata":{}},{"cell_type":"code","source":"test.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:36.525106Z","iopub.execute_input":"2024-12-09T04:43:36.525412Z","iopub.status.idle":"2024-12-09T04:43:36.950384Z","shell.execute_reply.started":"2024-12-09T04:43:36.525382Z","shell.execute_reply":"2024-12-09T04:43:36.949251Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Test data also has missing values","metadata":{}},{"cell_type":"code","source":"cols_with_missing_values = []\n\nfor col in list(train.columns):\n    if train[col].isna().sum() > 0:\n        cols_with_missing_values.append(col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:36.951951Z","iopub.execute_input":"2024-12-09T04:43:36.952308Z","iopub.status.idle":"2024-12-09T04:43:37.565732Z","shell.execute_reply.started":"2024-12-09T04:43:36.952277Z","shell.execute_reply":"2024-12-09T04:43:37.564578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_with_missing_values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:37.566984Z","iopub.execute_input":"2024-12-09T04:43:37.56734Z","iopub.status.idle":"2024-12-09T04:43:37.574064Z","shell.execute_reply.started":"2024-12-09T04:43:37.567309Z","shell.execute_reply":"2024-12-09T04:43:37.572983Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's plot distribution of all those cols","metadata":{}},{"cell_type":"code","source":"for col in cols_with_missing_values: \n    \n    plt.figure(figsize=(8, 6)) \n\n    # For categorical columns \n    if train[col].dtype == 'object': \n        sns.countplot(x=train[col]) \n    \n    # For numerical columns\n    else:  \n        sns.histplot(train[col], kde=True) \n        plt.title(f'Distribution of {col}') \n        plt.xlabel(col) \n        plt.ylabel('Frequency') \n        plt.grid(True)\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:43:37.575571Z","iopub.execute_input":"2024-12-09T04:43:37.576022Z","iopub.status.idle":"2024-12-09T04:44:11.672692Z","shell.execute_reply.started":"2024-12-09T04:43:37.575975Z","shell.execute_reply":"2024-12-09T04:44:11.671582Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"None of the distribution is close to a normal distribution, and thus it makes replacing na values with mean, median or mode not suitable. Thus, we would either need to use some kind of imputer, like KNNImputer or we can only use rows that doesn't have any missing values.\n\nIf we choose the latter option, then the number of rows remaining for training the model will be very less compared to original dataset.","metadata":{}},{"cell_type":"markdown","source":"Initially, lets make the model with only non-na rows, and we will see if it performs good. Then, we can compare it with the model trained using imputed missing rows.","metadata":{}},{"cell_type":"markdown","source":"making a new training dataset, with only non-missing rows","metadata":{}},{"cell_type":"code","source":"new_train = train.dropna()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:11.67409Z","iopub.execute_input":"2024-12-09T04:44:11.67445Z","iopub.status.idle":"2024-12-09T04:44:12.421176Z","shell.execute_reply.started":"2024-12-09T04:44:11.674416Z","shell.execute_reply":"2024-12-09T04:44:12.420073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:12.422511Z","iopub.execute_input":"2024-12-09T04:44:12.422861Z","iopub.status.idle":"2024-12-09T04:44:12.677627Z","shell.execute_reply.started":"2024-12-09T04:44:12.422828Z","shell.execute_reply":"2024-12-09T04:44:12.676398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:12.679192Z","iopub.execute_input":"2024-12-09T04:44:12.679566Z","iopub.status.idle":"2024-12-09T04:44:12.686536Z","shell.execute_reply.started":"2024-12-09T04:44:12.679532Z","shell.execute_reply":"2024-12-09T04:44:12.685204Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Oh i forgot to convert the policy start date dtype from object to datetime**","metadata":{}},{"cell_type":"code","source":"new_train['Policy Start Date'] = pd.to_datetime(new_train['Policy Start Date'], errors='coerce')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:12.688232Z","iopub.execute_input":"2024-12-09T04:44:12.68866Z","iopub.status.idle":"2024-12-09T04:44:12.838935Z","shell.execute_reply.started":"2024-12-09T04:44:12.688614Z","shell.execute_reply":"2024-12-09T04:44:12.837728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:12.840543Z","iopub.execute_input":"2024-12-09T04:44:12.840955Z","iopub.status.idle":"2024-12-09T04:44:13.048238Z","shell.execute_reply.started":"2024-12-09T04:44:12.840912Z","shell.execute_reply":"2024-12-09T04:44:13.047075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.duplicated().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.04948Z","iopub.execute_input":"2024-12-09T04:44:13.049803Z","iopub.status.idle":"2024-12-09T04:44:13.439407Z","shell.execute_reply.started":"2024-12-09T04:44:13.049771Z","shell.execute_reply":"2024-12-09T04:44:13.438371Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are no duplicate rows in new_train","metadata":{}},{"cell_type":"markdown","source":"Let's start training the regression model","metadata":{}},{"cell_type":"markdown","source":"## Training the model","metadata":{}},{"cell_type":"markdown","source":"### Importing important libraries for model training:","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error\n# from sklearn.metrics import root_mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.440533Z","iopub.execute_input":"2024-12-09T04:44:13.440828Z","iopub.status.idle":"2024-12-09T04:44:13.745663Z","shell.execute_reply.started":"2024-12-09T04:44:13.440799Z","shell.execute_reply":"2024-12-09T04:44:13.744636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"root mean square log error is the metric upon which our solutions will be scored, so we need to import that. But it might give you error, because in sklearn, it was added around verion 1.4. If your sklearn had previous version, it will give you an error.","metadata":{}},{"cell_type":"markdown","source":"If you are working on your own environment, then try running the following code blocks. Else just don't import that into this kaggle notebook","metadata":{}},{"cell_type":"code","source":"import sklearn\nprint(sklearn.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.746809Z","iopub.execute_input":"2024-12-09T04:44:13.747263Z","iopub.status.idle":"2024-12-09T04:44:13.75337Z","shell.execute_reply.started":"2024-12-09T04:44:13.747218Z","shell.execute_reply":"2024-12-09T04:44:13.752133Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Below line is to update sklearn, if already installed, else it will install latest sklearn","metadata":{}},{"cell_type":"code","source":"# !pip install -U scikit-learn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.754622Z","iopub.execute_input":"2024-12-09T04:44:13.754943Z","iopub.status.idle":"2024-12-09T04:44:13.765758Z","shell.execute_reply.started":"2024-12-09T04:44:13.754913Z","shell.execute_reply":"2024-12-09T04:44:13.764511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !pip freeze","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.767276Z","iopub.execute_input":"2024-12-09T04:44:13.767734Z","iopub.status.idle":"2024-12-09T04:44:13.777841Z","shell.execute_reply.started":"2024-12-09T04:44:13.767673Z","shell.execute_reply":"2024-12-09T04:44:13.776578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.779349Z","iopub.execute_input":"2024-12-09T04:44:13.77982Z","iopub.status.idle":"2024-12-09T04:44:13.795312Z","shell.execute_reply.started":"2024-12-09T04:44:13.779774Z","shell.execute_reply":"2024-12-09T04:44:13.79397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"non_numerical_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.79657Z","iopub.execute_input":"2024-12-09T04:44:13.796912Z","iopub.status.idle":"2024-12-09T04:44:13.813679Z","shell.execute_reply.started":"2024-12-09T04:44:13.796879Z","shell.execute_reply":"2024-12-09T04:44:13.812607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['Policy Start Date'].dtype","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.815411Z","iopub.execute_input":"2024-12-09T04:44:13.815831Z","iopub.status.idle":"2024-12-09T04:44:13.828545Z","shell.execute_reply.started":"2024-12-09T04:44:13.815796Z","shell.execute_reply":"2024-12-09T04:44:13.827239Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's make different cols for year, month and day","metadata":{}},{"cell_type":"code","source":"new_train['Policy Start Year'] = new_train['Policy Start Date'].dt.year \nnew_train['Policy Start Month'] = new_train['Policy Start Date'].dt.month \nnew_train['Policy Start Day'] = new_train['Policy Start Date'].dt.day","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.83021Z","iopub.execute_input":"2024-12-09T04:44:13.830666Z","iopub.status.idle":"2024-12-09T04:44:13.901884Z","shell.execute_reply.started":"2024-12-09T04:44:13.830616Z","shell.execute_reply":"2024-12-09T04:44:13.900821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.903166Z","iopub.execute_input":"2024-12-09T04:44:13.903448Z","iopub.status.idle":"2024-12-09T04:44:13.910824Z","shell.execute_reply.started":"2024-12-09T04:44:13.90342Z","shell.execute_reply":"2024-12-09T04:44:13.909752Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_train['Policy Start Date'].describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.912165Z","iopub.execute_input":"2024-12-09T04:44:13.912576Z","iopub.status.idle":"2024-12-09T04:44:13.944139Z","shell.execute_reply.started":"2024-12-09T04:44:13.912542Z","shell.execute_reply":"2024-12-09T04:44:13.942957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### One-hot encoding categorical cols","metadata":{}},{"cell_type":"code","source":"non_numerical_cols.remove('Policy Start Date')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.945382Z","iopub.execute_input":"2024-12-09T04:44:13.945788Z","iopub.status.idle":"2024-12-09T04:44:13.951222Z","shell.execute_reply.started":"2024-12-09T04:44:13.945754Z","shell.execute_reply":"2024-12-09T04:44:13.949827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"non_numerical_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.952779Z","iopub.execute_input":"2024-12-09T04:44:13.953303Z","iopub.status.idle":"2024-12-09T04:44:13.968244Z","shell.execute_reply.started":"2024-12-09T04:44:13.953256Z","shell.execute_reply":"2024-12-09T04:44:13.966649Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This line creates n-1 dummy variables, n meaning number of levels - for example in gender it will create just one dummy variable (0 for female and 1 for male, like this. However, it depends on which level is the first one, because of the drop_first setting. It will assume 0 for the first level encountered)","metadata":{}},{"cell_type":"code","source":"new_train = pd.get_dummies(new_train, columns = non_numerical_cols, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:13.969689Z","iopub.execute_input":"2024-12-09T04:44:13.970183Z","iopub.status.idle":"2024-12-09T04:44:14.361688Z","shell.execute_reply.started":"2024-12-09T04:44:13.970135Z","shell.execute_reply":"2024-12-09T04:44:14.36074Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Splitting new_train","metadata":{}},{"cell_type":"markdown","source":"Before training the model, we need to check some things:","metadata":{}},{"cell_type":"code","source":"new_train.drop(columns='Policy Start Date', inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.363208Z","iopub.execute_input":"2024-12-09T04:44:14.363647Z","iopub.status.idle":"2024-12-09T04:44:14.386742Z","shell.execute_reply.started":"2024-12-09T04:44:14.363594Z","shell.execute_reply":"2024-12-09T04:44:14.385677Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We need to remove policy start date, because its datetime","metadata":{}},{"cell_type":"code","source":"X = new_train.drop(columns='Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.388163Z","iopub.execute_input":"2024-12-09T04:44:14.388498Z","iopub.status.idle":"2024-12-09T04:44:14.408619Z","shell.execute_reply.started":"2024-12-09T04:44:14.388465Z","shell.execute_reply":"2024-12-09T04:44:14.407577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = new_train['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.410261Z","iopub.execute_input":"2024-12-09T04:44:14.410605Z","iopub.status.idle":"2024-12-09T04:44:14.415724Z","shell.execute_reply.started":"2024-12-09T04:44:14.410573Z","shell.execute_reply":"2024-12-09T04:44:14.414563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.416778Z","iopub.execute_input":"2024-12-09T04:44:14.417135Z","iopub.status.idle":"2024-12-09T04:44:14.506316Z","shell.execute_reply.started":"2024-12-09T04:44:14.417081Z","shell.execute_reply":"2024-12-09T04:44:14.505386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Initializing the model as a linear regression model","metadata":{}},{"cell_type":"code","source":"model = LinearRegression()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.507884Z","iopub.execute_input":"2024-12-09T04:44:14.508357Z","iopub.status.idle":"2024-12-09T04:44:14.51398Z","shell.execute_reply.started":"2024-12-09T04:44:14.508309Z","shell.execute_reply":"2024-12-09T04:44:14.512742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(new_train.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.515306Z","iopub.execute_input":"2024-12-09T04:44:14.515593Z","iopub.status.idle":"2024-12-09T04:44:14.528691Z","shell.execute_reply.started":"2024-12-09T04:44:14.515564Z","shell.execute_reply":"2024-12-09T04:44:14.527551Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Training the model using training data","metadata":{}},{"cell_type":"code","source":"model.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:14.530064Z","iopub.execute_input":"2024-12-09T04:44:14.530389Z","iopub.status.idle":"2024-12-09T04:44:15.293888Z","shell.execute_reply.started":"2024-12-09T04:44:14.530358Z","shell.execute_reply":"2024-12-09T04:44:15.292836Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Our model has been trained","metadata":{}},{"cell_type":"markdown","source":"### Model Prediction","metadata":{}},{"cell_type":"code","source":"y_preds = model.predict(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.294969Z","iopub.execute_input":"2024-12-09T04:44:15.295388Z","iopub.status.idle":"2024-12-09T04:44:15.322477Z","shell.execute_reply.started":"2024-12-09T04:44:15.295343Z","shell.execute_reply":"2024-12-09T04:44:15.320747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Model Evaluation","metadata":{}},{"cell_type":"code","source":"mae = mean_absolute_error(y_val, y_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.324425Z","iopub.execute_input":"2024-12-09T04:44:15.326608Z","iopub.status.idle":"2024-12-09T04:44:15.340525Z","shell.execute_reply.started":"2024-12-09T04:44:15.326552Z","shell.execute_reply":"2024-12-09T04:44:15.337843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mse = mean_squared_error(y_val, y_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.343653Z","iopub.execute_input":"2024-12-09T04:44:15.346182Z","iopub.status.idle":"2024-12-09T04:44:15.359384Z","shell.execute_reply.started":"2024-12-09T04:44:15.346129Z","shell.execute_reply":"2024-12-09T04:44:15.358128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rmse = np.sqrt(mse)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.360865Z","iopub.execute_input":"2024-12-09T04:44:15.36314Z","iopub.status.idle":"2024-12-09T04:44:15.371623Z","shell.execute_reply.started":"2024-12-09T04:44:15.363066Z","shell.execute_reply":"2024-12-09T04:44:15.370128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f'Mean Absolute Error: {mae}')\nprint(f'Mean Squared Error: {mse}')\nprint(f'Root Mean Squared Error: {rmse}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.373239Z","iopub.execute_input":"2024-12-09T04:44:15.374247Z","iopub.status.idle":"2024-12-09T04:44:15.393147Z","shell.execute_reply.started":"2024-12-09T04:44:15.374176Z","shell.execute_reply":"2024-12-09T04:44:15.391806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rmsle = np.sqrt(mean_squared_log_error(y_val, y_preds))\nrmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.39432Z","iopub.execute_input":"2024-12-09T04:44:15.394762Z","iopub.status.idle":"2024-12-09T04:44:15.42437Z","shell.execute_reply.started":"2024-12-09T04:44:15.394711Z","shell.execute_reply":"2024-12-09T04:44:15.422381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(sklearn.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.425651Z","iopub.execute_input":"2024-12-09T04:44:15.426527Z","iopub.status.idle":"2024-12-09T04:44:15.433646Z","shell.execute_reply.started":"2024-12-09T04:44:15.426454Z","shell.execute_reply":"2024-12-09T04:44:15.432599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test = pd.get_dummies(test, columns = non_numerical_cols, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:15.434803Z","iopub.execute_input":"2024-12-09T04:44:15.436417Z","iopub.status.idle":"2024-12-09T04:44:16.242336Z","shell.execute_reply.started":"2024-12-09T04:44:15.436346Z","shell.execute_reply":"2024-12-09T04:44:16.241426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test['Policy Start Date'] = pd.to_datetime(new_test['Policy Start Date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.243472Z","iopub.execute_input":"2024-12-09T04:44:16.24376Z","iopub.status.idle":"2024-12-09T04:44:16.524999Z","shell.execute_reply.started":"2024-12-09T04:44:16.243732Z","shell.execute_reply":"2024-12-09T04:44:16.523829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test['Policy Start Year'] = new_test['Policy Start Date'].dt.year \nnew_test['Policy Start Month'] = new_test['Policy Start Date'].dt.month \nnew_test['Policy Start Day'] = new_test['Policy Start Date'].dt.day","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.526592Z","iopub.execute_input":"2024-12-09T04:44:16.527079Z","iopub.status.idle":"2024-12-09T04:44:16.652314Z","shell.execute_reply.started":"2024-12-09T04:44:16.526999Z","shell.execute_reply":"2024-12-09T04:44:16.651226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test.drop(columns=['Policy Start Date'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.653785Z","iopub.execute_input":"2024-12-09T04:44:16.654118Z","iopub.status.idle":"2024-12-09T04:44:16.702884Z","shell.execute_reply.started":"2024-12-09T04:44:16.654081Z","shell.execute_reply":"2024-12-09T04:44:16.701729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(X_train.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.704159Z","iopub.execute_input":"2024-12-09T04:44:16.704551Z","iopub.status.idle":"2024-12-09T04:44:16.711615Z","shell.execute_reply.started":"2024-12-09T04:44:16.704468Z","shell.execute_reply":"2024-12-09T04:44:16.710484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(new_test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.713073Z","iopub.execute_input":"2024-12-09T04:44:16.713451Z","iopub.status.idle":"2024-12-09T04:44:16.723291Z","shell.execute_reply.started":"2024-12-09T04:44:16.713398Z","shell.execute_reply":"2024-12-09T04:44:16.722144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test = new_test[X_train.columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.724707Z","iopub.execute_input":"2024-12-09T04:44:16.725079Z","iopub.status.idle":"2024-12-09T04:44:16.776289Z","shell.execute_reply.started":"2024-12-09T04:44:16.725012Z","shell.execute_reply":"2024-12-09T04:44:16.774963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nimputer = SimpleImputer(strategy='median') \nnew_test_imputed = imputer.fit_transform(new_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:16.777811Z","iopub.execute_input":"2024-12-09T04:44:16.77829Z","iopub.status.idle":"2024-12-09T04:44:22.359184Z","shell.execute_reply.started":"2024-12-09T04:44:16.778243Z","shell.execute_reply":"2024-12-09T04:44:22.358284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:22.360366Z","iopub.execute_input":"2024-12-09T04:44:22.360698Z","iopub.status.idle":"2024-12-09T04:44:22.387962Z","shell.execute_reply.started":"2024-12-09T04:44:22.360666Z","shell.execute_reply":"2024-12-09T04:44:22.386882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test_preds = model.predict(new_test_imputed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:22.389419Z","iopub.execute_input":"2024-12-09T04:44:22.389823Z","iopub.status.idle":"2024-12-09T04:44:22.428308Z","shell.execute_reply.started":"2024-12-09T04:44:22.389763Z","shell.execute_reply":"2024-12-09T04:44:22.427005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output  = pd.DataFrame({'id': test.id, 'Premium Amount': y_test_preds})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:22.429833Z","iopub.execute_input":"2024-12-09T04:44:22.430402Z","iopub.status.idle":"2024-12-09T04:44:22.445474Z","shell.execute_reply.started":"2024-12-09T04:44:22.430337Z","shell.execute_reply":"2024-12-09T04:44:22.442419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output.to_csv('Insurance_Premium_test_preds', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:22.447801Z","iopub.execute_input":"2024-12-09T04:44:22.44846Z","iopub.status.idle":"2024-12-09T04:44:24.189703Z","shell.execute_reply.started":"2024-12-09T04:44:22.448389Z","shell.execute_reply":"2024-12-09T04:44:24.188829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:24.190978Z","iopub.execute_input":"2024-12-09T04:44:24.191406Z","iopub.status.idle":"2024-12-09T04:44:24.202622Z","shell.execute_reply.started":"2024-12-09T04:44:24.191361Z","shell.execute_reply":"2024-12-09T04:44:24.201414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Now, training a new model on Imputed data","metadata":{}},{"cell_type":"code","source":"from sklearn.impute import KNNImputer\nfrom sklearn.impute import SimpleImputer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:24.204216Z","iopub.execute_input":"2024-12-09T04:44:24.204514Z","iopub.status.idle":"2024-12-09T04:44:24.212957Z","shell.execute_reply.started":"2024-12-09T04:44:24.204485Z","shell.execute_reply":"2024-12-09T04:44:24.211812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2 = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:24.222995Z","iopub.execute_input":"2024-12-09T04:44:24.223392Z","iopub.status.idle":"2024-12-09T04:44:28.959483Z","shell.execute_reply.started":"2024-12-09T04:44:24.22336Z","shell.execute_reply":"2024-12-09T04:44:28.957948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:28.961015Z","iopub.execute_input":"2024-12-09T04:44:28.961401Z","iopub.status.idle":"2024-12-09T04:44:29.627838Z","shell.execute_reply.started":"2024-12-09T04:44:28.961365Z","shell.execute_reply":"2024-12-09T04:44:29.626946Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Converting Policy start date to datetime first, then extracting year, month and date as different cols, then finally removing policy start date col from training data.","metadata":{}},{"cell_type":"code","source":"train_2['Policy Start Date'] = pd.to_datetime(train_2['Policy Start Date'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:29.629313Z","iopub.execute_input":"2024-12-09T04:44:29.629788Z","iopub.status.idle":"2024-12-09T04:44:30.050255Z","shell.execute_reply.started":"2024-12-09T04:44:29.62973Z","shell.execute_reply":"2024-12-09T04:44:30.048868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:30.051511Z","iopub.execute_input":"2024-12-09T04:44:30.05184Z","iopub.status.idle":"2024-12-09T04:44:30.060729Z","shell.execute_reply.started":"2024-12-09T04:44:30.051809Z","shell.execute_reply":"2024-12-09T04:44:30.05952Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2['Year'] = train_2['Policy Start Date'].dt.year\ntrain_2['Month'] = train_2['Policy Start Date'].dt.month\ntrain_2['Day'] = train_2['Policy Start Date'].dt.day","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:30.062258Z","iopub.execute_input":"2024-12-09T04:44:30.063273Z","iopub.status.idle":"2024-12-09T04:44:30.249691Z","shell.execute_reply.started":"2024-12-09T04:44:30.063218Z","shell.execute_reply":"2024-12-09T04:44:30.248609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2.drop(columns=['Policy Start Date'], inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:30.251208Z","iopub.execute_input":"2024-12-09T04:44:30.251664Z","iopub.status.idle":"2024-12-09T04:44:30.504875Z","shell.execute_reply.started":"2024-12-09T04:44:30.251617Z","shell.execute_reply":"2024-12-09T04:44:30.503897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2 = pd.get_dummies(train_2, columns=non_numerical_cols, drop_first=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:30.506088Z","iopub.execute_input":"2024-12-09T04:44:30.506494Z","iopub.status.idle":"2024-12-09T04:44:31.761203Z","shell.execute_reply.started":"2024-12-09T04:44:30.50645Z","shell.execute_reply":"2024-12-09T04:44:31.760167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_2.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:31.76251Z","iopub.execute_input":"2024-12-09T04:44:31.762933Z","iopub.status.idle":"2024-12-09T04:44:31.821825Z","shell.execute_reply.started":"2024-12-09T04:44:31.762887Z","shell.execute_reply":"2024-12-09T04:44:31.820712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2 = train_2.drop(columns=['Premium Amount'])\ny2 = train_2['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:31.823655Z","iopub.execute_input":"2024-12-09T04:44:31.824116Z","iopub.status.idle":"2024-12-09T04:44:31.88988Z","shell.execute_reply.started":"2024-12-09T04:44:31.824069Z","shell.execute_reply":"2024-12-09T04:44:31.888739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2_train, X2_val, y2_train, y2_val = train_test_split(X2, y2, test_size=0.2,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:31.891516Z","iopub.execute_input":"2024-12-09T04:44:31.891816Z","iopub.status.idle":"2024-12-09T04:44:32.252516Z","shell.execute_reply.started":"2024-12-09T04:44:31.891786Z","shell.execute_reply":"2024-12-09T04:44:32.251346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2_train.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:32.253813Z","iopub.execute_input":"2024-12-09T04:44:32.254133Z","iopub.status.idle":"2024-12-09T04:44:32.317114Z","shell.execute_reply.started":"2024-12-09T04:44:32.254102Z","shell.execute_reply":"2024-12-09T04:44:32.3162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2_val.drop(columns=['id'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:32.318227Z","iopub.execute_input":"2024-12-09T04:44:32.318512Z","iopub.status.idle":"2024-12-09T04:44:32.333215Z","shell.execute_reply.started":"2024-12-09T04:44:32.318483Z","shell.execute_reply":"2024-12-09T04:44:32.332137Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Imputing missing data in X2, our training data for the new model.","metadata":{}},{"cell_type":"code","source":"simple_imputer = SimpleImputer(strategy='median')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:32.334597Z","iopub.execute_input":"2024-12-09T04:44:32.334925Z","iopub.status.idle":"2024-12-09T04:44:32.339595Z","shell.execute_reply.started":"2024-12-09T04:44:32.334895Z","shell.execute_reply":"2024-12-09T04:44:32.338494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2_train = simple_imputer.fit_transform(X2_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:32.340889Z","iopub.execute_input":"2024-12-09T04:44:32.341215Z","iopub.status.idle":"2024-12-09T04:44:38.872415Z","shell.execute_reply.started":"2024-12-09T04:44:32.341185Z","shell.execute_reply":"2024-12-09T04:44:38.87122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X2_val = simple_imputer.transform(X2_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:38.87368Z","iopub.execute_input":"2024-12-09T04:44:38.874006Z","iopub.status.idle":"2024-12-09T04:44:39.194881Z","shell.execute_reply.started":"2024-12-09T04:44:38.873974Z","shell.execute_reply":"2024-12-09T04:44:39.193577Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Creating a new model","metadata":{}},{"cell_type":"code","source":"model2 = LinearRegression()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:39.196425Z","iopub.execute_input":"2024-12-09T04:44:39.19678Z","iopub.status.idle":"2024-12-09T04:44:39.201836Z","shell.execute_reply.started":"2024-12-09T04:44:39.196742Z","shell.execute_reply":"2024-12-09T04:44:39.200655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y2_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:39.203075Z","iopub.execute_input":"2024-12-09T04:44:39.203411Z","iopub.status.idle":"2024-12-09T04:44:39.224978Z","shell.execute_reply.started":"2024-12-09T04:44:39.203378Z","shell.execute_reply":"2024-12-09T04:44:39.223911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model2.fit(X2_train, y2_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:52:46.394676Z","iopub.execute_input":"2024-12-09T04:52:46.39512Z","iopub.status.idle":"2024-12-09T04:52:48.434779Z","shell.execute_reply.started":"2024-12-09T04:52:46.395088Z","shell.execute_reply":"2024-12-09T04:52:48.433691Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y2_preds = model2.predict(X2_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.26434Z","iopub.execute_input":"2024-12-09T04:44:41.264675Z","iopub.status.idle":"2024-12-09T04:44:41.279951Z","shell.execute_reply.started":"2024-12-09T04:44:41.264644Z","shell.execute_reply":"2024-12-09T04:44:41.278329Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.28166Z","iopub.execute_input":"2024-12-09T04:44:41.282864Z","iopub.status.idle":"2024-12-09T04:44:41.291333Z","shell.execute_reply.started":"2024-12-09T04:44:41.282784Z","shell.execute_reply":"2024-12-09T04:44:41.289631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rmsle = np.sqrt((mean_squared_log_error(y2_val, y2_preds)))\nrmsle","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.293469Z","iopub.execute_input":"2024-12-09T04:44:41.29465Z","iopub.status.idle":"2024-12-09T04:44:41.345536Z","shell.execute_reply.started":"2024-12-09T04:44:41.294547Z","shell.execute_reply":"2024-12-09T04:44:41.343075Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Predict test data on new model","metadata":{}},{"cell_type":"code","source":"y2_test_preds = model2.predict(new_test_imputed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.347093Z","iopub.execute_input":"2024-12-09T04:44:41.34765Z","iopub.status.idle":"2024-12-09T04:44:41.392321Z","shell.execute_reply.started":"2024-12-09T04:44:41.347584Z","shell.execute_reply":"2024-12-09T04:44:41.390832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output2  = pd.DataFrame({'id': test.id, 'Premium Amount': y2_test_preds})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.393933Z","iopub.execute_input":"2024-12-09T04:44:41.394558Z","iopub.status.idle":"2024-12-09T04:44:41.409085Z","shell.execute_reply.started":"2024-12-09T04:44:41.394481Z","shell.execute_reply":"2024-12-09T04:44:41.407741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output2.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:44:41.410697Z","iopub.execute_input":"2024-12-09T04:44:41.4113Z","iopub.status.idle":"2024-12-09T04:44:43.130761Z","shell.execute_reply.started":"2024-12-09T04:44:41.411235Z","shell.execute_reply":"2024-12-09T04:44:43.129696Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Trying natural log transformation on target variable","metadata":{}},{"cell_type":"code","source":"model3 = LinearRegression()\nmodel3.fit(X2_train, np.log1p(y2_train))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:53:33.714277Z","iopub.execute_input":"2024-12-09T04:53:33.714706Z","iopub.status.idle":"2024-12-09T04:53:35.751912Z","shell.execute_reply.started":"2024-12-09T04:53:33.714673Z","shell.execute_reply":"2024-12-09T04:53:35.750811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transformed_pred = np.expm1(model3.predict(new_test_imputed))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:57:16.839112Z","iopub.execute_input":"2024-12-09T04:57:16.839534Z","iopub.status.idle":"2024-12-09T04:57:16.91014Z","shell.execute_reply.started":"2024-12-09T04:57:16.83949Z","shell.execute_reply":"2024-12-09T04:57:16.908754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output3 = pd.DataFrame({'id': test.id, 'Premium Amount': transformed_pred})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:58:35.097741Z","iopub.execute_input":"2024-12-09T04:58:35.098147Z","iopub.status.idle":"2024-12-09T04:58:35.106091Z","shell.execute_reply.started":"2024-12-09T04:58:35.098111Z","shell.execute_reply":"2024-12-09T04:58:35.104884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output3.to_csv(\"submission_y_log_transformed.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-09T04:59:21.71402Z","iopub.execute_input":"2024-12-09T04:59:21.714452Z","iopub.status.idle":"2024-12-09T04:59:23.336855Z","shell.execute_reply.started":"2024-12-09T04:59:21.71442Z","shell.execute_reply":"2024-12-09T04:59:23.335876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}