{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:13:49.433761Z","iopub.execute_input":"2024-12-30T09:13:49.434129Z","iopub.status.idle":"2024-12-30T09:13:49.832959Z","shell.execute_reply.started":"2024-12-30T09:13:49.434102Z","shell.execute_reply":"2024-12-30T09:13:49.83201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.listdir('/kaggle/input'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:13:49.834081Z","iopub.execute_input":"2024-12-30T09:13:49.834724Z","iopub.status.idle":"2024-12-30T09:13:49.839785Z","shell.execute_reply.started":"2024-12-30T09:13:49.834682Z","shell.execute_reply":"2024-12-30T09:13:49.838931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as InterruptedError\n\ntrain_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:13:49.840788Z","iopub.execute_input":"2024-12-30T09:13:49.84111Z","iopub.status.idle":"2024-12-30T09:14:00.074364Z","shell.execute_reply.started":"2024-12-30T09:13:49.841071Z","shell.execute_reply":"2024-12-30T09:14:00.072965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:00.075559Z","iopub.execute_input":"2024-12-30T09:14:00.076103Z","iopub.status.idle":"2024-12-30T09:14:00.728138Z","shell.execute_reply.started":"2024-12-30T09:14:00.076059Z","shell.execute_reply":"2024-12-30T09:14:00.72717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:00.729147Z","iopub.execute_input":"2024-12-30T09:14:00.729397Z","iopub.status.idle":"2024-12-30T09:14:01.348763Z","shell.execute_reply.started":"2024-12-30T09:14:00.729376Z","shell.execute_reply":"2024-12-30T09:14:01.347421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:01.351496Z","iopub.execute_input":"2024-12-30T09:14:01.351785Z","iopub.status.idle":"2024-12-30T09:14:01.356944Z","shell.execute_reply.started":"2024-12-30T09:14:01.351761Z","shell.execute_reply":"2024-12-30T09:14:01.355874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Unique values for categorical columns\nprint(train_df['Marital Status'].value_counts())\nprint(train_df['Occupation'].value_counts())\nprint(train_df['Education Level'].value_counts())\nprint(train_df['Policy Type'].value_counts())\nprint(train_df['Exercise Frequency'].value_counts())\nprint(train_df['Smoking Status'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:01.358643Z","iopub.execute_input":"2024-12-30T09:14:01.358989Z","iopub.status.idle":"2024-12-30T09:14:01.887265Z","shell.execute_reply.started":"2024-12-30T09:14:01.358961Z","shell.execute_reply":"2024-12-30T09:14:01.886402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:01.888097Z","iopub.execute_input":"2024-12-30T09:14:01.888342Z","iopub.status.idle":"2024-12-30T09:14:01.894351Z","shell.execute_reply.started":"2024-12-30T09:14:01.888321Z","shell.execute_reply":"2024-12-30T09:14:01.893234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now that we have a better overview of the dataframe we need to understand which columns are going to affect the Premium Amount column, our goal column.\n\nTo do so we can use the correlation matrix.\n\nBefore implementing the correlation matrix we need however to encode the categorical columns, as well as deal with missing values, and transform non-numerical data into something our correlation matrix and model will be able to interpret.","metadata":{"execution":{"iopub.status.busy":"2024-12-20T11:57:19.381538Z","iopub.execute_input":"2024-12-20T11:57:19.381964Z","iopub.status.idle":"2024-12-20T11:57:19.438897Z","shell.execute_reply.started":"2024-12-20T11:57:19.381934Z","shell.execute_reply":"2024-12-20T11:57:19.43693Z"}}},{"cell_type":"markdown","source":"how to deal with NaN values for different columns:\n\nAge, assign AVG to NaN, Annual Income - assign Mean, Marital Status - assign Single, Number of Dependents - assign 0, Occupation - assign Unemployed, Health Score - assign Mean, Previous Claims - assign 0, Vehicle Age - assign 0, Credit Score - assign mean, Insurance Duration - assign MEAN, Customer Feedback - drop column as not influencial on the premium amount.","metadata":{}},{"cell_type":"code","source":"#Dealing with missing data\ntrain_df['Age'] = train_df['Age'].fillna(train_df['Age'].mean())\ntrain_df['Annual Income'] = train_df['Annual Income'].fillna(train_df['Annual Income'].mean())\ntrain_df['Marital Status'] = train_df['Marital Status'].fillna('Single')\ntrain_df['Number of Dependents'] = train_df['Number of Dependents'].fillna(0)\ntrain_df['Occupation'] = train_df['Occupation'].fillna('Unemployed')\ntrain_df['Health Score'] = train_df['Health Score'].fillna(train_df['Health Score'].mean())\ntrain_df['Previous Claims'] = train_df['Previous Claims'].fillna(0)\ntrain_df['Vehicle Age'] = train_df['Vehicle Age'].fillna(0)\ntrain_df['Credit Score'] = train_df['Credit Score'].fillna(train_df['Credit Score'].mean())\ntrain_df['Insurance Duration'] = train_df['Insurance Duration'].fillna(train_df['Insurance Duration'].mean())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:01.895231Z","iopub.execute_input":"2024-12-30T09:14:01.895562Z","iopub.status.idle":"2024-12-30T09:14:02.160539Z","shell.execute_reply.started":"2024-12-30T09:14:01.895537Z","shell.execute_reply":"2024-12-30T09:14:02.159334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#dropping customer feedback\ntrain_df = train_df.drop(columns=['Customer Feedback'])\nprint(train_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:02.161543Z","iopub.execute_input":"2024-12-30T09:14:02.161948Z","iopub.status.idle":"2024-12-30T09:14:02.375915Z","shell.execute_reply.started":"2024-12-30T09:14:02.161912Z","shell.execute_reply":"2024-12-30T09:14:02.374615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:02.377017Z","iopub.execute_input":"2024-12-30T09:14:02.377315Z","iopub.status.idle":"2024-12-30T09:14:02.949102Z","shell.execute_reply.started":"2024-12-30T09:14:02.377283Z","shell.execute_reply":"2024-12-30T09:14:02.948176Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now that we have dealt with missing values we must modify the date columns to timeframe for better readibility of the model.","metadata":{}},{"cell_type":"markdown","source":"Next steps for me to carry on:\n\ntimeframe columns\n\nencoding\n\ncorrelation matrix\n\nmodel training","metadata":{}},{"cell_type":"code","source":"#converting the date to datetime\ntrain_df['Policy Start Date'] = pd.to_datetime(train_df['Policy Start Date'])\n\n#confirm the changes\nprint(train_df['Policy Start Date'].dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:02.950049Z","iopub.execute_input":"2024-12-30T09:14:02.950399Z","iopub.status.idle":"2024-12-30T09:14:03.451633Z","shell.execute_reply.started":"2024-12-30T09:14:02.950361Z","shell.execute_reply":"2024-12-30T09:14:03.450434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#encoding the categorical columns Marital Status and Occupation\nencoded_df = pd.get_dummies(train_df, columns=['Gender','Marital Status', 'Occupation','Location','Property Type','Smoking Status'], drop_first = True)\n\nprint(encoded_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:03.452938Z","iopub.execute_input":"2024-12-30T09:14:03.453345Z","iopub.status.idle":"2024-12-30T09:14:04.425895Z","shell.execute_reply.started":"2024-12-30T09:14:03.453307Z","shell.execute_reply":"2024-12-30T09:14:04.424821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#encoding the ordinal categorical values for Education Level and Exercise Frequency\n\neducation_mapping = {'High School': 0, 'Bachelor\\'s': 1, 'Master\\'s': 2, 'PhD': 3}\nencoded_df['Education Level'] = train_df['Education Level'].map(education_mapping)\n\nexercise_mapping = {'Rarely': 0, 'Monthly': 1, 'Weekly': 2, 'Daily': 3}\nencoded_df['Exercise Frequency'] = train_df['Exercise Frequency'].map(exercise_mapping)\n\npolicy_mapping = {'Basic': 0, 'Comprehensive': 1, 'Premium': 2}\nencoded_df['Policy Type'] = train_df['Policy Type'].map(policy_mapping)\n\nprint(encoded_df[['Education Level','Exercise Frequency', 'Policy Type']].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:04.426989Z","iopub.execute_input":"2024-12-30T09:14:04.42738Z","iopub.status.idle":"2024-12-30T09:14:04.71506Z","shell.execute_reply.started":"2024-12-30T09:14:04.427344Z","shell.execute_reply":"2024-12-30T09:14:04.714211Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now that we have encoded the df we are going to check that all of the columns are numerical.","metadata":{}},{"cell_type":"code","source":"print(encoded_df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:04.716198Z","iopub.execute_input":"2024-12-30T09:14:04.716574Z","iopub.status.idle":"2024-12-30T09:14:04.722581Z","shell.execute_reply.started":"2024-12-30T09:14:04.716537Z","shell.execute_reply":"2024-12-30T09:14:04.721779Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Datetime is going to be a problem for the correlation matrix. For this reason we are going to calculate the age of the account with a datetime difference function.","metadata":{}},{"cell_type":"code","source":"#transforming the start date column into an age account column\nimport datetime\nencoded_df['Policy Duration (days)'] = (datetime.datetime.now() - encoded_df['Policy Start Date']).dt.days\n\nencoded_df = encoded_df.drop(columns=['Policy Start Date'])\n\nprint(encoded_df['Policy Duration (days)'].head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:04.723631Z","iopub.execute_input":"2024-12-30T09:14:04.723913Z","iopub.status.idle":"2024-12-30T09:14:04.83917Z","shell.execute_reply.started":"2024-12-30T09:14:04.723891Z","shell.execute_reply":"2024-12-30T09:14:04.8382Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(encoded_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:04.840166Z","iopub.execute_input":"2024-12-30T09:14:04.840415Z","iopub.status.idle":"2024-12-30T09:14:04.854414Z","shell.execute_reply.started":"2024-12-30T09:14:04.840394Z","shell.execute_reply":"2024-12-30T09:14:04.853018Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now we can finally proceed to the correlation matrix.","metadata":{}},{"cell_type":"code","source":"correlation_matrix = encoded_df.corr()\nprint(correlation_matrix['Premium Amount'].sort_values(ascending = False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:04.855472Z","iopub.execute_input":"2024-12-30T09:14:04.855733Z","iopub.status.idle":"2024-12-30T09:14:06.912933Z","shell.execute_reply.started":"2024-12-30T09:14:04.85571Z","shell.execute_reply":"2024-12-30T09:14:06.911937Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Correlations seem to be weak. This is somehow expected when working with insurance policies as the Premium Amount is often a result of complex calculations with multiple models.\n\nSince the competition requests us to use Linear Regression but the correlation is weak there are a few extra steps we need to complete before proceeding and training the model.\n\nFeature Engineering:\n\nWeak correlations indicate that raw features might not capture relationships effectively. Transformations and interaction terms can help.\n\nMulticollinearity:\nOne-hot encoding increases the number of features, which could lead to multicollinearity (high correlation between features).\n\nScaling Continuous Features:\nFeatures like Age, Annual Income, and Policy Duration (days) might have very different scales, which could make the model less stable.","metadata":{}},{"cell_type":"markdown","source":"Steps to Ensure a Strong Linear Regression Model\n1. Prepare the Data\nBefore training the model:\n\nHandle Feature Scaling:\n\nStandardize or normalize numerical features (Age, Annual Income, Credit Score, etc.) to ensure they have comparable scales.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Select numerical columns\nnumerical_columns = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score',\n                     'Policy Duration (days)', 'Credit Score', 'Insurance Duration']\n\n# Initialize scaler and apply to numerical columns\nscaler = StandardScaler()\nencoded_df[numerical_columns] = scaler.fit_transform(encoded_df[numerical_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:06.913838Z","iopub.execute_input":"2024-12-30T09:14:06.914105Z","iopub.status.idle":"2024-12-30T09:14:07.596559Z","shell.execute_reply.started":"2024-12-30T09:14:06.914082Z","shell.execute_reply":"2024-12-30T09:14:07.595686Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Interaction Terms (Optional):\n\nIntroduce interaction terms to capture relationships between features:","metadata":{"execution":{"iopub.status.busy":"2024-12-29T14:09:47.795434Z","iopub.execute_input":"2024-12-29T14:09:47.796118Z","iopub.status.idle":"2024-12-29T14:09:47.804669Z","shell.execute_reply.started":"2024-12-29T14:09:47.796079Z","shell.execute_reply":"2024-12-29T14:09:47.802956Z"}}},{"cell_type":"code","source":"# Example: Interaction between Age and Health Score\nencoded_df['Age_Health_Interaction'] = encoded_df['Age'] * encoded_df['Health Score']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:07.600589Z","iopub.execute_input":"2024-12-30T09:14:07.601004Z","iopub.status.idle":"2024-12-30T09:14:07.61009Z","shell.execute_reply.started":"2024-12-30T09:14:07.600979Z","shell.execute_reply":"2024-12-30T09:14:07.608704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Check Multicollinearity:\n\nUse variance inflation factor (VIF) to ensure features are not highly collinear:","metadata":{}},{"cell_type":"code","source":"from statsmodels.stats.outliers_influence import variance_inflation_factor\n\n# Calculate VIF for each feature\nX = encoded_df.drop(columns=['Premium Amount'])  # Exclude target\n\n#Let's first make sure we have all numerical columns. If some columns are different than int64 we need to transform them.\nprint(X.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:07.611894Z","iopub.execute_input":"2024-12-30T09:14:07.612138Z","iopub.status.idle":"2024-12-30T09:14:07.761207Z","shell.execute_reply.started":"2024-12-30T09:14:07.612118Z","shell.execute_reply":"2024-12-30T09:14:07.760222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Convert bool Columns to Integers\n\nX = X.astype({col: 'int64' for col in X.select_dtypes(include='bool').columns})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:07.762226Z","iopub.execute_input":"2024-12-30T09:14:07.762607Z","iopub.status.idle":"2024-12-30T09:14:07.888889Z","shell.execute_reply.started":"2024-12-30T09:14:07.762568Z","shell.execute_reply":"2024-12-30T09:14:07.887951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check for infinite values\n\nprint(np.isfinite(X).all().all())  # Check for infinite values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:07.889886Z","iopub.execute_input":"2024-12-30T09:14:07.890245Z","iopub.status.idle":"2024-12-30T09:14:07.913712Z","shell.execute_reply.started":"2024-12-30T09:14:07.89021Z","shell.execute_reply":"2024-12-30T09:14:07.912808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#fixing infite values\n\nX = X.replace([np.inf, -np.inf], np.nan)  # Replace inf with NaN\nX = X.dropna()  # Drop rows with NaN","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:07.91478Z","iopub.execute_input":"2024-12-30T09:14:07.915201Z","iopub.status.idle":"2024-12-30T09:14:08.367986Z","shell.execute_reply.started":"2024-12-30T09:14:07.915164Z","shell.execute_reply":"2024-12-30T09:14:08.36709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(X.isnull().sum())  # Should return 0 for all columns if no NaN values remain","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:08.369032Z","iopub.execute_input":"2024-12-30T09:14:08.369324Z","iopub.status.idle":"2024-12-30T09:14:08.407959Z","shell.execute_reply.started":"2024-12-30T09:14:08.369298Z","shell.execute_reply":"2024-12-30T09:14:08.406855Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We can now calculate the VIF once again.","metadata":{}},{"cell_type":"code","source":"from statsmodels.stats.outliers_influence import variance_inflation_factor\n\nvif_data = pd.DataFrame()\nvif_data['feature'] = X.columns\nvif_data['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]\n\nprint(vif_data)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:14:08.409016Z","iopub.execute_input":"2024-12-30T09:14:08.4093Z","iopub.status.idle":"2024-12-30T09:15:43.620565Z","shell.execute_reply.started":"2024-12-30T09:14:08.409275Z","shell.execute_reply":"2024-12-30T09:15:43.617749Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The VIF results are acceptable.absabs\n\nWe can either train the model with this dataset or deep dive even further to try and remove outliers from the df to reduce overfitting and noise.","metadata":{}},{"cell_type":"code","source":"#Recursive Feature Elimination (RFE) to try and identify the noisy features \ny = encoded_df['Premium Amount']\nX = encoded_df.drop(columns=['Premium Amount'])\n\nfrom sklearn.feature_selection import RFE\nfrom sklearn.linear_model import LinearRegression\n\nmodel = LinearRegression()\nrfe = RFE(model, n_features_to_select=15)  # Select top 15 features\nrfe.fit(X, y)\n\nprint(\"Feature Ranking:\", rfe.ranking_)  # Features ranked 1 are most important\nprint(\"Selected Features:\", X.columns[rfe.support_])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:15:43.6217Z","iopub.execute_input":"2024-12-30T09:15:43.622169Z","iopub.status.idle":"2024-12-30T09:15:59.712984Z","shell.execute_reply.started":"2024-12-30T09:15:43.622125Z","shell.execute_reply":"2024-12-30T09:15:59.711813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_selected = X.loc[:, rfe.support_]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:15:59.713882Z","iopub.execute_input":"2024-12-30T09:15:59.714207Z","iopub.status.idle":"2024-12-30T09:15:59.795299Z","shell.execute_reply.started":"2024-12-30T09:15:59.714177Z","shell.execute_reply":"2024-12-30T09:15:59.794205Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Before fitting the model we need to transform the test_df the same way we did with the train_df.","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:15:59.796189Z","iopub.execute_input":"2024-12-30T09:15:59.796457Z","iopub.status.idle":"2024-12-30T09:15:59.816333Z","shell.execute_reply.started":"2024-12-30T09:15:59.796435Z","shell.execute_reply":"2024-12-30T09:15:59.815315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_df.info())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:15:59.817374Z","iopub.execute_input":"2024-12-30T09:15:59.817898Z","iopub.status.idle":"2024-12-30T09:16:00.295794Z","shell.execute_reply.started":"2024-12-30T09:15:59.817864Z","shell.execute_reply":"2024-12-30T09:16:00.294946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_df.isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:00.296611Z","iopub.execute_input":"2024-12-30T09:16:00.296864Z","iopub.status.idle":"2024-12-30T09:16:00.710581Z","shell.execute_reply.started":"2024-12-30T09:16:00.296827Z","shell.execute_reply":"2024-12-30T09:16:00.709477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(test_df.dtypes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:00.711756Z","iopub.execute_input":"2024-12-30T09:16:00.712126Z","iopub.status.idle":"2024-12-30T09:16:00.718338Z","shell.execute_reply.started":"2024-12-30T09:16:00.712098Z","shell.execute_reply":"2024-12-30T09:16:00.717219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Dealing with missing data\ntest_df['Age'] = test_df['Age'].fillna(test_df['Age'].mean())\ntest_df['Annual Income'] = test_df['Annual Income'].fillna(test_df['Annual Income'].mean())\ntest_df['Marital Status'] = test_df['Marital Status'].fillna('Single')\ntest_df['Number of Dependents'] = test_df['Number of Dependents'].fillna(0)\ntest_df['Occupation'] = test_df['Occupation'].fillna('Unemployed')\ntest_df['Health Score'] = test_df['Health Score'].fillna(test_df['Health Score'].mean())\ntest_df['Previous Claims'] = test_df['Previous Claims'].fillna(0)\ntest_df['Vehicle Age'] = test_df['Vehicle Age'].fillna(0)\ntest_df['Credit Score'] = test_df['Credit Score'].fillna(test_df['Credit Score'].mean())\ntest_df['Insurance Duration'] = test_df['Insurance Duration'].fillna(test_df['Insurance Duration'].mean())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:00.719173Z","iopub.execute_input":"2024-12-30T09:16:00.719419Z","iopub.status.idle":"2024-12-30T09:16:00.920701Z","shell.execute_reply.started":"2024-12-30T09:16:00.719397Z","shell.execute_reply":"2024-12-30T09:16:00.919907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#dropping customer feedback\ntest_df = test_df.drop(columns=['Customer Feedback'])\nprint(test_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:00.921588Z","iopub.execute_input":"2024-12-30T09:16:00.921865Z","iopub.status.idle":"2024-12-30T09:16:01.072627Z","shell.execute_reply.started":"2024-12-30T09:16:00.921827Z","shell.execute_reply":"2024-12-30T09:16:01.071572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#converting the date to datetime\ntest_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:01.073725Z","iopub.execute_input":"2024-12-30T09:16:01.074153Z","iopub.status.idle":"2024-12-30T09:16:01.387174Z","shell.execute_reply.started":"2024-12-30T09:16:01.074114Z","shell.execute_reply":"2024-12-30T09:16:01.386265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#confirm the changes\nprint(train_df['Policy Start Date'].dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:01.388084Z","iopub.execute_input":"2024-12-30T09:16:01.388433Z","iopub.status.idle":"2024-12-30T09:16:01.394074Z","shell.execute_reply.started":"2024-12-30T09:16:01.388398Z","shell.execute_reply":"2024-12-30T09:16:01.393057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#encoding the categorical columns Marital Status and Occupation\ntest_encoded_df = pd.get_dummies(test_df, columns=['Gender','Marital Status', 'Occupation','Location','Property Type','Smoking Status'], drop_first = True)\n\nprint(test_encoded_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:01.39516Z","iopub.execute_input":"2024-12-30T09:16:01.395466Z","iopub.status.idle":"2024-12-30T09:16:02.02493Z","shell.execute_reply.started":"2024-12-30T09:16:01.395442Z","shell.execute_reply":"2024-12-30T09:16:02.023862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#encoding the ordinal categorical values for Education Level and Exercise Frequency\n\neducation_mapping = {'High School': 0, 'Bachelor\\'s': 1, 'Master\\'s': 2, 'PhD': 3}\ntest_encoded_df['Education Level'] = test_df['Education Level'].map(education_mapping)\n\nexercise_mapping = {'Rarely': 0, 'Monthly': 1, 'Weekly': 2, 'Daily': 3}\ntest_encoded_df['Exercise Frequency'] = test_df['Exercise Frequency'].map(exercise_mapping)\n\npolicy_mapping = {'Basic': 0, 'Comprehensive': 1, 'Premium': 2}\ntest_encoded_df['Policy Type'] = test_df['Policy Type'].map(policy_mapping)\n\nprint(test_encoded_df[['Education Level','Exercise Frequency', 'Policy Type']].head())\n\nprint(test_encoded_df.dtypes)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.026052Z","iopub.execute_input":"2024-12-30T09:16:02.026321Z","iopub.status.idle":"2024-12-30T09:16:02.216179Z","shell.execute_reply.started":"2024-12-30T09:16:02.026298Z","shell.execute_reply":"2024-12-30T09:16:02.21511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#transforming the start date column into an age account column\nimport datetime\ntest_encoded_df['Policy Duration (days)'] = (datetime.datetime.now() - test_encoded_df['Policy Start Date']).dt.days\n\ntest_encoded_df = test_encoded_df.drop(columns=['Policy Start Date'])\n\nprint(test_encoded_df['Policy Duration (days)'].head())\n\nprint(test_encoded_df.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.217019Z","iopub.execute_input":"2024-12-30T09:16:02.217342Z","iopub.status.idle":"2024-12-30T09:16:02.303745Z","shell.execute_reply.started":"2024-12-30T09:16:02.217314Z","shell.execute_reply":"2024-12-30T09:16:02.302694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Select numerical columns\nnumerical_columns = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score',\n                     'Policy Duration (days)', 'Credit Score', 'Insurance Duration']\n\n# Initialize scaler and apply to numerical columns\nscaler = StandardScaler()\ntest_encoded_df[numerical_columns] = scaler.fit_transform(test_encoded_df[numerical_columns])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.304747Z","iopub.execute_input":"2024-12-30T09:16:02.305121Z","iopub.status.idle":"2024-12-30T09:16:02.451225Z","shell.execute_reply.started":"2024-12-30T09:16:02.305081Z","shell.execute_reply":"2024-12-30T09:16:02.450251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example: Interaction between Age and Health Score\ntest_encoded_df['Age_Health_Interaction'] = test_encoded_df['Age'] * test_encoded_df['Health Score']\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.452097Z","iopub.execute_input":"2024-12-30T09:16:02.452391Z","iopub.status.idle":"2024-12-30T09:16:02.465329Z","shell.execute_reply.started":"2024-12-30T09:16:02.452366Z","shell.execute_reply":"2024-12-30T09:16:02.464355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Separate features and target in the training data\nX_train = encoded_df.drop(columns=['Premium Amount'])  # Exclude the target column from features\ny_train = encoded_df['Premium Amount']  # Target column\n\nX_test = test_encoded_df\n\nprint(\"Columns in X_train:\")\nprint(X_train.columns)\n\nprint(\"\\nColumns in X_test:\")\nprint(X_test.columns)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.466223Z","iopub.execute_input":"2024-12-30T09:16:02.466666Z","iopub.status.idle":"2024-12-30T09:16:02.578572Z","shell.execute_reply.started":"2024-12-30T09:16:02.466629Z","shell.execute_reply":"2024-12-30T09:16:02.577668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Number of columns in X_train: {X_train.shape[1]}\")\nprint(f\"Number of columns in X_test: {X_test.shape[1]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.579483Z","iopub.execute_input":"2024-12-30T09:16:02.579857Z","iopub.status.idle":"2024-12-30T09:16:02.585136Z","shell.execute_reply.started":"2024-12-30T09:16:02.579812Z","shell.execute_reply":"2024-12-30T09:16:02.584148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Drop 'id' from both X_train and X_test\nX_train = encoded_df.drop(columns=['id', 'Premium Amount'])  # Drop 'id' and the target\ny_train = encoded_df['Premium Amount']  # Target variable\n\nX_test = test_encoded_df.drop(columns=['id'])  # Drop 'id' from the test data\n\nfrom sklearn.linear_model import LinearRegression\n\n# Fit the Linear Regression model\nmodel = LinearRegression()\nmodel.fit(X_train, y_train)\n\n# Predict Premium Amount\ny_pred = model.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:16:02.586329Z","iopub.execute_input":"2024-12-30T09:16:02.586704Z","iopub.status.idle":"2024-12-30T09:16:04.915445Z","shell.execute_reply.started":"2024-12-30T09:16:02.586667Z","shell.execute_reply":"2024-12-30T09:16:04.912896Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Next steps:\n\ntesting manually the model\ndownload the file to upload to Kaggle","metadata":{}},{"cell_type":"code","source":"# Example manual input with the specified columns\nmanual_input = pd.DataFrame({\n    'Age': [30],\n    'Annual Income': [60000],\n    'Number of Dependents': [1],\n    'Education Level': [2],  # Assuming 2 = Master's degree\n    'Health Score': [75],\n    'Policy Type': [1],  # Assuming 1 = Comprehensive\n    'Previous Claims': [0],\n    'Vehicle Age': [3],\n    'Credit Score': [720],\n    'Insurance Duration': [10],\n    'Exercise Frequency': [2],  # Assuming 2 = Weekly\n    'Gender_Male': [1],  # 1 = Male, 0 = Female\n    'Marital Status_Married': [1],  # 1 = Married, 0 = Not Married\n    'Marital Status_Single': [0],  # 1 = Single, 0 = Not Single\n    'Occupation_Self-Employed': [0],  # 1 = Self-Employed, 0 = Otherwise\n    'Occupation_Unemployed': [0],  # 1 = Unemployed, 0 = Otherwise\n    'Location_Suburban': [1],  # 1 = Suburban, 0 = Otherwise\n    'Location_Urban': [0],  # 1 = Urban, 0 = Otherwise\n    'Property Type_Condo': [1],  # 1 = Condo, 0 = Otherwise\n    'Property Type_House': [0],  # 1 = House, 0 = Otherwise\n    'Smoking Status_Yes': [0],  # 1 = Smoker, 0 = Non-Smoker\n    'Policy Duration (days)': [365],  # 1 Year Policy Duration\n    'Age_Health_Interaction': [30 * 75],  # Interaction term (Age * Health Score)\n})\n\nmanual_input","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:28:21.432428Z","iopub.execute_input":"2024-12-30T09:28:21.432829Z","iopub.status.idle":"2024-12-30T09:28:21.449999Z","shell.execute_reply.started":"2024-12-30T09:28:21.432802Z","shell.execute_reply":"2024-12-30T09:28:21.4487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\n# Assuming the scaler was already fitted on encoded_df or X_train\n# scaler = StandardScaler()\n# scaler.fit(X_train)  # This step is already done during preprocessing\n\n# Apply the scaling to the manual_input\nmanual_input[numerical_columns] = scaler.transform(manual_input[numerical_columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:31:38.72038Z","iopub.execute_input":"2024-12-30T09:31:38.720755Z","iopub.status.idle":"2024-12-30T09:31:38.729219Z","shell.execute_reply.started":"2024-12-30T09:31:38.720729Z","shell.execute_reply":"2024-12-30T09:31:38.728045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predict Premium Amount\nmanual_prediction = model.predict(manual_input)\n\nprint(f\"Predicted Premium Amount: {manual_prediction[0]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T09:32:20.342793Z","iopub.execute_input":"2024-12-30T09:32:20.343165Z","iopub.status.idle":"2024-12-30T09:32:20.351119Z","shell.execute_reply.started":"2024-12-30T09:32:20.343138Z","shell.execute_reply":"2024-12-30T09:32:20.350181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a DataFrame for the submission file\nsubmission = pd.DataFrame({\n    'id': test_encoded_df['id'],  # ID column from the test set\n    'Premium Amount': y_pred      # Predicted Premium Amount values\n})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T10:17:23.968898Z","iopub.execute_input":"2024-12-30T10:17:23.969324Z","iopub.status.idle":"2024-12-30T10:17:23.977239Z","shell.execute_reply.started":"2024-12-30T10:17:23.969292Z","shell.execute_reply":"2024-12-30T10:17:23.976138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save the submission file as 'submission.csv'\nsubmission.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-30T10:17:46.090451Z","iopub.execute_input":"2024-12-30T10:17:46.090822Z","iopub.status.idle":"2024-12-30T10:17:47.70928Z","shell.execute_reply.started":"2024-12-30T10:17:46.090791Z","shell.execute_reply":"2024-12-30T10:17:47.708229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}