{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feature Description:\n\n1. **id**: A unique identifier for each record or customer.\\\n2. **Age**: The age of the customer. May have missing values (indicated by 1181295 non-null out of 1200000).\\\n3. **Gender**: The gender of the customer (e.g., Male, Female).\\\n4. **Annual Income**: The customer's yearly income in a monetary value, with some missing data.\\\n5. **Marital Status**: Whether the customer is single, married, divorced, etc. Missing for some records.\\\n6. **Number of Dependents**: Number of dependents (e.g., children or others relying on the customer).\\\n7. **Education Level**: The customer's level of education (e.g., High School, Bachelor's, Master's).\\\n8. **Occupation**: The customer's profession or job type, with significant missing values.\\\n9. **Health Score**: A numerical representation of the customer's health condition.\\\n10. **Location**: Geographic location or region of the customer.\\\n11. **Policy Type**: The type of insurance policy purchased (e.g., health, auto, life).\\\n12. **Previous Claims**: Number of previous insurance claims filed by the customer.\\\n13. **Vehicle Age**: Age of the insured vehicle (if applicable).\\\n14. **Credit Score**: A score representing the financial trustworthiness of the customer.\\\n15. **Insurance Duration**: Duration of the current insurance policy in years.\\\n16. **Policy Start Date**: The start date of the insurance policy, stored as an object (likely a date string).\\\n17. **Customer Feedback**: Customer’s qualitative feedback (e.g., satisfaction level).\\\n18. **Smoking Status**: Whether the customer smokes (e.g., Yes, No).\\\n19. **Exercise Frequency**: The frequency of physical activity by the customer.\\\n20. **Property Type**: Type of property insured, if applicable (e.g., house, apartment).\\\n21. **Premium Amount**: The insurance premium amount the customer pays.","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:35.341247Z","iopub.execute_input":"2024-12-10T07:31:35.342177Z","iopub.status.idle":"2024-12-10T07:31:35.35415Z","shell.execute_reply.started":"2024-12-10T07:31:35.342109Z","shell.execute_reply":"2024-12-10T07:31:35.352007Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:35.356018Z","iopub.execute_input":"2024-12-10T07:31:35.356531Z","iopub.status.idle":"2024-12-10T07:31:36.411616Z","shell.execute_reply.started":"2024-12-10T07:31:35.35647Z","shell.execute_reply":"2024-12-10T07:31:36.410376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\norig = pd.read_csv('/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:36.414583Z","iopub.execute_input":"2024-12-10T07:31:36.41526Z","iopub.status.idle":"2024-12-10T07:31:45.058779Z","shell.execute_reply.started":"2024-12-10T07:31:36.415207Z","shell.execute_reply":"2024-12-10T07:31:45.0572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.shape, test.shape, orig.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:45.060519Z","iopub.execute_input":"2024-12-10T07:31:45.0609Z","iopub.status.idle":"2024-12-10T07:31:45.069404Z","shell.execute_reply.started":"2024-12-10T07:31:45.060863Z","shell.execute_reply":"2024-12-10T07:31:45.068053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print('Train Set Info:')\nprint('----'*22)\ntrain.info()\nprint('----'*22)\nprint('Test Set Info:')\nprint('----'*22)\ntest.info()\nprint('----'*22)\nprint('Original Set Info:')\nprint('----'*22)\norig.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:45.072166Z","iopub.execute_input":"2024-12-10T07:31:45.072559Z","iopub.status.idle":"2024-12-10T07:31:46.346144Z","shell.execute_reply.started":"2024-12-10T07:31:45.072504Z","shell.execute_reply":"2024-12-10T07:31:46.344956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Set Statistical parameters","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:46.347705Z","iopub.execute_input":"2024-12-10T07:31:46.348048Z","iopub.status.idle":"2024-12-10T07:31:46.377052Z","shell.execute_reply.started":"2024-12-10T07:31:46.348013Z","shell.execute_reply":"2024-12-10T07:31:46.375733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:46.378245Z","iopub.execute_input":"2024-12-10T07:31:46.378577Z","iopub.status.idle":"2024-12-10T07:31:47.074867Z","shell.execute_reply.started":"2024-12-10T07:31:46.378543Z","shell.execute_reply":"2024-12-10T07:31:47.073826Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.describe(include='O').T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:47.076332Z","iopub.execute_input":"2024-12-10T07:31:47.076766Z","iopub.status.idle":"2024-12-10T07:31:49.37916Z","shell.execute_reply.started":"2024-12-10T07:31:47.076718Z","shell.execute_reply":"2024-12-10T07:31:49.377731Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Set Statistical parameters","metadata":{}},{"cell_type":"code","source":"test.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:49.380765Z","iopub.execute_input":"2024-12-10T07:31:49.381149Z","iopub.status.idle":"2024-12-10T07:31:49.403907Z","shell.execute_reply.started":"2024-12-10T07:31:49.381079Z","shell.execute_reply":"2024-12-10T07:31:49.402697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:49.405316Z","iopub.execute_input":"2024-12-10T07:31:49.40579Z","iopub.status.idle":"2024-12-10T07:31:49.841609Z","shell.execute_reply.started":"2024-12-10T07:31:49.405736Z","shell.execute_reply":"2024-12-10T07:31:49.840441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.describe(include='O').T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:49.845022Z","iopub.execute_input":"2024-12-10T07:31:49.845388Z","iopub.status.idle":"2024-12-10T07:31:51.375554Z","shell.execute_reply.started":"2024-12-10T07:31:49.845351Z","shell.execute_reply":"2024-12-10T07:31:51.374338Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Original Set Statistical parameters","metadata":{}},{"cell_type":"code","source":"orig.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:51.376734Z","iopub.execute_input":"2024-12-10T07:31:51.377112Z","iopub.status.idle":"2024-12-10T07:31:51.397857Z","shell.execute_reply.started":"2024-12-10T07:31:51.377062Z","shell.execute_reply":"2024-12-10T07:31:51.396689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"orig.describe().T","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:51.399552Z","iopub.execute_input":"2024-12-10T07:31:51.400007Z","iopub.status.idle":"2024-12-10T07:31:51.569794Z","shell.execute_reply.started":"2024-12-10T07:31:51.399958Z","shell.execute_reply":"2024-12-10T07:31:51.568682Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Categorical Columns","metadata":{}},{"cell_type":"code","source":"train = train.drop('id', axis=1)\ntest_id = test['id']\ntest = test.drop('id', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:51.571162Z","iopub.execute_input":"2024-12-10T07:31:51.571467Z","iopub.status.idle":"2024-12-10T07:31:51.854777Z","shell.execute_reply.started":"2024-12-10T07:31:51.571436Z","shell.execute_reply":"2024-12-10T07:31:51.85346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = [col for col in train.select_dtypes('O')]\n\nprint('Train Set:\\n')\nfor col in cat_cols:\n    print(col, ':', train[col].unique())\nprint('--'*45)\nprint('Test Set:\\n')\nfor col in cat_cols:\n    print(col, ':', test[col].unique())\nprint('--'*45)\nprint('Original Set:\\n')\nfor col in cat_cols:\n    print(col, ':', orig[col].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:51.85612Z","iopub.execute_input":"2024-12-10T07:31:51.856461Z","iopub.status.idle":"2024-12-10T07:31:53.766494Z","shell.execute_reply.started":"2024-12-10T07:31:51.856426Z","shell.execute_reply":"2024-12-10T07:31:53.765316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#check for duplicated values\ntrain.duplicated().sum(), test.duplicated().sum(), orig.duplicated().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:53.768009Z","iopub.execute_input":"2024-12-10T07:31:53.768481Z","iopub.status.idle":"2024-12-10T07:31:57.082177Z","shell.execute_reply.started":"2024-12-10T07:31:53.768429Z","shell.execute_reply":"2024-12-10T07:31:57.081171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"orig['Policy Start Date'].head()\n#just check the detail and it seems all the times are equal so just consider the month date and year","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:57.083445Z","iopub.execute_input":"2024-12-10T07:31:57.083886Z","iopub.status.idle":"2024-12-10T07:31:57.092875Z","shell.execute_reply.started":"2024-12-10T07:31:57.083832Z","shell.execute_reply":"2024-12-10T07:31:57.091642Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Null Values","metadata":{}},{"cell_type":"code","source":"train.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:57.094317Z","iopub.execute_input":"2024-12-10T07:31:57.094745Z","iopub.status.idle":"2024-12-10T07:31:57.734253Z","shell.execute_reply.started":"2024-12-10T07:31:57.094697Z","shell.execute_reply":"2024-12-10T07:31:57.733167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:57.735539Z","iopub.execute_input":"2024-12-10T07:31:57.735838Z","iopub.status.idle":"2024-12-10T07:31:58.159023Z","shell.execute_reply.started":"2024-12-10T07:31:57.735808Z","shell.execute_reply":"2024-12-10T07:31:58.157972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"orig.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:58.160472Z","iopub.execute_input":"2024-12-10T07:31:58.160895Z","iopub.status.idle":"2024-12-10T07:31:58.315761Z","shell.execute_reply.started":"2024-12-10T07:31:58.160859Z","shell.execute_reply":"2024-12-10T07:31:58.314707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Policy Start Date\n\ntrain['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])\ntest['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])\norig['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])\n\ntrain['year'] = train['Policy Start Date'].dt.year\ntrain['month'] = train['Policy Start Date'].dt.month\ntrain['day'] = train['Policy Start Date'].dt.dayofweek\n\ntest['year'] = test['Policy Start Date'].dt.year\ntest['month'] = test['Policy Start Date'].dt.month\ntest['day'] = test['Policy Start Date'].dt.dayofweek\n\norig['year'] = orig['Policy Start Date'].dt.year\norig['month'] = orig['Policy Start Date'].dt.month\norig['day'] = orig['Policy Start Date'].dt.dayofweek","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:58.317265Z","iopub.execute_input":"2024-12-10T07:31:58.317701Z","iopub.status.idle":"2024-12-10T07:31:59.097652Z","shell.execute_reply.started":"2024-12-10T07:31:58.317651Z","shell.execute_reply":"2024-12-10T07:31:59.096571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_policy = train['Policy Start Date']\ntest_policy = test['Policy Start Date']\norig_policy = orig['Policy Start Date']\n\ntrain = train.drop('Policy Start Date', axis=1)\ntest = test.drop('Policy Start Date', axis=1)\norig = orig.drop('Policy Start Date', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:59.098901Z","iopub.execute_input":"2024-12-10T07:31:59.099208Z","iopub.status.idle":"2024-12-10T07:31:59.415933Z","shell.execute_reply.started":"2024-12-10T07:31:59.099177Z","shell.execute_reply":"2024-12-10T07:31:59.414826Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualization","metadata":{}},{"cell_type":"markdown","source":"# Numeric Column Visualization ","metadata":{}},{"cell_type":"code","source":"num_cols = [col for col in train.select_dtypes('number')]\nlen(num_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:59.417443Z","iopub.execute_input":"2024-12-10T07:31:59.417924Z","iopub.status.idle":"2024-12-10T07:31:59.47376Z","shell.execute_reply.started":"2024-12-10T07:31:59.417878Z","shell.execute_reply":"2024-12-10T07:31:59.472687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in num_cols:\n    sns.histplot(train[col], bins=50, ax=axs[index])\n    index += 1\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:31:59.474882Z","iopub.execute_input":"2024-12-10T07:31:59.47521Z","iopub.status.idle":"2024-12-10T07:32:12.061524Z","shell.execute_reply.started":"2024-12-10T07:31:59.475179Z","shell.execute_reply":"2024-12-10T07:32:12.060413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in num_cols:\n    sns.boxplot(y=col,data=train, ax=axs[index])\n    index += 1\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:12.063136Z","iopub.execute_input":"2024-12-10T07:32:12.063557Z","iopub.status.idle":"2024-12-10T07:32:14.263693Z","shell.execute_reply.started":"2024-12-10T07:32:12.063514Z","shell.execute_reply":"2024-12-10T07:32:14.262504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in num_cols:\n    sns.histplot(orig[col], bins=50, ax=axs[index])\n    index += 1\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:14.265257Z","iopub.execute_input":"2024-12-10T07:32:14.265691Z","iopub.status.idle":"2024-12-10T07:32:19.179267Z","shell.execute_reply.started":"2024-12-10T07:32:14.265644Z","shell.execute_reply":"2024-12-10T07:32:19.178146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in num_cols:\n    sns.boxplot(data=orig,y=col, ax=axs[index]);\n    index += 1\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:19.180829Z","iopub.execute_input":"2024-12-10T07:32:19.181289Z","iopub.status.idle":"2024-12-10T07:32:20.770219Z","shell.execute_reply.started":"2024-12-10T07:32:19.18124Z","shell.execute_reply":"2024-12-10T07:32:20.768771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in test.select_dtypes('number').columns:\n    sns.histplot(test[col], bins=50, ax=axs[index])\n    index += 1\n\nfor ax in axs[index:]:\n    ax.axis('off')\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:20.771726Z","iopub.execute_input":"2024-12-10T07:32:20.772152Z","iopub.status.idle":"2024-12-10T07:32:29.163252Z","shell.execute_reply.started":"2024-12-10T07:32:20.772074Z","shell.execute_reply":"2024-12-10T07:32:29.161973Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(dpi=100)\n\nfig, axs = plt.subplots(nrows=4,ncols=3,figsize=(10,6))\nindex = 0\naxs = axs.flatten()\n\nfor col in test.select_dtypes('number').columns:\n    sns.boxplot(data=test,y= col, ax=axs[index])\n    index += 1\n\nfor ax in axs[index:]:\n    ax.axis('off')\n    \nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:29.168955Z","iopub.execute_input":"2024-12-10T07:32:29.169333Z","iopub.status.idle":"2024-12-10T07:32:30.780034Z","shell.execute_reply.started":"2024-12-10T07:32:29.169297Z","shell.execute_reply":"2024-12-10T07:32:30.7789Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Category Columns Visulaization","metadata":{}},{"cell_type":"code","source":"cat_cols = [col for col in train.select_dtypes('O') if col!='Policy Start Date']\n\nfig, axs = plt.subplots(nrows=5,ncols=2,figsize=(12,20))\nindex = 0\naxs = axs.flatten()\nfor col in cat_cols:\n    sns.countplot(train,x=col, ax=axs[index])\n    index += 1\n\nplt.tight_layout();","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:30.78132Z","iopub.execute_input":"2024-12-10T07:32:30.781699Z","iopub.status.idle":"2024-12-10T07:32:39.290507Z","shell.execute_reply.started":"2024-12-10T07:32:30.781664Z","shell.execute_reply":"2024-12-10T07:32:39.289269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12,8))\ncorr_matrix = train.select_dtypes('number').corr()\nsns.heatmap(corr_matrix, annot=True, fmt='.1%')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:39.292167Z","iopub.execute_input":"2024-12-10T07:32:39.292491Z","iopub.status.idle":"2024-12-10T07:32:40.869337Z","shell.execute_reply.started":"2024-12-10T07:32:39.29246Z","shell.execute_reply":"2024-12-10T07:32:40.868169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_order = ['Age', 'Gender', 'Annual Income', 'Marital Status',\n       'Number of Dependents', 'Education Level', 'Occupation', 'Health Score',\n       'Location', 'Policy Type', 'Previous Claims', 'Vehicle Age',\n       'Credit Score', 'Insurance Duration', 'Customer Feedback',\n       'Smoking Status', 'Exercise Frequency', 'Property Type','year', 'month', 'day','Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:40.870903Z","iopub.execute_input":"2024-12-10T07:32:40.871673Z","iopub.status.idle":"2024-12-10T07:32:40.877527Z","shell.execute_reply.started":"2024-12-10T07:32:40.871624Z","shell.execute_reply":"2024-12-10T07:32:40.876368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train[new_order]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:40.879063Z","iopub.execute_input":"2024-12-10T07:32:40.879415Z","iopub.status.idle":"2024-12-10T07:32:41.028579Z","shell.execute_reply.started":"2024-12-10T07:32:40.879384Z","shell.execute_reply":"2024-12-10T07:32:41.027568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"set(train.columns)-set(orig.columns), set(orig.columns)-set(train.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:41.030043Z","iopub.execute_input":"2024-12-10T07:32:41.030896Z","iopub.status.idle":"2024-12-10T07:32:41.038185Z","shell.execute_reply.started":"2024-12-10T07:32:41.030843Z","shell.execute_reply":"2024-12-10T07:32:41.037116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"orig = orig[new_order]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:41.039534Z","iopub.execute_input":"2024-12-10T07:32:41.040002Z","iopub.status.idle":"2024-12-10T07:32:41.078707Z","shell.execute_reply.started":"2024-12-10T07:32:41.039954Z","shell.execute_reply":"2024-12-10T07:32:41.077504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Concat train + orig\n\ndata = pd.concat([train,orig], axis=0)\ndata.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:41.079861Z","iopub.execute_input":"2024-12-10T07:32:41.080202Z","iopub.status.idle":"2024-12-10T07:32:41.242078Z","shell.execute_reply.started":"2024-12-10T07:32:41.080168Z","shell.execute_reply":"2024-12-10T07:32:41.240877Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:41.243709Z","iopub.execute_input":"2024-12-10T07:32:41.244532Z","iopub.status.idle":"2024-12-10T07:32:41.251297Z","shell.execute_reply.started":"2024-12-10T07:32:41.244467Z","shell.execute_reply":"2024-12-10T07:32:41.250263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in cat_cols:\n    print(col, data[col].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:41.252764Z","iopub.execute_input":"2024-12-10T07:32:41.253109Z","iopub.status.idle":"2024-12-10T07:32:42.022653Z","shell.execute_reply.started":"2024-12-10T07:32:41.253061Z","shell.execute_reply":"2024-12-10T07:32:42.021199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# data['Gender'] = data['Gender'].map({'Female':0, 'Male':1})\n# data['Marital Status'] = data['Marital Status'].map({'Married':1, 'Divorced':2, 'Single':0})\n# data['Education Level'] = data['Education Level'].map({\"Bachelor's\":0, \"Master's\":1, 'High School':2, 'PhD':3})\n# data['Occupation'] = data['Occupation'].map({'Self-Employed':2,'Employed':1, 'Unemployed':0})\n# data['Location'] = data['Location'].map({'Urban':2, 'Rural':0, 'Suburban':1})\n# data['Policy Type'] = data['Policy Type'].map({'Premium':2, 'Comprehensive':1, 'Basic':0})\n# data['Customer Feedback'] = data['Customer Feedback'].map({'Poor':0, 'Average':1, 'Good':2})\n# data['Smoking Status'] = data['Smoking Status'].map({'No':0, 'Yes':1})\n# data['Exercise Frequency'] = data['Exercise Frequency'].map({'Weekly':2, 'Monthly':1, 'Daily':3, 'Rarely':0})\n# data['Property Type'] = data['Property Type'].map({'House':2, 'Apartment':0, 'Condo':1})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:42.023925Z","iopub.execute_input":"2024-12-10T07:32:42.024285Z","iopub.status.idle":"2024-12-10T07:32:42.029929Z","shell.execute_reply.started":"2024-12-10T07:32:42.02425Z","shell.execute_reply":"2024-12-10T07:32:42.02867Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# test['Gender'] = test['Gender'].map({'Female':0, 'Male':1})\n# test['Marital Status'] = test['Marital Status'].map({'Married':1, 'Divorced':2, 'Single':0})\n# test['Education Level'] = test['Education Level'].map({\"Bachelor's\":0, \"Master's\":1, 'High School':2, 'PhD':3})\n# test['Occupation'] = test['Occupation'].map({'Self-Employed':2,'Employed':1, 'Unemployed':0})\n# test['Location'] = test['Location'].map({'Urban':2, 'Rural':0, 'Suburban':1})\n# test['Policy Type'] = test['Policy Type'].map({'Premium':2, 'Comprehensive':1, 'Basic':0})\n# test['Customer Feedback'] = test['Customer Feedback'].map({'Poor':0, 'Average':1, 'Good':2})\n# test['Smoking Status'] = test['Smoking Status'].map({'No':0, 'Yes':1})\n# test['Exercise Frequency'] = test['Exercise Frequency'].map({'Weekly':2, 'Monthly':1, 'Daily':3, 'Rarely':0})\n# test['Property Type'] = test['Property Type'].map({'House':2, 'Apartment':0, 'Condo':1})\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:42.031702Z","iopub.execute_input":"2024-12-10T07:32:42.032062Z","iopub.status.idle":"2024-12-10T07:32:42.040889Z","shell.execute_reply.started":"2024-12-10T07:32:42.032026Z","shell.execute_reply":"2024-12-10T07:32:42.039657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['Premium Amount'].mean(), data['Premium Amount'].median()\ndata['Premium Amount'].skew()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:42.042366Z","iopub.execute_input":"2024-12-10T07:32:42.042722Z","iopub.status.idle":"2024-12-10T07:32:42.145301Z","shell.execute_reply.started":"2024-12-10T07:32:42.042687Z","shell.execute_reply":"2024-12-10T07:32:42.144164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['Premium Amount'] = data['Premium Amount'].fillna(data['Premium Amount'].median())\n\n#data['Annual Income'] = data['Annual Income'].fillna(data['Annual Income'].median())\n#test['Annual Income'] = test['Annual Income'].fillna(test['Annual Income'].median())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:32:42.146615Z","iopub.execute_input":"2024-12-10T07:32:42.146954Z","iopub.status.idle":"2024-12-10T07:32:42.195438Z","shell.execute_reply.started":"2024-12-10T07:32:42.146921Z","shell.execute_reply":"2024-12-10T07:32:42.194226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#data['Premium Amount'] = np.log1p(data['Premium Amount'])\n\ndata['Annual Income'] =  np.log1p(data['Annual Income'])\ntest['Annual Income'] = np.log1p(test['Annual Income'])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['Income Per Dependent'] = data.apply(\n    lambda row: row['Annual Income'] if row['Number of Dependents'] == 0 else row['Annual Income'] / row['Number of Dependents'],\n    axis=1\n)\n\ntest['Income Per Dependent'] = test.apply(\n    lambda row: row['Annual Income'] if row['Number of Dependents'] == 0 else row['Annual Income'] / row['Number of Dependents'],\n    axis=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:33:04.070892Z","iopub.execute_input":"2024-12-10T07:33:04.071307Z","iopub.status.idle":"2024-12-10T07:33:30.264717Z","shell.execute_reply.started":"2024-12-10T07:33:04.071267Z","shell.execute_reply":"2024-12-10T07:33:30.263668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['LifestyleIndicator'] = data.apply(\n    lambda row: 1 if row['Smoking Status'] == \"Yes\" and row['Exercise Frequency'] == \"Rarely\" else 0,\n    axis=1)\n\ntest['LifestyleIndicator'] = test.apply(\n    lambda row: 1 if row['Smoking Status'] == \"Yes\" and row['Exercise Frequency'] == \"Rarely\" else 0,\n    axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:33:30.266443Z","iopub.execute_input":"2024-12-10T07:33:30.266866Z","iopub.status.idle":"2024-12-10T07:33:49.504799Z","shell.execute_reply.started":"2024-12-10T07:33:30.266829Z","shell.execute_reply":"2024-12-10T07:33:49.503651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data['FamilySize'] = data.apply(\n    lambda row: row['Number of Dependents'] + (2 if row['Marital Status'] == \"Married\" else 1),\n    axis=1\n)\ntest['FamilySize'] = test.apply(\n    lambda row: row['Number of Dependents'] + (2 if row['Marital Status'] == \"Married\" else 1),\n    axis=1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:33:49.506225Z","iopub.execute_input":"2024-12-10T07:33:49.506583Z","iopub.status.idle":"2024-12-10T07:34:11.828822Z","shell.execute_reply.started":"2024-12-10T07:33:49.50654Z","shell.execute_reply":"2024-12-10T07:34:11.827588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:11.831709Z","iopub.execute_input":"2024-12-10T07:34:11.832204Z","iopub.status.idle":"2024-12-10T07:34:11.839427Z","shell.execute_reply.started":"2024-12-10T07:34:11.832146Z","shell.execute_reply":"2024-12-10T07:34:11.83848Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = data.drop('Premium Amount', axis=1)\ny = data['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:11.840681Z","iopub.execute_input":"2024-12-10T07:34:11.841132Z","iopub.status.idle":"2024-12-10T07:34:12.172767Z","shell.execute_reply.started":"2024-12-10T07:34:11.841061Z","shell.execute_reply":"2024-12-10T07:34:12.171598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X,y,test_size=0.2, random_state=42)\nX_train.shape, y_train.shape, X_val.shape, y_val.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:12.174287Z","iopub.execute_input":"2024-12-10T07:34:12.174768Z","iopub.status.idle":"2024-12-10T07:34:13.730975Z","shell.execute_reply.started":"2024-12-10T07:34:12.174708Z","shell.execute_reply":"2024-12-10T07:34:13.729672Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Categorical Handling","metadata":{}},{"cell_type":"code","source":"X_train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:13.732502Z","iopub.execute_input":"2024-12-10T07:34:13.732978Z","iopub.status.idle":"2024-12-10T07:34:13.761888Z","shell.execute_reply.started":"2024-12-10T07:34:13.732926Z","shell.execute_reply":"2024-12-10T07:34:13.760744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:13.76345Z","iopub.execute_input":"2024-12-10T07:34:13.763915Z","iopub.status.idle":"2024-12-10T07:34:13.773199Z","shell.execute_reply.started":"2024-12-10T07:34:13.763864Z","shell.execute_reply":"2024-12-10T07:34:13.771897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = X_train.select_dtypes('object').columns.tolist()\nnum_cols = X_train.select_dtypes('number').columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:13.775216Z","iopub.execute_input":"2024-12-10T07:34:13.775595Z","iopub.status.idle":"2024-12-10T07:34:14.439702Z","shell.execute_reply.started":"2024-12-10T07:34:13.775559Z","shell.execute_reply":"2024-12-10T07:34:14.438549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train[cat_cols] = X_train[cat_cols].fillna('nan').astype(str)\nX_val[cat_cols] = X_val[cat_cols].fillna('nan').astype(str)\nX_test=test.copy()\nX_test[cat_cols] = X_test[cat_cols].fillna('nan').astype(str)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:14.442814Z","iopub.execute_input":"2024-12-10T07:34:14.443222Z","iopub.status.idle":"2024-12-10T07:34:17.313998Z","shell.execute_reply.started":"2024-12-10T07:34:14.443185Z","shell.execute_reply":"2024-12-10T07:34:17.312993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.metrics import mean_squared_error,r2_score,mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:17.315564Z","iopub.execute_input":"2024-12-10T07:34:17.31602Z","iopub.status.idle":"2024-12-10T07:34:17.363422Z","shell.execute_reply.started":"2024-12-10T07:34:17.315971Z","shell.execute_reply":"2024-12-10T07:34:17.362456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cb = CatBoostRegressor(iterations=500,\n                      learning_rate=0.02,\n                      depth=8,\n                      cat_features=cat_cols,\n                      loss_function='RMSE',\n                       verbose=100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:17.364723Z","iopub.execute_input":"2024-12-10T07:34:17.36506Z","iopub.status.idle":"2024-12-10T07:34:17.370899Z","shell.execute_reply.started":"2024-12-10T07:34:17.365027Z","shell.execute_reply":"2024-12-10T07:34:17.369667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cb.fit(X_train, y_train, eval_set=(X_val, y_val))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:34:17.372458Z","iopub.execute_input":"2024-12-10T07:34:17.372915Z","iopub.status.idle":"2024-12-10T07:45:08.221205Z","shell.execute_reply.started":"2024-12-10T07:34:17.372865Z","shell.execute_reply":"2024-12-10T07:45:08.219077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = cb.predict(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:45:08.223556Z","iopub.execute_input":"2024-12-10T07:45:08.224247Z","iopub.status.idle":"2024-12-10T07:45:08.878811Z","shell.execute_reply.started":"2024-12-10T07:45:08.224191Z","shell.execute_reply":"2024-12-10T07:45:08.877731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"msle = mean_squared_log_error(y_val, val_preds)\nrmsle = np.sqrt(msle)\nprint(rmsle)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:45:08.880338Z","iopub.execute_input":"2024-12-10T07:45:08.880793Z","iopub.status.idle":"2024-12-10T07:45:08.904378Z","shell.execute_reply.started":"2024-12-10T07:45:08.880739Z","shell.execute_reply":"2024-12-10T07:45:08.903247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds = cb.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:45:08.90566Z","iopub.execute_input":"2024-12-10T07:45:08.905957Z","iopub.status.idle":"2024-12-10T07:45:10.554489Z","shell.execute_reply.started":"2024-12-10T07:45:08.905928Z","shell.execute_reply":"2024-12-10T07:45:10.553322Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output = pd.DataFrame({'id': test_id, 'Premium Amount':test_preds})\noutput.to_csv('submission.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:45:10.556168Z","iopub.execute_input":"2024-12-10T07:45:10.557483Z","iopub.status.idle":"2024-12-10T07:45:12.254706Z","shell.execute_reply.started":"2024-12-10T07:45:10.557432Z","shell.execute_reply":"2024-12-10T07:45:12.253509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:45:12.256274Z","iopub.execute_input":"2024-12-10T07:45:12.25675Z","iopub.status.idle":"2024-12-10T07:45:12.269575Z","shell.execute_reply.started":"2024-12-10T07:45:12.256701Z","shell.execute_reply":"2024-12-10T07:45:12.268585Z"}},"outputs":[],"execution_count":null}]}