{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:18.447211Z","iopub.execute_input":"2024-12-29T00:48:18.447811Z","iopub.status.idle":"2024-12-29T00:48:18.457097Z","shell.execute_reply.started":"2024-12-29T00:48:18.447703Z","shell.execute_reply":"2024-12-29T00:48:18.456127Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Import pacakges\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom matplotlib.gridspec import GridSpec\n\n# Reading the train dataframe in the variable df\ndf = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:04:36.19803Z","iopub.execute_input":"2025-01-05T15:04:36.198357Z","iopub.status.idle":"2025-01-05T15:04:49.579964Z","shell.execute_reply.started":"2025-01-05T15:04:36.19833Z","shell.execute_reply":"2025-01-05T15:04:49.578805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prep data for viz\ndef mpl_base_plot(figsize: tuple = (10,6) ,shape: tuple = (1,1), facecolor: str= '#fffbe8') -> tuple[plt.Figure, GridSpec]:\n    fig = plt.figure(figsize=figsize, facecolor='#fffbe8')\n    gs = fig.add_gridspec(*shape)\n    return fig, gs\n\ndef get_axes(fig: plt.Figure,gs: Gridspec, grid_row: int = 0, grid_col: int = 0) -> plt.Axes:\n    return fig.add_subplot(gs[grid_row, grid_col])\n\ndef disable_splines(ax: plt.Axes, splines =  [\"right\", \"top\"]):\n    for s in splines:\n        ax.spines[s].set_visible(False)\n    return ax\n\ndef add_text_label_bar(ax: plt.Axes, show_values_filter: float | None = None, horizontal: bool = True, text_displace: float = 0.05):\n    if horizontal:\n        for p in ax.patches:\n            value = f'{p.get_width()*100:.2f}%'\n            x = p.get_x() + p.get_width() + text_displace\n            y = p.get_y() + p.get_height() / 2 \n            if x < show_values_filter:\n                ax.text(x, y, value, ha='center', va='center', fontsize=6)\n    else:\n        for p in ax.patches:\n            value = f'{p.get_width()*100:.2f}%'\n            x = p.get_x() + p.get_width() / 2\n            y = p.get_y() + p.get_height() + text_displace\n            if x < show_values_filter:\n                ax.text(x, y, value, ha='center', va='center', fontsize=6)\n    return ax\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:42:33.72408Z","iopub.execute_input":"2025-01-05T15:42:33.724487Z","iopub.status.idle":"2025-01-05T15:42:33.729537Z","shell.execute_reply.started":"2025-01-05T15:42:33.724457Z","shell.execute_reply":"2025-01-05T15:42:33.72838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, gs = mpl_base_plot()\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-05T15:42:34.385111Z","iopub.execute_input":"2025-01-05T15:42:34.385504Z","iopub.status.idle":"2025-01-05T15:42:34.392092Z","shell.execute_reply.started":"2025-01-05T15:42:34.385469Z","shell.execute_reply":"2025-01-05T15:42:34.391167Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data exploration\n\n## Data types and structures\n","metadata":{}},{"cell_type":"code","source":"df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:28.156243Z","iopub.execute_input":"2024-12-29T00:48:28.156508Z","iopub.status.idle":"2024-12-29T00:48:28.178273Z","shell.execute_reply.started":"2024-12-29T00:48:28.156484Z","shell.execute_reply":"2024-12-29T00:48:28.177307Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"\"\"\nNumber of rows: {df.shape[0]}\nNumber of cols: {df.shape[1]}\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:28.1799Z","iopub.execute_input":"2024-12-29T00:48:28.180306Z","iopub.status.idle":"2024-12-29T00:48:28.196213Z","shell.execute_reply.started":"2024-12-29T00:48:28.180271Z","shell.execute_reply":"2024-12-29T00:48:28.194818Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Missing values","metadata":{}},{"cell_type":"code","source":"# Data\nisna_df = df.isna().sum().sort_values(ascending=True)/df.shape[0]\nisna_df = isna_df[isna_df != 0]\n\n# Figures\nfig = plt.figure(figsize=(10,6), facecolor='#fffbe8')\n\ngs = fig.add_gridspec(1, 1)\nax0: plt.Axes = fig.add_subplot(gs[0, 0])\nax0.set_facecolor(\"#fffbe8\")\nax0.grid(alpha=0.2)\nax0.barh(isna_df.index, [1 for _ in range(len(isna_df.index))], alpha=0.3, color='grey')\nhbars = ax0.barh(isna_df.index, isna_df.values, color='red',edgecolor='black', linewidth=0.5)\nfor s in [\"right\", \"top\"]:\n    ax0.spines[s].set_visible(False)\n\nfor p in ax0.patches:\n    value = f'{p.get_width()*100:.2f}%'\n    x = p.get_x() + p.get_width() + 0.05\n    y = p.get_y() + p.get_height() / 2 \n    if x < 1:\n        ax0.text(x, y, value, ha='center', va='center', fontsize=6) \n\nax0.text(0, 1 + 0.02, \"Percentage of NaN values (if exists)\", transform=ax0.transAxes, fontsize = 8, fontweight='bold')\nax0.tick_params(labelsize=6, width=0.5, length=1.5)\n# ax0.text(0, 1 + 0.04, \"Subtitulo\", transform=ax0.transAxes, fontsize = 8)\n\n# ax0.bar_label(hbars, fmt=lambda x: f'{x*10:.1f}%')\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:26:26.778619Z","iopub.execute_input":"2024-12-31T21:26:26.779049Z","iopub.status.idle":"2024-12-31T21:26:27.67356Z","shell.execute_reply.started":"2024-12-31T21:26:26.779011Z","shell.execute_reply":"2024-12-31T21:26:27.672288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"isna_df = df.isna().sum().sort_values(ascending=True)/df.shape[0]\nisna_df = isna_df[isna_df != 0]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_isna_columns(df: pd.DataFrame) -> pd.Series:\n    \"\"\"\n    Returns a series where the values are the percentage of each column of the dataframe that has NaN values (only returns the columns that does\n    contain a NaN value)\n    \"\"\"\n    isna_df = df.isna().sum().sort_values(ascending=True)/df.shape[0]\n    return isna_df[isna_df != 0]\n    ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Columns\ncat_cols = ['Gender','Marital Status','Number of Dependents','Education Level','Occupation','Location','Policy Type', 'Previous Claims','Customer Feedback', 'Smoking Status', 'Exercise Frequency','Property Type']\nnum_cols = ['Age','Annual Income','Health Score','Vehicle Age', 'Credit Score', 'Insurance Duration']\ndate_cols = ['Policy Start Date']\nidx_cols = ['id']\ntarget_cols = ['Premium Amount']\n\ndf['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\ndf['Policy Date'] = df['Policy Start Date'].dt.strftime('%Y%m')\ndf['Policy Date Year'] = df['Policy Start Date'].dt.strftime('%Y')\n\ndate_cols += ['Policy Date','Policy Date Year']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:29.738682Z","iopub.execute_input":"2024-12-29T00:48:29.739019Z","iopub.status.idle":"2024-12-29T00:48:41.093727Z","shell.execute_reply.started":"2024-12-29T00:48:29.738993Z","shell.execute_reply":"2024-12-29T00:48:41.092847Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Categorical features","metadata":{}},{"cell_type":"code","source":"# Figures\nfig = plt.figure(figsize=(15,8))\ngs = fig.add_gridspec(3, 5)\n\nfor i in range(12):\n    cat_df = df[cat_cols[i]].value_counts(dropna=False)\n\n    # Plot\n    ax = fig.add_subplot(gs[i // 5, i % 5])\n    ax.set_title(cat_cols[i])\n    ax.pie(cat_df.values, labels=cat_df.index, autopct='%1.1f%%',frame=False)\n\nfig.tight_layout()\nfig.show()","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2024-12-29T00:48:41.094668Z","iopub.execute_input":"2024-12-29T00:48:41.094974Z","iopub.status.idle":"2024-12-29T00:48:42.957402Z","shell.execute_reply.started":"2024-12-29T00:48:41.094948Z","shell.execute_reply":"2024-12-29T00:48:42.956216Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Numerical features\n\n","metadata":{}},{"cell_type":"code","source":"# Distribution\n\nfig = plt.figure(figsize=(5,10))\ngs = fig.add_gridspec(6, 1)\n\nfor i in range(6):\n    ax = fig.add_subplot(gs[i,0])\n    ax.hist(df[num_cols[i]])\n    ax.set_title(num_cols[i])\nfig.tight_layout()\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:42.960556Z","iopub.execute_input":"2024-12-29T00:48:42.960923Z","iopub.status.idle":"2024-12-29T00:48:44.350663Z","shell.execute_reply.started":"2024-12-29T00:48:42.960892Z","shell.execute_reply":"2024-12-29T00:48:44.349572Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Target distribution\n\nfig = plt.figure(figsize = (6,4))\ngs = fig.add_gridspec(1,1)\n\nax = fig.add_subplot(gs[0, 0])\nax.hist(df[target_cols], bins=100)\nax.set_title(\"Premium Amount distribution\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:44.352414Z","iopub.execute_input":"2024-12-29T00:48:44.352811Z","iopub.status.idle":"2024-12-29T00:48:44.747351Z","shell.execute_reply.started":"2024-12-29T00:48:44.352763Z","shell.execute_reply":"2024-12-29T00:48:44.74638Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Target by date","metadata":{}},{"cell_type":"code","source":"_temp = df[date_cols + target_cols].groupby([\"Policy Date\"]).agg({'Premium Amount':'mean'}).sort_index()\nx1 = _temp.index\ny1 = _temp['Premium Amount']\n\n_temp2 = df[date_cols + target_cols].groupby([\"Policy Date Year\"]).agg({'Premium Amount':'mean'}).sort_index()\nx2 = _temp2.index\ny2 = _temp2['Premium Amount']\n\nfig = plt.figure(figsize = (10, 8))\ngs = fig.add_gridspec(2,1)\n# First plot (By month)\n\nax = fig.add_subplot(gs[0, 0])\nax.plot(x1, y1)\nax.tick_params(axis='x', labelrotation=90, labelsize=8)\nax.grid(alpha=0.2)\nax.set_title(\"Premium amount avarage (By month)\")\n\n# Second plot (By Year)\nax2 = fig.add_subplot(gs[1, 0])\nax2.plot(x2, y2)\nax2.tick_params(axis='x', labelrotation=90, labelsize=8)\nax2.grid(alpha=0.2)\nax2.set_title(\"Premium amount avarage (By year)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:44.748317Z","iopub.execute_input":"2024-12-29T00:48:44.748579Z","iopub.status.idle":"2024-12-29T00:48:45.988782Z","shell.execute_reply.started":"2024-12-29T00:48:44.748556Z","shell.execute_reply":"2024-12-29T00:48:45.987486Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[date_cols + target_cols].groupby([\"Policy Date Year\"]).agg({'Premium Amount':'count'}).sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:45.98987Z","iopub.execute_input":"2024-12-29T00:48:45.990136Z","iopub.status.idle":"2024-12-29T00:48:46.121373Z","shell.execute_reply.started":"2024-12-29T00:48:45.990113Z","shell.execute_reply":"2024-12-29T00:48:46.12018Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_columns = num_cols + target_cols\ncorr_matrix = df[corr_columns].corr()\n\nfig, (ax1, ax2) = plt.subplots(nrows=2, ncols=1, figsize=(8, 10))\nax1.set_title(\"Correlation between columns (Pearson)\")\nsns.heatmap(corr_matrix, annot=True, linewidths=.5, ax=ax1)\n\nax2.set_title(\"Correlation (Spearman)\")\nsns.heatmap(df[corr_columns].corr(method='spearman'), annot=True, linewidths=.5, ax=ax2)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:46.122465Z","iopub.execute_input":"2024-12-29T00:48:46.122856Z","iopub.status.idle":"2024-12-29T00:48:57.394611Z","shell.execute_reply.started":"2024-12-29T00:48:46.122814Z","shell.execute_reply":"2024-12-29T00:48:57.393057Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Dealing with missing values","metadata":{}},{"cell_type":"code","source":"missing_val_cols = [\"Age\",\"Annual Income\",\"Marital Status\",\"Number of Dependents\",\"Occupation\",\"Health Score\", \"Previous Claims\",\"Vehicle Age\",\"Insurance Duration\", \"Credit Score\",\"Customer Feedback\",\"Premium Amount\"]\n\nmissing_df = df.loc[:, missing_val_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:57.39659Z","iopub.execute_input":"2024-12-29T00:48:57.397141Z","iopub.status.idle":"2024-12-29T00:48:57.486046Z","shell.execute_reply.started":"2024-12-29T00:48:57.397094Z","shell.execute_reply":"2024-12-29T00:48:57.484683Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_temp = missing_df[[\"Occupation\",\"Premium Amount\"]].fillna(\"NaN Values\").groupby(\"Occupation\").agg({'Premium Amount': ['mean','std','sum','count']})['Premium Amount']\n\nfig = plt.figure(figsize=(8,6))\ngs = fig.add_gridspec(2,1)\n\nax1 = fig.add_subplot(gs[0,0])\nax1.barh(_temp['mean'].index, _temp['mean'].values)\nax1.set_title(\"Mean of Premium Amount per occupation\")\n\nax2 = fig.add_subplot(gs[1,0])\nax2.barh(_temp['count'].index, _temp['count'].values)\nax2.set_title(\"Count fo Premium Amount per occupation\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:57.487276Z","iopub.execute_input":"2024-12-29T00:48:57.487689Z","iopub.status.idle":"2024-12-29T00:48:58.134822Z","shell.execute_reply.started":"2024-12-29T00:48:57.487658Z","shell.execute_reply":"2024-12-29T00:48:58.133463Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_temp = missing_df[[\"Previous Claims\",\"Premium Amount\"]].fillna(\"IsNa\").groupby(\"Previous Claims\").agg({'Premium Amount': ['mean','std','sum','count']})['Premium Amount']\n\nfig = plt.figure(figsize = (6, 10))\nfig.add_gridspec(2,1)\n\nax = fig.add_subplot(gs[0,0])\nax.grid(alpha=0.2)\nax.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['mean'])\nax.set_title(\"Previous Claims Mean Value\")\n\nax2 = fig.add_subplot(gs[1,0])\nax2.grid(alpha=0.2)\nax2.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['count'])\nax2.set_title(\"Previous Claims Counts\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:58.136005Z","iopub.execute_input":"2024-12-29T00:48:58.136278Z","iopub.status.idle":"2024-12-29T00:48:58.868181Z","shell.execute_reply.started":"2024-12-29T00:48:58.136254Z","shell.execute_reply":"2024-12-29T00:48:58.867213Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_temp = missing_df[[\"Number of Dependents\",\"Premium Amount\"]].fillna(\"IsNa\").groupby(\"Number of Dependents\").agg({'Premium Amount': ['mean','std','sum','count']})['Premium Amount']\n\nfig = plt.figure(figsize = (6, 10))\nfig.add_gridspec(2,1)\n\nax = fig.add_subplot(gs[0,0])\nax.grid(alpha=0.2)\nax.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['mean'])\nax.set_title(\"Number of Dependents Mean Value\")\n\nax2 = fig.add_subplot(gs[1,0])\nax2.grid(alpha=0.2)\nax2.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['count'])\nax2.set_title(\"Number of Dependents Counts\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:58.869316Z","iopub.execute_input":"2024-12-29T00:48:58.869606Z","iopub.status.idle":"2024-12-29T00:48:59.413465Z","shell.execute_reply.started":"2024-12-29T00:48:58.86957Z","shell.execute_reply":"2024-12-29T00:48:59.412283Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_temp = missing_df[[\"Customer Feedback\",\"Premium Amount\"]].fillna(\"IsNa\").groupby(\"Customer Feedback\").agg({'Premium Amount': ['mean','std','sum','count']})['Premium Amount']\n\nfig = plt.figure(figsize = (6, 10))\nfig.add_gridspec(2,1)\n\nax = fig.add_subplot(gs[0,0])\nax.grid(alpha=0.2)\nax.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['mean'])\nax.set_title(\"Customer Feedback Mean Value\")\n\nax2 = fig.add_subplot(gs[1,0])\nax2.grid(alpha=0.2)\nax2.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['count'])\nax2.set_title(\"Customer Feedback Counts\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:59.414567Z","iopub.execute_input":"2024-12-29T00:48:59.414966Z","iopub.status.idle":"2024-12-29T00:48:59.88038Z","shell.execute_reply.started":"2024-12-29T00:48:59.414938Z","shell.execute_reply":"2024-12-29T00:48:59.87922Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Credit Score\")\npd.DataFrame([{\n    \"Column\": \"NaN Values\",\n    \"Count\": missing_df[\"Credit Score\"].isna().sum(),\n    \"Mean\":missing_df[missing_df[\"Credit Score\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[missing_df[\"Credit Score\"].isna()][\"Premium Amount\"].std()\n    \n},\n{\n    \"Column\": \"Not NaN Values\",\n    \"Count\":(~missing_df[\"Credit Score\"].isna()).sum(),\n    \"Mean\":missing_df[~missing_df[\"Credit Score\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[~missing_df[\"Credit Score\"].isna()][\"Premium Amount\"].std()\n}])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:48:59.881442Z","iopub.execute_input":"2024-12-29T00:48:59.881722Z","iopub.status.idle":"2024-12-29T00:49:00.063873Z","shell.execute_reply.started":"2024-12-29T00:48:59.88169Z","shell.execute_reply":"2024-12-29T00:49:00.062764Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Health Score\")\npd.DataFrame([{\n    \"Column\": \"NaN Values\",\n    \"Count\": missing_df[\"Health Score\"].isna().sum(),\n    \"Mean\":missing_df[missing_df[\"Health Score\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[missing_df[\"Health Score\"].isna()][\"Premium Amount\"].std()\n    \n},\n{\n    \"Column\": \"Not NaN Values\",\n    \"Count\":(~missing_df[\"Health Score\"].isna()).sum(),\n    \"Mean\":missing_df[~missing_df[\"Health Score\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[~missing_df[\"Health Score\"].isna()][\"Premium Amount\"].std()\n}])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:00.06492Z","iopub.execute_input":"2024-12-29T00:49:00.065181Z","iopub.status.idle":"2024-12-29T00:49:00.243682Z","shell.execute_reply.started":"2024-12-29T00:49:00.065159Z","shell.execute_reply":"2024-12-29T00:49:00.242807Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Anual Income\")\npd.DataFrame([{\n    \"Column\": \"NaN Values\",\n    \"Count\": missing_df[\"Annual Income\"].isna().sum(),\n    \"Mean\":missing_df[missing_df[\"Annual Income\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[missing_df[\"Annual Income\"].isna()][\"Premium Amount\"].std()\n    \n},\n{\n    \"Column\": \"Not NaN Values\",\n    \"Count\":(~missing_df[\"Annual Income\"].isna()).sum(),\n    \"Mean\":missing_df[~missing_df[\"Annual Income\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[~missing_df[\"Annual Income\"].isna()][\"Premium Amount\"].std()\n}])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:00.244703Z","iopub.execute_input":"2024-12-29T00:49:00.245003Z","iopub.status.idle":"2024-12-29T00:49:00.414625Z","shell.execute_reply.started":"2024-12-29T00:49:00.24498Z","shell.execute_reply":"2024-12-29T00:49:00.41365Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Age\")\npd.DataFrame([{\n    \"Column\": \"NaN Values\",\n    \"Count\": missing_df[\"Age\"].isna().sum(),\n    \"Mean\":missing_df[missing_df[\"Age\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[missing_df[\"Age\"].isna()][\"Premium Amount\"].std()\n    \n},\n{\n    \"Column\": \"Not NaN Values\",\n    \"Count\":(~missing_df[\"Age\"].isna()).sum(),\n    \"Mean\":missing_df[~missing_df[\"Age\"].isna()][\"Premium Amount\"].mean(),\n    \"Std\":missing_df[~missing_df[\"Age\"].isna()][\"Premium Amount\"].std()\n}])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:00.415753Z","iopub.execute_input":"2024-12-29T00:49:00.416041Z","iopub.status.idle":"2024-12-29T00:49:00.581761Z","shell.execute_reply.started":"2024-12-29T00:49:00.416018Z","shell.execute_reply":"2024-12-29T00:49:00.580656Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Features without null values","metadata":{}},{"cell_type":"code","source":"non_missing_val_cols = [\"Gender\",\"Education Level\",\"Location\",\"Policy Type\",\"Policy Start Date\",\"Smoking Status\",\"Exercise Frequency\",\"Property Type\",\"Policy Date\",\"Policy Date Year\",\"Premium Amount\"]\ndf_without_null_vals = df[non_missing_val_cols].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:00.582845Z","iopub.execute_input":"2024-12-29T00:49:00.583217Z","iopub.status.idle":"2024-12-29T00:49:01.472303Z","shell.execute_reply.started":"2024-12-29T00:49:00.583181Z","shell.execute_reply":"2024-12-29T00:49:01.470826Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_without_null_vals.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:01.477867Z","iopub.execute_input":"2024-12-29T00:49:01.478273Z","iopub.status.idle":"2024-12-29T00:49:01.494446Z","shell.execute_reply.started":"2024-12-29T00:49:01.478242Z","shell.execute_reply":"2024-12-29T00:49:01.493219Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def categories_plot(dataframe: pd.DataFrame, select_col: str, target_col: str = \"Premium Amount\"):\n    _temp = dataframe[[select_col,target_col]].fillna(\"IsNa\").groupby(select_col).agg({target_col: ['mean','count']})[target_col]\n    \n    fig = plt.figure(figsize = (8, 3))\n    gs = fig.add_gridspec(1,2)\n    \n    ax = fig.add_subplot(gs[0,0])\n    ax.grid(alpha=0.2)\n    ax.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['mean'])\n    ax.set_title(f\"{select_col} Mean Value\")\n    \n    ax2 = fig.add_subplot(gs[0,1])\n    ax2.grid(alpha=0.2)\n    ax2.barh(sorted([str(i) for i in _temp.index.tolist()]), _temp['count'])\n    ax2.set_title(f\"{select_col} Counts\")\n\n    plt.tight_layout()\n    plt.show()\n    return\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:01.497161Z","iopub.execute_input":"2024-12-29T00:49:01.497471Z","iopub.status.idle":"2024-12-29T00:49:01.517232Z","shell.execute_reply.started":"2024-12-29T00:49:01.497443Z","shell.execute_reply":"2024-12-29T00:49:01.515908Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categories_plot(df_without_null_vals, \"Policy Type\")\ncategories_plot(df_without_null_vals, \"Gender\")\ncategories_plot(df_without_null_vals, \"Location\")\ncategories_plot(df_without_null_vals, \"Policy Type\")\ncategories_plot(df_without_null_vals, \"Education Level\")\ncategories_plot(df_without_null_vals, \"Smoking Status\")\ncategories_plot(df_without_null_vals, \"Exercise Frequency\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:01.518374Z","iopub.execute_input":"2024-12-29T00:49:01.519067Z","iopub.status.idle":"2024-12-29T00:49:05.105834Z","shell.execute_reply.started":"2024-12-29T00:49:01.519031Z","shell.execute_reply":"2024-12-29T00:49:05.104711Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Matching missing val with non missing val","metadata":{}},{"cell_type":"code","source":"def get_stats(dataframe: pd.DataFrame, col: str, target_col: str = 'Premium Amount') -> pd.DataFrame:\n    _temp = dataframe.groupby(col).agg({target_col:['mean','count']})[target_col]\n\n    _temp_mean = _temp['mean'].mean()\n    _temp_count_total = _temp['count'].sum()\n    \n    _temp['count_ratio'] = _temp['count']/_temp_count_total\n    _temp['mean_abs_diff'] = np.abs(_temp['mean'] - _temp_mean)\n    return _temp\n    \ndef compare_dataframes(df1: pd.DataFrame, df2: pd.DataFrame):\n    _temp = pd.DataFrame()\n    _temp['count_ratio_diff'] = np.abs(df1['count_ratio'] - df2['count_ratio'])\n    _temp['mean_diff'] = df1['mean'] - df2['mean']\n    return _temp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:05.106756Z","iopub.execute_input":"2024-12-29T00:49:05.107138Z","iopub.status.idle":"2024-12-29T00:49:05.11427Z","shell.execute_reply.started":"2024-12-29T00:49:05.107112Z","shell.execute_reply":"2024-12-29T00:49:05.112742Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's start trying occupation column\n\ntracked_col = 'Occupation'\n\n_is_na_df = df[[tracked_col] + non_missing_val_cols]\nonly_one_col_nan = _is_na_df[_is_na_df[tracked_col].isna()]\n\n# Overall\nmain_stats = {}\nfor col in ['Gender', 'Education Level', 'Location', 'Policy Type', 'Smoking Status', 'Exercise Frequency','Property Type','Policy Date Year']:\n    _temp = get_stats(df, col)\n    _temp2 = get_stats(only_one_col_nan, col)\n    tmp = compare_dataframes(_temp, _temp2)\n    print(col)\n    print(tmp)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:05.115327Z","iopub.execute_input":"2024-12-29T00:49:05.115615Z","iopub.status.idle":"2024-12-29T00:49:06.623099Z","shell.execute_reply.started":"2024-12-29T00:49:05.115591Z","shell.execute_reply":"2024-12-29T00:49:06.62193Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Multiple groups combination with nan\n","metadata":{}},{"cell_type":"code","source":"# We can create a column with this category\ndf.groupby(['Policy Date Year','Occupation'], dropna=False).agg({'Premium Amount':['mean','count']})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:06.624265Z","iopub.execute_input":"2024-12-29T00:49:06.624661Z","iopub.status.idle":"2024-12-29T00:49:06.90812Z","shell.execute_reply.started":"2024-12-29T00:49:06.624608Z","shell.execute_reply":"2024-12-29T00:49:06.906947Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.groupby(['Gender','Previous Claims'], dropna=False).agg({'Premium Amount':['mean','count']})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:06.909348Z","iopub.execute_input":"2024-12-29T00:49:06.909743Z","iopub.status.idle":"2024-12-29T00:49:07.101087Z","shell.execute_reply.started":"2024-12-29T00:49:06.909704Z","shell.execute_reply":"2024-12-29T00:49:07.100131Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Start some feature engineering","metadata":{}},{"cell_type":"code","source":"df[cat_cols] = df[cat_cols].fillna(\"Not declared\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:07.102066Z","iopub.execute_input":"2024-12-29T00:49:07.102349Z","iopub.status.idle":"2024-12-29T00:49:08.445902Z","shell.execute_reply.started":"2024-12-29T00:49:07.102323Z","shell.execute_reply":"2024-12-29T00:49:08.444702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"Declared Annual Income\"] = df['Annual Income'].isna().astype(\"category\")\ndf[\"Declared Health Score\"] = df[\"Health Score\"].isna().astype(\"category\")\ndf[\"Declared Credit Score\"] = df[\"Credit Score\"].isna().astype(\"category\")\ndf['Annual Income Greater than mean'] = df['Annual Income'] > df['Annual Income'].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:08.446914Z","iopub.execute_input":"2024-12-29T00:49:08.4472Z","iopub.status.idle":"2024-12-29T00:49:08.505523Z","shell.execute_reply.started":"2024-12-29T00:49:08.447175Z","shell.execute_reply":"2024-12-29T00:49:08.504222Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# From the isna columns, we can try not using the age column and the credit score we can fill using the mean\ndf.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:08.506935Z","iopub.execute_input":"2024-12-29T00:49:08.507343Z","iopub.status.idle":"2024-12-29T00:49:09.292266Z","shell.execute_reply.started":"2024-12-29T00:49:08.507305Z","shell.execute_reply":"2024-12-29T00:49:09.290982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Categorical without missing values","metadata":{}},{"cell_type":"markdown","source":"## Selecting our initial columns to test","metadata":{}},{"cell_type":"code","source":"model_columns = ['Age','Gender','Declared Annual Income','Annual Income Greater than mean',\n                 'Marital Status','Number of Dependents','Occupation','Education Level',\n                 'Declared Health Score','Location','Policy Type','Previous Claims','Vehicle Age','Declared Credit Score',\n                 'Insurance Duration','Policy Date Year','Customer Feedback','Smoking Status',\n                 'Exercise Frequency','Property Type','Premium Amount']\n\ndf = df[model_columns].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:09.293464Z","iopub.execute_input":"2024-12-29T00:49:09.2939Z","iopub.status.idle":"2024-12-29T00:49:10.555137Z","shell.execute_reply.started":"2024-12-29T00:49:09.293861Z","shell.execute_reply":"2024-12-29T00:49:10.553934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.metrics import mean_absolute_error as mae\nfrom sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor\nfrom sklearn.metrics import mean_squared_log_error\nimport numpy as np","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:10.556304Z","iopub.execute_input":"2024-12-29T00:49:10.556586Z","iopub.status.idle":"2024-12-29T00:49:10.562393Z","shell.execute_reply.started":"2024-12-29T00:49:10.55656Z","shell.execute_reply":"2024-12-29T00:49:10.561082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.pipeline import Pipeline\nfrom xgboost import XGBRegressor\nfrom sklearn.compose import ColumnTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:10.56332Z","iopub.execute_input":"2024-12-29T00:49:10.563693Z","iopub.status.idle":"2024-12-29T00:49:10.588348Z","shell.execute_reply.started":"2024-12-29T00:49:10.563664Z","shell.execute_reply":"2024-12-29T00:49:10.587141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:10.589673Z","iopub.execute_input":"2024-12-29T00:49:10.590103Z","iopub.status.idle":"2024-12-29T00:49:11.322859Z","shell.execute_reply.started":"2024-12-29T00:49:10.590074Z","shell.execute_reply":"2024-12-29T00:49:11.321701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"categorical_cols = ['Gender','Declared Annual Income','Previous Claims','Annual Income Greater than mean','Number of Dependents','Occupation','Education Level','Declared Health Score','Location','Policy Type','Declared Credit Score','Insurance Duration','Policy Date Year','Customer Feedback','Smoking Status',\n                 'Exercise Frequency','Property Type']\ndf[categorical_cols].isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:11.323924Z","iopub.execute_input":"2024-12-29T00:49:11.324235Z","iopub.status.idle":"2024-12-29T00:49:12.210317Z","shell.execute_reply.started":"2024-12-29T00:49:11.324211Z","shell.execute_reply":"2024-12-29T00:49:12.208907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# numerical_cols = ['Vehicle Age']\n\ncat_cols1 = ['Gender','Declared Annual Income','Previous Claims','Annual Income Greater than mean','Number of Dependents','Occupation','Education Level','Declared Health Score','Location','Policy Type','Declared Credit Score','Insurance Duration','Policy Date Year','Customer Feedback','Smoking Status','Exercise Frequency','Property Type']\n\ndf_tmp = df[cat_cols1 + ['Premium Amount']].copy()\n\ndf_tmp[cat_cols1] = df_tmp[cat_cols1].astype('str')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:12.211814Z","iopub.execute_input":"2024-12-29T00:49:12.212197Z","iopub.status.idle":"2024-12-29T00:49:16.563132Z","shell.execute_reply.started":"2024-12-29T00:49:12.212167Z","shell.execute_reply":"2024-12-29T00:49:16.562015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_tmp['Previous Claims'].dtypes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:16.564227Z","iopub.execute_input":"2024-12-29T00:49:16.564521Z","iopub.status.idle":"2024-12-29T00:49:16.570614Z","shell.execute_reply.started":"2024-12-29T00:49:16.564495Z","shell.execute_reply":"2024-12-29T00:49:16.569737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_pipeline = Pipeline([\n    (\"encoding\",OneHotEncoder(handle_unknown='ignore')),\n    (\"xgb_model\",XGBRegressor())\n])\n\n\nX, y = df_tmp.iloc[:,:-1], df_tmp.iloc[:,-1]\nscores = cross_val_score(xgb_pipeline, X, y,scoring=\"neg_mean_squared_log_error\",cv=5)\n\nfinal_avg_rmse = np.mean(np.sqrt(np.abs(scores)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:49:16.571573Z","iopub.execute_input":"2024-12-29T00:49:16.5719Z","iopub.status.idle":"2024-12-29T00:50:26.935057Z","shell.execute_reply.started":"2024-12-29T00:49:16.57186Z","shell.execute_reply":"2024-12-29T00:50:26.933773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\nxgb_pipeline.fit(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:26.936307Z","iopub.execute_input":"2024-12-29T00:50:26.936709Z","iopub.status.idle":"2024-12-29T00:50:43.627415Z","shell.execute_reply.started":"2024-12-29T00:50:26.93666Z","shell.execute_reply":"2024-12-29T00:50:43.626223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_avg_rmse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:43.628636Z","iopub.execute_input":"2024-12-29T00:50:43.629035Z","iopub.status.idle":"2024-12-29T00:50:43.635242Z","shell.execute_reply.started":"2024-12-29T00:50:43.628992Z","shell.execute_reply":"2024-12-29T00:50:43.634169Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prep for the test","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:43.636145Z","iopub.execute_input":"2024-12-29T00:50:43.636455Z","iopub.status.idle":"2024-12-29T00:50:43.669835Z","shell.execute_reply.started":"2024-12-29T00:50:43.636428Z","shell.execute_reply":"2024-12-29T00:50:43.668415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Columns\ncat_cols = ['Gender','Marital Status','Number of Dependents','Education Level','Occupation','Location','Policy Type', 'Previous Claims','Customer Feedback', 'Smoking Status', 'Exercise Frequency','Property Type']\nnum_cols = ['Age','Annual Income','Health Score','Vehicle Age', 'Credit Score', 'Insurance Duration']\ndate_cols = ['Policy Start Date']\nidx_cols = ['id']\ntarget_cols = ['Premium Amount']\n\ntest_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'])\ntest_df['Policy Date'] = test_df['Policy Start Date'].dt.strftime('%Y%m')\ntest_df['Policy Date Year'] = test_df['Policy Start Date'].dt.strftime('%Y')\n\ndate_cols += ['Policy Date','Policy Date Year']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:43.670828Z","iopub.execute_input":"2024-12-29T00:50:43.67113Z","iopub.status.idle":"2024-12-29T00:50:51.041975Z","shell.execute_reply.started":"2024-12-29T00:50:43.6711Z","shell.execute_reply":"2024-12-29T00:50:51.04056Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[cat_cols] = df[cat_cols].fillna(\"Not declared\")\n\ntest_df[\"Declared Annual Income\"] = test_df['Annual Income'].isna().astype(\"category\")\ntest_df[\"Declared Health Score\"] = test_df[\"Health Score\"].isna().astype(\"category\")\ntest_df[\"Declared Credit Score\"] = test_df[\"Credit Score\"].isna().astype(\"category\")\ntest_df['Annual Income Greater than mean'] = test_df['Annual Income'] > test_df['Annual Income'].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:51.043232Z","iopub.execute_input":"2024-12-29T00:50:51.043591Z","iopub.status.idle":"2024-12-29T00:50:52.731434Z","shell.execute_reply.started":"2024-12-29T00:50:51.043559Z","shell.execute_reply":"2024-12-29T00:50:52.730296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_columns = ['Gender','Declared Annual Income','Annual Income Greater than mean',\n                 'Number of Dependents','Occupation','Education Level',\n                 'Declared Health Score','Location','Policy Type','Previous Claims','Declared Credit Score',\n                 'Insurance Duration','Policy Date Year','Customer Feedback','Smoking Status',\n                 'Exercise Frequency','Property Type']\n\ntest_df = test_df[X.columns].copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:52.732379Z","iopub.execute_input":"2024-12-29T00:50:52.732666Z","iopub.status.idle":"2024-12-29T00:50:53.326783Z","shell.execute_reply.started":"2024-12-29T00:50:52.732641Z","shell.execute_reply":"2024-12-29T00:50:53.325611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_mean = test_df['Insurance Duration'].mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:53.327928Z","iopub.execute_input":"2024-12-29T00:50:53.328214Z","iopub.status.idle":"2024-12-29T00:50:53.336937Z","shell.execute_reply.started":"2024-12-29T00:50:53.328189Z","shell.execute_reply":"2024-12-29T00:50:53.335702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_mean","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:53.337749Z","iopub.execute_input":"2024-12-29T00:50:53.338101Z","iopub.status.idle":"2024-12-29T00:50:53.347263Z","shell.execute_reply.started":"2024-12-29T00:50:53.338074Z","shell.execute_reply":"2024-12-29T00:50:53.346257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['Insurance Duration'] = test_df['Insurance Duration'].fillna(_mean)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:53.348369Z","iopub.execute_input":"2024-12-29T00:50:53.348745Z","iopub.status.idle":"2024-12-29T00:50:53.372024Z","shell.execute_reply.started":"2024-12-29T00:50:53.348707Z","shell.execute_reply":"2024-12-29T00:50:53.370854Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.isna().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:53.373233Z","iopub.execute_input":"2024-12-29T00:50:53.373641Z","iopub.status.idle":"2024-12-29T00:50:53.823307Z","shell.execute_reply.started":"2024-12-29T00:50:53.373602Z","shell.execute_reply":"2024-12-29T00:50:53.822288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = xgb_pipeline.predict(test_df.values)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:50:53.824163Z","iopub.execute_input":"2024-12-29T00:50:53.824526Z","iopub.status.idle":"2024-12-29T00:51:00.381187Z","shell.execute_reply.started":"2024-12-29T00:50:53.824498Z","shell.execute_reply":"2024-12-29T00:51:00.380092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T00:51:00.382181Z","iopub.execute_input":"2024-12-29T00:51:00.382464Z","iopub.status.idle":"2024-12-29T00:51:03.746918Z","shell.execute_reply.started":"2024-12-29T00:51:00.382439Z","shell.execute_reply":"2024-12-29T00:51:03.745721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test_df['Premium Amount'] = y_pred","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T01:23:45.534365Z","iopub.execute_input":"2024-12-29T01:23:45.535739Z","iopub.status.idle":"2024-12-29T01:23:45.550001Z","shell.execute_reply.started":"2024-12-29T01:23:45.535697Z","shell.execute_reply":"2024-12-29T01:23:45.548264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test_df[['id','Premium Amount']].to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T01:23:47.349788Z","iopub.execute_input":"2024-12-29T01:23:47.350218Z","iopub.status.idle":"2024-12-29T01:23:48.610208Z","shell.execute_reply.started":"2024-12-29T01:23:47.35019Z","shell.execute_reply":"2024-12-29T01:23:48.609137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_test_df[['id','Premium Amount']]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-29T01:23:49.352929Z","iopub.execute_input":"2024-12-29T01:23:49.35327Z","iopub.status.idle":"2024-12-29T01:23:49.375926Z","shell.execute_reply.started":"2024-12-29T01:23:49.353243Z","shell.execute_reply":"2024-12-29T01:23:49.37476Z"}},"outputs":[],"execution_count":null}]}