{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### The obective of this notebook is to plot the missing values for each column and find their relationship with the target variable.\n\n### Feel free to use the function - \"plot_average_premium_by_bins\"  and modify it according to your requirement for plotting missing value trends.\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:22:59.025548Z","iopub.execute_input":"2024-12-25T07:22:59.025926Z","iopub.status.idle":"2024-12-25T07:22:59.413116Z","shell.execute_reply.started":"2024-12-25T07:22:59.025892Z","shell.execute_reply":"2024-12-25T07:22:59.412181Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1 Import Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:22:59.484655Z","iopub.execute_input":"2024-12-25T07:22:59.485184Z","iopub.status.idle":"2024-12-25T07:23:00.360838Z","shell.execute_reply.started":"2024-12-25T07:22:59.485153Z","shell.execute_reply":"2024-12-25T07:23:00.359768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_data = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:57:46.965147Z","iopub.execute_input":"2024-12-25T07:57:46.965479Z","iopub.status.idle":"2024-12-25T07:57:55.287027Z","shell.execute_reply.started":"2024-12-25T07:57:46.965454Z","shell.execute_reply":"2024-12-25T07:57:55.28603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:23:11.630742Z","iopub.execute_input":"2024-12-25T07:23:11.631224Z","iopub.status.idle":"2024-12-25T07:23:11.636064Z","shell.execute_reply.started":"2024-12-25T07:23:11.631176Z","shell.execute_reply":"2024-12-25T07:23:11.635042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:23:11.637644Z","iopub.execute_input":"2024-12-25T07:23:11.63807Z","iopub.status.idle":"2024-12-25T07:23:11.69499Z","shell.execute_reply.started":"2024-12-25T07:23:11.638031Z","shell.execute_reply":"2024-12-25T07:23:11.69396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.drop(columns = ['id'],inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:58:00.963135Z","iopub.execute_input":"2024-12-25T07:58:00.963468Z","iopub.status.idle":"2024-12-25T07:58:01.164552Z","shell.execute_reply.started":"2024-12-25T07:58:00.963445Z","shell.execute_reply":"2024-12-25T07:58:01.163336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:23:11.868117Z","iopub.execute_input":"2024-12-25T07:23:11.868481Z","iopub.status.idle":"2024-12-25T07:23:11.875461Z","shell.execute_reply.started":"2024-12-25T07:23:11.868442Z","shell.execute_reply":"2024-12-25T07:23:11.874414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Basic EDA","metadata":{}},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:23:11.876656Z","iopub.execute_input":"2024-12-25T07:23:11.876962Z","iopub.status.idle":"2024-12-25T07:23:12.540475Z","shell.execute_reply.started":"2024-12-25T07:23:11.876925Z","shell.execute_reply":"2024-12-25T07:23:12.539438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Finding out which columns have missing values \n\nmissing_values = train_data.isnull().sum()\nfill_rate = (1 - missing_values / len(train_data)) * 100\n\nmissing_data_info = pd.DataFrame({\n    'Column Name': train_data.columns,\n    'Missing Values': missing_values,\n    'Fill Rate (%)': fill_rate\n}).reset_index(drop=True)\n\nmissing_data_info = missing_data_info[missing_data_info['Missing Values'] > 0]\nprint(missing_data_info)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:30:20.028635Z","iopub.execute_input":"2024-12-25T07:30:20.029065Z","iopub.status.idle":"2024-12-25T07:30:20.662998Z","shell.execute_reply.started":"2024-12-25T07:30:20.029035Z","shell.execute_reply":"2024-12-25T07:30:20.661935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dfd = train_data.describe()\ndfd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:58:04.765048Z","iopub.execute_input":"2024-12-25T07:58:04.765385Z","iopub.status.idle":"2024-12-25T07:58:05.424972Z","shell.execute_reply.started":"2024-12-25T07:58:04.765359Z","shell.execute_reply":"2024-12-25T07:58:05.424143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dfd.loc['min']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T07:24:37.665387Z","iopub.execute_input":"2024-12-25T07:24:37.665784Z","iopub.status.idle":"2024-12-25T07:24:37.674035Z","shell.execute_reply.started":"2024-12-25T07:24:37.665756Z","shell.execute_reply":"2024-12-25T07:24:37.672627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This shows that all values in numerical columns are 0 or positive. This could be useful in the future for missing value imputation.","metadata":{}},{"cell_type":"code","source":"numerical_columns = train_data.select_dtypes(include=np.number).columns\n\nnum_cols = len(numerical_columns)\ncols = 3  \nrows = 3  \n\nfig, axes = plt.subplots(rows, cols, figsize=(15, 12))  \n\naxes = axes.flatten()\nfor i, column in enumerate(numerical_columns):\n    ax = axes[i]\n    train_data[column].plot(kind='kde', ax=ax)\n    ax.set_title(f'Distribution: {column}')\n    ax.set_xlabel(column)\n    ax.set_ylabel('Density')\n\nfor i in range(num_cols, len(axes)):\n    axes[i].axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:18:55.343836Z","iopub.execute_input":"2024-12-25T08:18:55.344269Z","iopub.status.idle":"2024-12-25T08:22:08.264924Z","shell.execute_reply.started":"2024-12-25T08:18:55.344207Z","shell.execute_reply":"2024-12-25T08:22:08.263704Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Missing Value Indepth Analysis","metadata":{}},{"cell_type":"code","source":"train_data_2 = train_data.copy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:16:56.468281Z","iopub.execute_input":"2024-12-25T08:16:56.468708Z","iopub.status.idle":"2024-12-25T08:16:56.740806Z","shell.execute_reply.started":"2024-12-25T08:16:56.468677Z","shell.execute_reply":"2024-12-25T08:16:56.739506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_average_premium_by_bins(data, column_name, target_name='Premium Amount', bins=None, labels=None):\n    \"\"\"\n    Plots the average target variable by bins for a specified column, including missing values.\n\n    Parameters:\n        data (pd.DataFrame): The dataset.\n        column_name (str): The column to analyze.\n        target_name (str): The target variable. Default is 'Premium Amount'.\n        bins (list): The bins for numerical variables. If None, categorical variables are assumed.\n        labels (list): Labels for the bins. Required if bins are specified.\n    \"\"\"\n    # Check if the column is numerical or categorical\n    if bins:\n        # For numerical columns, create bins\n        binned_column = pd.cut(data[column_name], bins=bins, labels=labels, include_lowest=True)\n    else:\n        # For categorical columns, use unique values as categories\n        binned_column = data[column_name]\n    \n    # Add the binned column to the dataset\n    data[f'{column_name}_Binned'] = binned_column\n\n    # Calculate average of the target variable for non-missing values\n    non_missing_avg = (\n        data[data[f'{column_name}_Binned'].notnull()]\n        .groupby(f'{column_name}_Binned')[target_name]\n        .mean()\n    )\n\n    # Calculate average for missing values\n    missing_avg = data[data[column_name].isnull()][target_name].mean()\n\n    # Combine the non-missing and missing averages\n    avg_data = pd.concat(\n        [non_missing_avg, pd.Series({'Missing': missing_avg})]\n    )\n\n    plt.figure(figsize=(10, 6))\n    avg_data.plot(kind='bar', color='skyblue', edgecolor='black')\n\n    plt.title(f'Average {target_name} by Binned {column_name} (Including Missing Values)', fontsize=14)\n    plt.xlabel(f'{column_name} Bins (Including Missing)', fontsize=12)\n    plt.ylabel(f'Average {target_name}', fontsize=12)\n    plt.xticks(rotation=45)\n    plt.grid(axis='y', linestyle='--', alpha=0.7)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:16:57.315802Z","iopub.execute_input":"2024-12-25T08:16:57.31619Z","iopub.status.idle":"2024-12-25T08:16:57.325149Z","shell.execute_reply.started":"2024-12-25T08:16:57.316162Z","shell.execute_reply":"2024-12-25T08:16:57.323462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_average_premium_by_bins(\n    data=train_data_2,\n    column_name='Annual Income',\n    bins=[0, 10000, 20000, 40000, 60000, 80000, 100000, 140000],\n    labels=['0-10000', '10000-20000', '20000-40000', '40000-60000', '60000-80000', '80000-100000', '100000-140000']\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:17:00.782221Z","iopub.execute_input":"2024-12-25T08:17:00.782562Z","iopub.status.idle":"2024-12-25T08:17:01.491849Z","shell.execute_reply.started":"2024-12-25T08:17:00.782537Z","shell.execute_reply":"2024-12-25T08:17:01.490764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_average_premium_by_bins(\n    data=train_data_2,\n    column_name='Credit Score',\n    bins = [300, 400, 500, 600, 700, 800, 900],\n    labels = ['300-400', '400-500', '500-600', '600-700', '700-800', '800-900']\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:17:51.156697Z","iopub.execute_input":"2024-12-25T08:17:51.157174Z","iopub.status.idle":"2024-12-25T08:17:51.868842Z","shell.execute_reply.started":"2024-12-25T08:17:51.157135Z","shell.execute_reply":"2024-12-25T08:17:51.867728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nplot_average_premium_by_bins(\n    data=train_data_2,\n    column_name='Marital Status'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:16:26.72635Z","iopub.execute_input":"2024-12-25T08:16:26.726733Z","iopub.status.idle":"2024-12-25T08:16:27.53961Z","shell.execute_reply.started":"2024-12-25T08:16:26.726694Z","shell.execute_reply":"2024-12-25T08:16:27.538326Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_average_premium_by_bins(\n    data=train_data_2,\n    column_name='Vehicle Age'\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T08:23:45.306834Z","iopub.execute_input":"2024-12-25T08:23:45.307266Z","iopub.status.idle":"2024-12-25T08:23:46.049188Z","shell.execute_reply.started":"2024-12-25T08:23:45.307234Z","shell.execute_reply":"2024-12-25T08:23:46.04812Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### The above graphs show clearly that missing values contain valuable information as their dsitrbution with respect to average premium amount is quite different from the rest of bins (non missing values) of that particular column. This is farely common in financial datasets and stands as a reminder to always check missing value trends before performing imputation. ","metadata":{}},{"cell_type":"markdown","source":"## Please upvote if you find this useful. Thank You !!","metadata":{}}]}