{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\ntrain_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\nsample_submission_csv = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-02-06T11:01:20.446658Z","iopub.execute_input":"2025-02-06T11:01:20.447164Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for beautiful dataframes\nfrom IPython.display import display\n\n# for Heatmaps and \nimport seaborn as sns\nsns.set(style=\"whitegrid\", color_codes=True)\n%matplotlib inline\n\n# for density plots and histograms\nimport matplotlib.pyplot as plt","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Data preparation","metadata":{}},{"cell_type":"markdown","source":"#### Exploring our data","metadata":{}},{"cell_type":"code","source":"train_csv.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Missing values per feature","metadata":{}},{"cell_type":"code","source":"# missing values in the datasets\ntrain_null = train_csv.isnull()\ntest_null = test_csv.isnull()\n\n# percentage of missing values per feature in the training set\ntrain_not_null = (len(train_csv) - train_null.sum()) / len(train_csv) *100\ntest_not_null = (len(test_csv) - test_null.sum()) / len(test_csv) *100\n\ndisplay(train_not_null)\n\n# create a single figure with subplots for both datasets\nfig, axes = plt.subplots(1, 2, figsize=(18, 6))\n\n# heatmap for missing values in the training dataset\nsns.heatmap(train_null, cmap='viridis', cbar=False, yticklabels=False, ax=axes[0])\naxes[0].set_title('Missing Values Heatmap - Training Dataset', fontsize=14)\naxes[0].set_xlabel('Features', fontsize=12)\naxes[0].set_ylabel('Entries', fontsize=12)\n\n# heatmap for missing values in the test dataset\nsns.heatmap(test_null, cmap='viridis', cbar=False, yticklabels=False, ax=axes[1])\naxes[1].set_title('Missing Values Heatmap - Test Dataset', fontsize=14)\naxes[1].set_xlabel('Features', fontsize=12)\naxes[1].set_ylabel('Entries', fontsize=12)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Baseline of 100% features","metadata":{}},{"cell_type":"code","source":"train_full = train_not_null[train_not_null == 100]\ntest_full = test_not_null[test_not_null == 100]\n\ntrain_full = train_csv[train_full.index]\ntest_full = test_csv[test_full.index]\n\ndisplay(train_full.head())\ndisplay(test_full.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Dropping the 'id' column to simplify the processing, adding it later on for the submission","metadata":{}},{"cell_type":"code","source":"train_id = train_full['id']\ntest_id = test_full['id']\n\ntrain_full = train_full.drop('id', axis=1)\ntest_full = test_full.drop('id', axis=1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### One-Hot Encoding for categorical features\n#### The feature \"Policy Start Date\" can't just be One-Hot encoded, so we're dropping it for now","metadata":{}},{"cell_type":"code","source":"# dropping \"Policy Start Date\"\ntrain_full = train_full.drop('Policy Start Date', axis=1)\ntest_full = test_full.drop('Policy Start Date', axis=1)\n\n# get the train OHEs\ntrain_full = pd.get_dummies(\n    data = train_full,\n    columns=['Gender', 'Education Level', 'Location', 'Policy Type', 'Smoking Status', 'Exercise Frequency', 'Property Type'],\n    drop_first=True,\n    dtype=int\n)\n\n# get the test OHEs\ntest_full = pd.get_dummies(\n    data = test_full,\n    columns=['Gender', 'Education Level', 'Location', 'Policy Type', 'Smoking Status', 'Exercise Frequency', 'Property Type'],\n    drop_first=True,\n    dtype=int\n)\n\ndisplay(train_full.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr = train_full.corr(method='pearson')\n\n# Too many features, this kind of heatmap would be too big\n'''\nsns.heatmap(\n    corr,\n    cmap='Wistia',\n    annot=True,\n    fmt='.1f',\n)\n'''\n#display(corr) No significant correlations","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Let's add Age and Annual Income to the data","metadata":{}},{"cell_type":"code","source":"continuous_feats = ['Age', 'Annual Income']\n\nfor c in continuous_feats:\n    ax = train_csv[c].hist(\n        bins=15,\n        density=True,\n        stacked=True,\n        color='teal',\n        alpha=0.6\n    )\n    train_csv[c].plot(kind='density', color='teal')\n    ax.set(xlabel=c)\n    \n    mean = train_csv[c].mean(skipna=True)\n    median = train_csv[c].median(skipna=True)\n    plt.axvline(mean, color='r', label=f'Mean: {mean:.3f}')\n    plt.axvline(median, color='y', label=f'Median: {median:.3f}')\n    \n    plt.legend()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Fill Age and Annual Income features","metadata":{}},{"cell_type":"code","source":"train_full['Age'] = train_csv['Age'].fillna(train_csv['Age'].mean(skipna=True))\ntest_full['Age'] = test_csv['Age'].fillna(test_csv['Age'].mean(skipna=True))\n\ntrain_full['Annual Income'] = train_csv['Annual Income'].fillna(train_csv['Annual Income'].median(skipna=True))\ntest_full['Annual Income'] = test_csv['Annual Income'].fillna(test_csv['Annual Income'].median(skipna=True))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Let's add more continous features","metadata":{}},{"cell_type":"code","source":"continuous_feats = ['Number of Dependents', 'Health Score', 'Vehicle Age', 'Credit Score', 'Insurance Duration']\n\nfor c in continuous_feats:\n    ax = train_csv[c].hist(\n        bins=15,\n        density=True,\n        stacked=True,\n        color='teal',\n        alpha=0.6\n    )\n    train_csv[c].plot(kind='density', color='teal')\n    ax.set(xlabel=c)\n    \n    mean = train_csv[c].mean(skipna=True)\n    median = train_csv[c].median(skipna=True)\n    plt.axvline(mean, color='r', label=f'Mean: {mean:.3f}')\n    plt.axvline(median, color='y', label=f'Median: {median:.3f}')\n    \n    plt.legend()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Fill these features","metadata":{}},{"cell_type":"code","source":"train_full['Number of Dependents'] = train_csv['Number of Dependents'].fillna(train_csv['Number of Dependents'].mean(skipna=True))\ntest_full['Number of Dependents'] = test_csv['Number of Dependents'].fillna(test_csv['Number of Dependents'].mean(skipna=True))\n\ntrain_full['Health Score'] = train_csv['Health Score'].fillna(train_csv['Health Score'].median(skipna=True))\ntest_full['Health Score'] = test_csv['Health Score'].fillna(test_csv['Health Score'].median(skipna=True))\n\ntrain_full['Vehicle Age'] = train_csv['Vehicle Age'].fillna(train_csv['Vehicle Age'].median(skipna=True))\ntest_full['Vehicle Age'] = test_csv['Vehicle Age'].fillna(test_csv['Vehicle Age'].median(skipna=True))\n\ntrain_full['Credit Score'] = train_csv['Credit Score'].fillna(train_csv['Credit Score'].median(skipna=True))\ntest_full['Credit Score'] = test_csv['Credit Score'].fillna(test_csv['Credit Score'].median(skipna=True))\n\ntrain_full['Insurance Duration'] = train_csv['Insurance Duration'].fillna(train_csv['Insurance Duration'].mean(skipna=True))\ntest_full['Insurance Duration'] = test_csv['Insurance Duration'].fillna(test_csv['Insurance Duration'].mean(skipna=True))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Taking a look at the continuous variables 'Marital Status' and 'Customer Feedback'","metadata":{}},{"cell_type":"code","source":"# plot for the 'Marital Status' feature\nsns.countplot(x='Marital Status', data=train_csv)\n\n# add numbers above columns\nfor index, value in enumerate(train_csv['Marital Status'].value_counts()):\n    plt.text(index, value+1, str(value), ha='center')\nplt.show()\n\nprint(f\"Most common label: {train_csv['Marital Status'].value_counts().idxmax()}\")\n\n# plot for the 'Customer Feedback' feature\nsns.countplot(x='Customer Feedback', data=train_csv)\n\n# add numbers above columns\nfor index, value in enumerate(train_csv['Customer Feedback'].value_counts()):\n    plt.text(index, value+1, str(value), ha='center')\nplt.show()\n\nprint(f\"Most common label: {train_csv['Customer Feedback'].value_counts().idxmax()}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Model Evaluation, Training and Submission","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Preparation for the scaling procedures and definition of the loss function","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.compose import ColumnTransformer                  # to perform the scaling only on non-OHE features\nfrom sklearn.metrics import make_scorer, mean_squared_log_error\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.pipeline import Pipeline                          # to apply the StandardScaler to the k-folds within the cross-validation procedure\nfrom sklearn.preprocessing import MinMaxScaler\n\n# train data organization for cross-validation\nX = train_full.drop('Premium Amount', axis=1)\ny = train_full['Premium Amount']\n\n# specify the columns for different transformations\nnon_ohe_features = ['Age', 'Annual Income', 'Number of Dependents', 'Health Score', 'Vehicle Age', 'Credit Score', 'Insurance Duration']\nohe_features = [col for col in X.columns if col not in non_ohe_features]\n\n# define the ColumnTransformer\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('scaler', MinMaxScaler(), non_ohe_features),  # scale only the non-OHE features\n        ('passthrough', 'passthrough', ohe_features)     # keep OHE features as they are\n    ]\n)\n\n# define the loss function\ndef root_mean_squared_log_error(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\n# create the scorer to pass for the cross-validation\nrmsle_scorer = make_scorer(root_mean_squared_log_error, greater_is_better=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Linear Regression","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nmodel = LinearRegression()\n\npipeline = Pipeline([\n    ('preprocessor', preprocessor),\n    ('regressor', model)\n])\n\n# cross-validation\nscores_rmsle = cross_val_score(pipeline, X, y, cv=10, scoring=rmsle_scorer)\n\nprint('K-fold cross-validation results:')\nprint(model.__class__.__name__ + \" average RMSLE is %.3f\" % -scores_rmsle.mean())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.3 XGBoost Regressor","metadata":{}},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\nmodel = XGBRegressor(\n    n_estimators=100,\n    random_state=42\n)\n\npipeline = Pipeline([\n    ('preprocessor', preprocessor),\n    ('regressor', model)\n])\n\n# cross-validation\nscores_rmsle = cross_val_score(pipeline, X, y, cv=5, scoring=rmsle_scorer)\n\nprint('K-fold cross-validation results:')\nprint(model.__class__.__name__ + \" average RMSLE is %.3f\" % -scores_rmsle.mean())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.4 Training and Submission","metadata":{}},{"cell_type":"code","source":"# model instantiation\nmodel = XGBRegressor(\n    n_estimators=100,\n    random_state=42\n)\n\n# pipeline definition with the final model\npipeline = Pipeline([\n    ('preprocessor', preprocessor),\n    ('regressor', model)\n])\n\n# train and prediction\npipeline.fit(X, y)\ntest_full['Premium Amount'] = pipeline.predict(test_full)\n\n# submission crafting\ntest_full['id'] = test_id\nsubmission = test_full[['id', 'Premium Amount']]\n\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}