{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\n<font size=\"4\">\nIn this Playground episode, the goal is to use regression to predict insurance Premium Amount based on 18 given features.  In this notebook, I will gloss over data preparation in order to focus on feature selection (FS).\n</font>","metadata":{}},{"cell_type":"code","source":"# Load packages.\nimport numpy as np\nimport pandas as pd\nimport statistics\nimport matplotlib.pyplot as plt\nfrom sklearn import preprocessing \nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import mean_squared_log_error, make_scorer\nfrom sklearn.linear_model import LinearRegression, SGDRegressor\nfrom catboost import CatBoostRegressor\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom sklearn.neural_network import MLPRegressor\nfrom sklearn.preprocessing import StandardScaler\nimport optuna\n\n# Define a function to convert character columns to catgorical.\ndef char_to_cat(df):\n    \"\"\"Converts all character columns in a DataFrame to categorical type.\"\"\"\n\n    # Select object (string) columns\n    object_cols = df.select_dtypes(include=['object']).columns\n\n    # Convert selected columns to categorical\n    df[object_cols] = df[object_cols].astype('category')\n\n    return df\n\n# Load data.\ntrain = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv', parse_dates = ['Policy Start Date'])\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv', parse_dates = ['Policy Start Date'])\n\n# Make new features from Policy Start Date.\ntrain2 = train.copy()\ntest2 = test.copy()\ntrain2['year'] = train2['Policy Start Date'].dt.year\ntrain2['month'] = train2['Policy Start Date'].dt.month\ntrain2['day'] = train2['Policy Start Date'].dt.day\ntest2['year'] = test2['Policy Start Date'].dt.year\ntest2['month'] = test2['Policy Start Date'].dt.month\ntest2['day'] = test2['Policy Start Date'].dt.day\n\n# Check for missing values.\ntrain2.isnull().sum()\n\n# Replace missing Age with 53.5.\nstatistics.median(train2['Age'])\ntrain2['Age'].fillna(53.5, inplace = True)\ntest2['Age'].fillna(53.5, inplace = True)\n\n# Replace missing Annual Income with 55212.\nstatistics.median(train2['Annual Income'])\ntrain2['Annual Income'].fillna(55212, inplace = True)\ntest2['Annual Income'].fillna(55212, inplace = True)\n\n# Replace missing Number of Dependents with 1.5.\nstatistics.median(train2['Number of Dependents'])\ntrain2['Number of Dependents'].fillna(1.5, inplace = True)\ntest2['Number of Dependents'].fillna(1.5, inplace = True)\n\n# Replace missing Health Score with 24.5.\nmin(train2['Health Score'])\nmax(train2['Health Score'])\ntrain2['Health Score'].mean()\nstatistics.median(train2['Health Score'])\ntrain2['Health Score'].fillna(24.5, inplace = True)\ntest2['Health Score'].fillna(24.5, inplace = True)\n\n# Replace missing Previous Claims with 1.0.\nstatistics.median(train2['Previous Claims'])\ntrain2['Previous Claims'].fillna(1.0, inplace = True)\ntest2['Previous Claims'].fillna(1.0, inplace = True)\n\n# Replace missing Credit Score with 640.5.\nstatistics.median(train2['Credit Score'])\ntrain2['Credit Score'].fillna(640.5, inplace = True)\ntest2['Credit Score'].fillna(640.5, inplace = True)\n\n# Replace missing Vehicle Age with 18.0.\nstatistics.median(train2['Vehicle Age'])\ntrain2['Vehicle Age'].fillna(18.0, inplace = True)\ntest2['Vehicle Age'].fillna(18.0, inplace = True)\n\n# Replace missing Insurance Duration with 5.0.\nstatistics.median(train2['Insurance Duration'])\ntrain2['Insurance Duration'].fillna(5.0, inplace = True)\ntest2['Insurance Duration'].fillna(5.0, inplace = True)\n\n# Replace missing Marital Status with unknown.\ntrain2['Marital Status'].value_counts()\ntrain2['Marital Status'].fillna('unknown', inplace = True)\ntest2['Marital Status'].fillna('unknown', inplace = True)\n\n# Replace missing Occupation with unknown.\ntrain2['Occupation'].value_counts()\ntrain2['Occupation'].fillna('unknown', inplace = True)\ntest2['Occupation'].fillna('unknown', inplace = True)\n\n# Replace missing Customer Feedback with unknown.\ntrain2['Customer Feedback'].value_counts()\ntrain2['Customer Feedback'].fillna('unknown', inplace = True)\ntest2['Customer Feedback'].fillna('unknown', inplace = True)\n\n# Drop Policy Start Date.\ntrain2.drop('Policy Start Date', axis = 1, inplace = True)\ntest2.drop('Policy Start Date', axis = 1, inplace = True)\n\n# Convert character columns to categorical.\ntrain3 = char_to_cat(train2.copy())\ntest3 = char_to_cat(test2.copy())\n\n# Label encode categorical features.\nlabel_encoder = preprocessing.LabelEncoder() \ntrain3lab = train3.copy()\ntrain3lab['Gender']= label_encoder.fit_transform(train3lab['Gender']) \ntrain3lab['Gender'].unique() \ntrain3lab['Marital Status']= label_encoder.fit_transform(train3lab['Marital Status']) \ntrain3lab['Education Level']= label_encoder.fit_transform(train3lab['Education Level']) \ntrain3lab['Occupation']= label_encoder.fit_transform(train3lab['Occupation']) \ntrain3lab['Location']= label_encoder.fit_transform(train3lab['Location']) \ntrain3lab['Policy Type']= label_encoder.fit_transform(train3lab['Policy Type']) \ntrain3lab['Customer Feedback']= label_encoder.fit_transform(train3lab['Customer Feedback']) \ntrain3lab['Smoking Status']= label_encoder.fit_transform(train3lab['Smoking Status']) \ntrain3lab['Exercise Frequency']= label_encoder.fit_transform(train3lab['Exercise Frequency']) \ntrain3lab['Property Type']= label_encoder.fit_transform(train3lab['Property Type']) \n\ntest3lab = test3.copy()\ntest3lab['Gender']= label_encoder.fit_transform(test3lab['Gender']) \ntest3lab['Gender'].unique() \ntest3lab['Marital Status']= label_encoder.fit_transform(test3lab['Marital Status']) \ntest3lab['Education Level']= label_encoder.fit_transform(test3lab['Education Level']) \ntest3lab['Occupation']= label_encoder.fit_transform(test3lab['Occupation']) \ntest3lab['Location']= label_encoder.fit_transform(test3lab['Location']) \ntest3lab['Policy Type']= label_encoder.fit_transform(test3lab['Policy Type']) \ntest3lab['Customer Feedback']= label_encoder.fit_transform(test3lab['Customer Feedback']) \ntest3lab['Smoking Status']= label_encoder.fit_transform(test3lab['Smoking Status']) \ntest3lab['Exercise Frequency']= label_encoder.fit_transform(test3lab['Exercise Frequency']) \ntest3lab['Property Type']= label_encoder.fit_transform(test3lab['Property Type']) \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:25:25.416712Z","iopub.execute_input":"2024-12-17T17:25:25.41718Z","iopub.status.idle":"2024-12-17T17:25:53.411474Z","shell.execute_reply.started":"2024-12-17T17:25:25.417139Z","shell.execute_reply":"2024-12-17T17:25:53.41025Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nI added three new features: day, month, and year, which are derived from Policy Start Date. <br />\nI will plot the target (Premium Amount) to see the distribution.\n</font>","metadata":{}},{"cell_type":"code","source":"# Plot the target.\nplt.hist(train3lab['Premium Amount'], bins = 5)\nplt.show()\nprint(min(train3lab['Premium Amount']))\nprint(max(train3lab['Premium Amount']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:08:21.075766Z","iopub.execute_input":"2024-12-13T22:08:21.076291Z","iopub.status.idle":"2024-12-13T22:08:21.550895Z","shell.execute_reply.started":"2024-12-13T22:08:21.076249Z","shell.execute_reply":"2024-12-13T22:08:21.549594Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nThe target is right-skewed with a range of 20-4999.\n</font>","metadata":{}},{"cell_type":"markdown","source":"# CatBoost Baseline\n\n<font size=\"4\">\nI will now define x (data) and y (label), scale the data, drop the id column, which I don't want in my model, and make a CatBoost baseline model.","metadata":{}},{"cell_type":"code","source":"# Define x and y.\nx = train3lab.drop('Premium Amount', axis=1)\ny = np.log1p(train3lab['Premium Amount'])  # Use log1p to handle RMSLE\n\n# Scale the train data.\nscaler = StandardScaler()\nx_scale = scaler.fit_transform(x)\nx_scale2 = pd.DataFrame(x_scale, columns=x.columns)\n\n# Drop id column.\nx_scale2.drop('id', axis = 1, inplace = True)\n\n# Scale the test data.\nscaler = StandardScaler()\ntest_scale = scaler.fit_transform(test3lab)\ntest_scale2 = pd.DataFrame(test_scale, columns=test3lab.columns)\n\n# Drop id column.\ntest_scale2.drop('id', axis = 1, inplace = True)\n\n# Make a CatBoost baseline model.\ncat1 = CatBoostRegressor(verbose = 0,\n                        random_seed = 12)\n\ncat1.fit(x_scale2, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:26:25.294728Z","iopub.execute_input":"2024-12-17T17:26:25.296066Z","iopub.status.idle":"2024-12-17T17:28:13.429633Z","shell.execute_reply.started":"2024-12-17T17:26:25.296016Z","shell.execute_reply":"2024-12-17T17:28:13.428378Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nNext, I will calculate a cross-validation (CV) score for my CatBoost baseline model using five-fold CV.\n</font>","metadata":{}},{"cell_type":"code","source":"# Define a function to compute RMSLE.\ndef rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))\n\n# Define custom RMSLE scorer.\nrmsle_scorer = make_scorer(rmsle, greater_is_better=False)\n\n# Calcualte CV score.\ncat1_score1 = -(cross_val_score(cat1, x_scale2, y, scoring = rmsle_scorer, cv = 5))\ncat1_score1.mean()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:28:32.191631Z","iopub.execute_input":"2024-12-17T17:28:32.192256Z","iopub.status.idle":"2024-12-17T17:35:15.542954Z","shell.execute_reply.started":"2024-12-17T17:28:32.192212Z","shell.execute_reply":"2024-12-17T17:35:15.54167Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nThe CV score for my CatBoost baseline model is 1.048.\n</font>","metadata":{}},{"cell_type":"markdown","source":"# Feature Selection\n\n<font size=\"4\">\nNow, I will get the feature importances to see which features are contributing the most to my model.\n</font>","metadata":{}},{"cell_type":"code","source":"# Get feature importances\nimportances = cat1.feature_importances_ \nfeature_importances = pd.Series(importances, index=x_scale2.columns)\nfeature_importances.sort_values(ascending=False, inplace=True)\n\n# Display feature importances\nprint(feature_importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T22:33:51.234605Z","iopub.execute_input":"2024-12-13T22:33:51.235778Z","iopub.status.idle":"2024-12-13T22:33:51.246106Z","shell.execute_reply.started":"2024-12-13T22:33:51.235723Z","shell.execute_reply":"2024-12-13T22:33:51.244528Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nThe top 10 features are contributing a lot to the model, while the others are contributing less.  I will exclude all features with a feature importance score of less than 1.\n</font>","metadata":{}},{"cell_type":"code","source":"# Only keep Annual Income, Credit Score, Health Score, Previous Claims, \n# Customer Feedback, year, Age, Marital Status, day, and Vehicle Age.\ncols2keep = ['Annual Income', 'Credit Score', 'Health Score', 'Previous Claims',\n             'Customer Feedback', 'year', 'Age', 'Marital Status', 'day',\n             'Vehicle Age']\n\ncat_train = x_scale2[cols2keep]\n\n# Process test data same as train data.\n\n# Keep the same 10 columns.\ncat_test = test_scale2[cols2keep]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:36:22.680337Z","iopub.execute_input":"2024-12-17T17:36:22.680714Z","iopub.status.idle":"2024-12-17T17:36:22.745134Z","shell.execute_reply.started":"2024-12-17T17:36:22.680683Z","shell.execute_reply":"2024-12-17T17:36:22.744121Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nNow I will make a new CatBoost model after feature selection and calculate the new CV score.\n</font>","metadata":{}},{"cell_type":"code","source":"cat2 = CatBoostRegressor(verbose = 0,\n                        random_seed = 17)\n\ncat2.fit(cat_train, y)\ncat2_score1 = -(cross_val_score(cat2, cat_train, y, scoring = rmsle_scorer, cv = 5))\ncat2_score1.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:46:46.415846Z","iopub.execute_input":"2024-12-17T17:46:46.41632Z","iopub.status.idle":"2024-12-17T17:54:12.627617Z","shell.execute_reply.started":"2024-12-17T17:46:46.416283Z","shell.execute_reply":"2024-12-17T17:54:12.626322Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nA CV score of 1.047 with only 10 features and default hyperparameters.  Not too shabby! <br />\nThe CV score improved slightly from 1.048 to 1.047 after feature selection.  \n</font>\n","metadata":{}},{"cell_type":"markdown","source":"# Feature Importance\n\n<font size=\"4\">\nI will now plot the feature importances of my new model.\n</font>","metadata":{}},{"cell_type":"code","source":"# Get feature importances\nfeature_importances2 = cat2.get_feature_importance()\n\n# Create a DataFrame for better visualization\nimportance_df = pd.DataFrame({\n    'Feature': cat_train.columns,\n    'Importance': feature_importances2\n})\n\n# Sort the DataFrame by importance\nimportance_df.sort_values(by='Importance', ascending=False, inplace=True)\n\n# Plot feature importances\nplt.figure(figsize=(10, 6))\nplt.barh(importance_df['Feature'], importance_df['Importance'], color='skyblue')\nplt.xlabel('Importance')\nplt.ylabel('Feature')\nplt.title('Feature Importances in CatBoost')\nplt.gca().invert_yaxis()  # Highest importance at the top\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T17:58:20.027836Z","iopub.execute_input":"2024-12-17T17:58:20.028304Z","iopub.status.idle":"2024-12-17T17:58:20.375277Z","shell.execute_reply.started":"2024-12-17T17:58:20.028271Z","shell.execute_reply":"2024-12-17T17:58:20.374101Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"<font size=\"4\">\nAnnual Income, Credit Score, and Health Score are clearly the most important features in this model.\n</font>","metadata":{}},{"cell_type":"markdown","source":"<font size=\"4\">\nFinally, I will predict and submit!  The predictions are clipped to be between 20 and 4999, which is the range of Premium Amount in the training data.\n</font>","metadata":{}},{"cell_type":"code","source":"# Predict\ncat2_pre = cat2.predict(cat_test)\ncat2_pred = np.expm1(cat2_pre)\ncat2_pred2 = np.clip(cat2_pred, 20.0, 4999.0)\n\n# Make a df of predictions.\ncat2_df = pd.DataFrame({'id':test['id'],\n                        'Premium Amount': cat2_pred2})\n\n# Submit\ncat2_df.to_csv('submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-13T23:15:45.628388Z","iopub.execute_input":"2024-12-13T23:15:45.628864Z","iopub.status.idle":"2024-12-13T23:15:47.73946Z","shell.execute_reply.started":"2024-12-13T23:15:45.628828Z","shell.execute_reply":"2024-12-13T23:15:47.738217Z"}},"outputs":[],"execution_count":null}]}