{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":10294148,"sourceType":"datasetVersion","datasetId":6291335}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.pipeline import make_pipeline, Pipeline\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder, LabelEncoder, label_binarize, OrdinalEncoder\nfrom category_encoders import CatBoostEncoder, MEstimateEncoder\n\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier, HistGradientBoostingClassifier, GradientBoostingClassifier, HistGradientBoostingRegressor\nfrom sklearn.linear_model import RidgeClassifier, LogisticRegression, LinearRegression\n\nfrom sklearn import set_config\nimport os\n\nimport optuna\nfrom sklearn.metrics import accuracy_score, roc_auc_score, roc_curve, mean_squared_error, precision_recall_curve, make_scorer, confusion_matrix, ConfusionMatrixDisplay, RocCurveDisplay, matthews_corrcoef\nfrom scipy.stats import norm, skew\n\nfrom colorama import Fore, Style, init\nfrom copy import deepcopy\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom pprint import pprint\nfrom sklearn.model_selection import train_test_split, RepeatedStratifiedKFold, StratifiedKFold, KFold, RepeatedKFold, cross_val_score, StratifiedGroupKFold\nfrom xgboost import DMatrix, XGBClassifier, XGBRegressor\nfrom lightgbm import log_evaluation, early_stopping, LGBMClassifier, LGBMRegressor, Dataset\nfrom catboost import CatBoostClassifier, CatBoostRegressor, Pool\nfrom tqdm.notebook import tqdm\nfrom optuna.samplers import TPESampler, CmaEsSampler\nfrom optuna.pruners import HyperbandPruner\nfrom functools import partial\nfrom IPython.display import display_html, clear_output\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.impute import SimpleImputer, KNNImputer\nfrom sklearn.compose import ColumnTransformer\nimport gc\nimport re\n\nimport keras\nfrom keras.models import Sequential\nfrom keras import layers\nimport tensorflow as tf\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras import regularizers\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:08:37.532943Z","iopub.execute_input":"2024-12-26T11:08:37.533326Z","iopub.status.idle":"2024-12-26T11:08:55.015538Z","shell.execute_reply.started":"2024-12-26T11:08:37.533289Z","shell.execute_reply":"2024-12-26T11:08:55.014313Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Dataset Description**\n\n1. **id**: Unique identifier for each record.\n2. **Age**: Age of the individual (in years)\n3. **Gender**: Gender of the individual (e.g., Male, Female).\n4. **Annual Income**: Annual income of the individual (in the dataset’s monetary unit).\n5. **Marital Status**: Marital status of the individual (e.g., Married, Divorced, Single).\n6. **Number of Dependents**: Number of dependents associated with the individual.\n7. **Education Level**: Highest level of education attained (e.g., High School, Bachelor’s, Master’s).\n8. **Occupation**: Job type or employment status (e.g., Self-Employed).\n9. **Health Score**: A numerical score representing the health status of the individual.\n10. **Location**: Living area of the individual (e.g., Urban, Rural, Suburban).\n11. **Previous Claims**: Number of previous claims made by the individual.\n12. **Vehicle Age**: Age of the vehicle owned by the individual (in years).\n13. **Credit Score**: Creditworthiness score of the individual.\n14. **Insurance Duration**: Duration of the insurance policy (in years).\n15. **Policy Start Date**: The start date of the individual’s insurance policy.\n16. **Customer Feedback**: Overall feedback provided by the customer (e.g., Poor, Average, Good).\n17. **Smoking Status**: Smoking habits of the individual (e.g., Yes, No).\n18. **Exercise Frequency**: Frequency of exercise (e.g., Weekly, Monthly, Daily).\n19. **Property Type**: Type of property insured (e.g., House, Apartment).\n20. **Premium Amount**: Premium amount paid by the individual for the insurance.","metadata":{}},{"cell_type":"markdown","source":"#

Configuration

","metadata":{}},{"cell_type":"code","source":"class Config:\n \n state = 42\n n_splits = 10\n early_stop = 200\n \n target = 'Premium Amount'\n train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv', index_col='id')\n test = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv', index_col='id')\n submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\n original_data = 'N'\n outliers = 'N'\n log_trf = 'Y'\n scaler_trf = 'N'\n feature_eng = 'Y'\n missing = 'Y'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:08:55.018012Z","iopub.execute_input":"2024-12-26T11:08:55.018697Z","iopub.status.idle":"2024-12-26T11:09:05.422313Z","shell.execute_reply.started":"2024-12-26T11:08:55.018657Z","shell.execute_reply":"2024-12-26T11:09:05.421129Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#

EDA

","metadata":{}},{"cell_type":"code","source":"class EDA(Config):\n \n def __init__(self):\n super().__init__()\n\n self.cat_features = self.train.drop(self.target, axis=1).select_dtypes(include=['object']).columns.tolist()\n self.num_features = self.train.drop(self.target, axis=1).select_dtypes(exclude=['object']).columns.tolist()\n self.data_info()\n self.heatmap()\n self.dist_plots()\n self.cat_feature_plots()\n self.target_plot()\n \n def data_info(self):\n \n for data, label in zip([self.train, self.test], ['Train', 'Test']):\n table_style = [{'selector': 'th:not(.index_name)',\n 'props': [('background-color', '#3cb371'),\n ('color', '#FFFFFF'),\n ('font-weight', 'bold'),\n ('border', '1px solid #DCDCDC'),\n ('text-align', 'center')]\n }, \n {'selector': 'tbody td',\n 'props': [('border', '1px solid #DCDCDC'),\n ('font-weight', 'normal')]\n }]\n print(Style.BRIGHT+Fore.GREEN+f'\\n{label} head\\n')\n display(data.head().style.set_table_styles(table_style))\n \n print(Style.BRIGHT+Fore.GREEN+f'\\n{label} info\\n'+Style.RESET_ALL) \n display(data.info())\n \n print(Style.BRIGHT+Fore.GREEN+f'\\n{label} describe\\n')\n display(data.describe().drop(index='count', columns=self.target, errors = 'ignore').T\n .style.set_table_styles(table_style).format('{:.3f}'))\n \n print(Style.BRIGHT+Fore.GREEN+f'\\n{label} missing values\\n'+Style.RESET_ALL) \n display(data.isna().sum())\n return self\n \n def heatmap(self):\n print(Style.BRIGHT+Fore.GREEN+f'\\nCorrelation Heatmap\\n')\n plt.figure(figsize=(7,7))\n corr = self.train.select_dtypes(exclude='object').corr(method='pearson')\n sns.heatmap(corr, fmt = '0.4f', cmap = 'Greens', annot=True, cbar=False)\n plt.show()\n \n def dist_plots(self):\n print(Style.BRIGHT+Fore.GREEN+f\"\\nDistribution analysis\\n\")\n df = pd.concat([self.train[self.num_features].assign(Source = 'Train'), \n self.test[self.num_features].assign(Source = 'Test'),], \n axis=0, ignore_index = True)\n\n fig, axes = plt.subplots(len(self.num_features), 2 ,figsize = (18, len(self.num_features) * 6), \n gridspec_kw = {'hspace': 0.3, \n 'wspace': 0.2, \n 'width_ratios': [0.70, 0.30]\n }\n )\n for i,col in enumerate(self.num_features):\n ax = axes[i,0]\n sns.kdeplot(data = df[[col, 'Source']], x = col, hue = 'Source', \n palette = ['#3cb371', 'r'], ax = ax, linewidth = 2\n )\n ax.set(xlabel = '', ylabel = '')\n ax.set_title(f\"\\n{col}\")\n ax.grid()\n\n ax = axes[i,1]\n sns.boxplot(data = df, y = col, x=df.Source, width = 0.5,\n linewidth = 1, fliersize= 1,\n ax = ax, palette=['#3cb371', 'r']\n )\n ax.set_title(f\"\\n{col}\")\n ax.set(xlabel = '', ylabel = '')\n ax.tick_params(axis='both', which='major')\n ax.set_xticklabels(['Train', 'Test'])\n\n plt.tight_layout()\n plt.show()\n \n def cat_feature_plots(self):\n fig, axes = plt.subplots(len(self.cat_features), 2 ,figsize = (18, len(self.cat_features) * 6), \n gridspec_kw = {'hspace': 0.5, \n 'wspace': 0.2,\n }\n )\n\n for i, col in enumerate(self.cat_features):\n \n ax = axes[i,0]\n sns.barplot(data=self.train[col].value_counts().nlargest(10).reset_index(), x=col, y='count', ax=ax, color='#3cb371')\n ax.set(xlabel = '', ylabel = '')\n ax.set_title(f\"\\n{col} Train\")\n \n ax = axes[i,1]\n sns.barplot(data=self.train[col].value_counts().nlargest(10).reset_index(), x=col, y='count', ax=ax, color='r')\n ax.set(xlabel = '', ylabel = '')\n ax.set_title(f\"\\n{col} Test\")\n\n plt.tight_layout()\n plt.show()\n \n def target_plot(self):\n print(Style.BRIGHT+Fore.GREEN+f\"\\nTarget feature distribution\\n\")\n \n fig, axes = plt.subplots(1, 2 ,figsize = (14, 6), \n gridspec_kw = {'hspace': 0.3, \n 'wspace': 0.2, \n 'width_ratios': [0.70, 0.30]\n }\n )\n ax = axes[0]\n sns.kdeplot(data = self.train[self.target], \n color = '#3cb371', ax = ax, linewidth = 2\n )\n ax.set(xlabel = '', ylabel = '')\n ax.set_title(f\"\\n{self.target}\")\n ax.grid()\n\n ax = axes[1]\n sns.boxplot(data = self.train, y = self.target, width = 0.5,\n linewidth = 1, fliersize= 1,\n ax = ax, color = '#3cb371'\n )\n ax.set_title(f\"\\n{self.target}\")\n ax.set(xlabel = '', ylabel = '')\n ax.tick_params(axis='both', which='major')\n\n plt.tight_layout()\n plt.show() ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:09:05.42408Z","iopub.execute_input":"2024-12-26T11:09:05.42455Z","iopub.status.idle":"2024-12-26T11:09:05.448951Z","shell.execute_reply.started":"2024-12-26T11:09:05.424494Z","shell.execute_reply":"2024-12-26T11:09:05.447469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eda = EDA()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:09:05.450307Z","iopub.execute_input":"2024-12-26T11:09:05.450742Z","iopub.status.idle":"2024-12-26T11:10:50.752947Z","shell.execute_reply.started":"2024-12-26T11:09:05.450708Z","shell.execute_reply":"2024-12-26T11:10:50.751283Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#

Data Transformation

","metadata":{}},{"cell_type":"code","source":"class Transform(Config):\n \n def __init__(self):\n super().__init__()\n if Config.original_data == 'Y':\n self.train = pd.concat([self.train, self.train_org], ignore_index=True).drop_duplicates()\n self.train.reset_index(drop=True, inplace=True)\n\n self.num_features = self.train.drop(self.target, axis=1).select_dtypes(exclude=['object', 'bool']).columns.tolist()\n self.cat_features = self.train.drop(self.target, axis=1).select_dtypes(include=['object', 'bool']).columns.tolist()\n \n if self.missing == 'Y':\n self.missing_values()\n \n self.train_raw = self.train.copy()\n \n if self.feature_eng == 'Y':\n self.train = self.new_features(self.train)\n self.test = self.new_features(self.test)\n self.train_raw = self.new_features(self.train_raw)\n \n self.num_features = self.train.drop(self.target, axis=1).select_dtypes(exclude=['object', 'bool']).columns.tolist()\n self.cat_features = self.train.drop(self.target, axis=1).select_dtypes(include=['object', 'bool']).columns.tolist()\n \n if self.outliers == 'Y': \n self.remove_outliers()\n \n if self.log_trf == 'Y':\n self.log_transformation()\n \n if self.scaler_trf == 'Y':\n self.scaler()\n \n self.train_enc = self.train.copy()\n self.test_enc = self.test.copy()\n self.encode()\n \n if self.outliers == 'Y' or self.log_trf == 'Y' or self.scaler_trf =='Y':\n self.distribution()\n \n def __call__(self):\n\n self.train[self.cat_features] = self.train[self.cat_features].astype('category')\n self.test[self.cat_features] = self.test[self.cat_features].astype('category')\n\n self.cat_features_card = []\n for f in self.cat_features:\n self.cat_features_card.append(self.train[f].nunique())\n\n self.train = self.reduce_mem(self.train)\n self.test = self.reduce_mem(self.test)\n \n self.y = self.train[self.target]\n self.train = self.train.drop(self.target, axis=1)\n self.train_enc = self.train_enc.drop(self.target, axis=1)\n \n return self.train, self.train_enc, self.y, self.test, self.test_enc, self.cat_features\n \n def encode(self):\n data = pd.concat([self.test, self.train])\n oe = OrdinalEncoder()\n data[self.cat_features] = oe.fit_transform(data[self.cat_features]).astype('int')\n \n scaler = StandardScaler()\n data[self.num_features + [self.target]] = scaler.fit_transform(data[self.num_features + [self.target]])\n \n self.train_enc = data[~data[self.target].isna()]\n self.test_enc = data[data[self.target].isna()].drop(self.target, axis=1)\n \n def new_features(self, data): \n data['Policy Start Date'] = pd.to_datetime(data['Policy Start Date'])\n data['Year'] = data['Policy Start Date'].dt.year\n data.drop('Policy Start Date', axis=1, inplace=True)\n return data\n\n def log_transformation(self):\n self.train[self.target] = np.log1p(self.train[self.target]) \n \n return self\n \n def distribution(self):\n num_features = self.num_features + [self.target]\n print(Style.BRIGHT+Fore.GREEN+f'\\nHistograms of distribution\\n')\n fig, axes = plt.subplots(nrows=len(num_features), ncols=2, figsize=(15, len(num_features)*5))\n\n for (ax_r, ax_n), col in zip(axes, num_features):\n\n ax_r.set_title(f'{col} ($\\mu=$ {self.train_raw[col].mean():.2f} and $\\sigma=$ {self.train_raw[col].std():.2f} )')\n ax_r.hist(self.train_raw[col], bins=30, color='#3cb371')\n ax_r.axvline(self.train_raw[col].mean(), color='r', label='Mean')\n ax_r.axvline(self.train_raw[col].median(), color='y', linestyle='--', label='Median')\n ax_r.legend()\n\n ax_n.set_title(f'{col} Normalized ($\\mu=$ {self.train_enc[col].mean():.2f} and $\\sigma=$ {self.train_enc[col].std():.2f} )')\n ax_n.hist(self.train_enc[col], bins=30, color='#3cb371')\n ax_n.axvline(self.train_enc[col].mean(), color='r', label='Mean')\n ax_n.axvline(self.train_enc[col].median(), color='y', linestyle='--', label='Median')\n ax_n.legend()\n \n def remove_outliers(self):\n Q1 = self.train[self.targets].quantile(0.25)\n Q3 = self.train[self.targets].quantile(0.75)\n IQR = Q3 - Q1\n lower_limit = Q1 - 1.5*IQR\n upper_limit = Q3 + 1.5*IQR\n self.train = self.train[(self.train[self.targets] >= lower_limit) & (self.train[self.targets] <= upper_limit)]\n self.train.reset_index(drop=True, inplace=True) \n \n def scaler(self):\n scaler = StandardScaler()\n self.train[self.num_features] = scaler.fit_transform(self.train[self.num_features])\n self.test[self.num_features] = scaler.transform(self.test[self.num_features])\n return self\n \n def missing_values(self):\n\n self.train[self.cat_features] = self.train[self.cat_features].fillna('None')\n self.test[self.cat_features] = self.test[self.cat_features].fillna('None')\n return self\n\n def reduce_mem(self, df):\n\n numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64', \"uint16\", \"uint32\", \"uint64\"]\n \n for col in df.columns:\n col_type = df[col].dtypes\n \n if col_type in numerics:\n c_min = df[col].min()\n c_max = df[col].max()\n\n if \"int\" in str(col_type):\n if c_min >= np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n df[col] = df[col].astype(np.int8)\n elif c_min >= np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n df[col] = df[col].astype(np.int16)\n elif c_min >= np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n df[col] = df[col].astype(np.int32)\n elif c_min >= np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n df[col] = df[col].astype(np.int64) \n else:\n if c_min >= np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n df[col] = df[col].astype(np.float16)\n if c_min >= np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n df[col] = df[col].astype(np.float32)\n else:\n df[col] = df[col].astype(np.float64) \n\n return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:10:50.757057Z","iopub.execute_input":"2024-12-26T11:10:50.75758Z","iopub.status.idle":"2024-12-26T11:10:50.79859Z","shell.execute_reply.started":"2024-12-26T11:10:50.757527Z","shell.execute_reply":"2024-12-26T11:10:50.79746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"t = Transform()\nX, X_enc, y, test, test_enc, cat_features = t()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:10:50.79994Z","iopub.execute_input":"2024-12-26T11:10:50.800397Z","iopub.status.idle":"2024-12-26T11:11:17.736795Z","shell.execute_reply.started":"2024-12-26T11:10:50.800321Z","shell.execute_reply":"2024-12-26T11:11:17.735095Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#

Model Training

","metadata":{}},{"cell_type":"code","source":"from sklearn.base import BaseEstimator, RegressorMixin\nimport contextlib, io\nimport ydf; ydf.verbose(2)\nfrom ydf import GradientBoostedTreesLearner\n\ndef YDFRegressor(learner_class):\n\n class YDFXRegressor(BaseEstimator, RegressorMixin):\n\n def __init__(self, params={}):\n self.params = params\n\n def fit(self, X, y):\n assert isinstance(X, pd.DataFrame)\n assert isinstance(y, pd.Series)\n target = y.name\n params = self.params.copy()\n params['label'] = target\n params['task'] = ydf.Task.REGRESSION\n X = pd.concat([X, y], axis=1)\n with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()):\n self.model = learner_class(**params).train(X)\n return self\n\n def predict(self, X):\n assert isinstance(X, pd.DataFrame)\n with contextlib.redirect_stderr(io.StringIO()), contextlib.redirect_stdout(io.StringIO()):\n return self.model.predict(X)\n\n return YDFXRegressor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:11:17.738478Z","iopub.execute_input":"2024-12-26T11:11:17.739212Z","iopub.status.idle":"2024-12-26T11:11:18.017097Z","shell.execute_reply.started":"2024-12-26T11:11:17.739157Z","shell.execute_reply":"2024-12-26T11:11:18.015591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models = {\n 'YDF': [YDFRegressor(GradientBoostedTreesLearner)({'num_trees': 1000,\n 'max_depth': 13,\n }),\n False],\n 'CAT': [CatBoostRegressor(**{'verbose': 0,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': \"RMSE\",\n 'n_estimators' : 2000,\n }),\n False],\n 'CAT2': [CatBoostRegressor(**{'verbose': 0,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': \"RMSE\",\n 'n_estimators' : 3000,\n 'objective': 'RMSE', \n 'depth': 11,\n 'min_data_in_leaf': 78,\n 'l2_leaf_reg': 6.628202811121375,\n 'max_bin': 8000,\n 'learning_rate': 0.01,\n }),\n False],\n 'CAT3': [CatBoostRegressor(**{'verbose': 0,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 200,\n 'eval_metric': \"RMSE\",\n 'objective': 'RMSE', \n 'depth': 11,\n 'min_data_in_leaf': 72, \n 'l2_leaf_reg': 2.005119577280367,\n \"iterations\" : 3000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n }),\n False],\n 'CAT4': [CatBoostRegressor(**{'verbose': 0,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 200,\n 'eval_metric': \"RMSE\",\n 'objective': 'RMSE', \n 'depth': 13, \n 'subsample': 0.9998449533801151,\n 'min_data_in_leaf': 63,\n 'l2_leaf_reg': 0.3242106539167982,\n \"iterations\" : 3000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'bootstrap_type': 'Poisson'\n }),\n False],\n 'CAT5': [CatBoostRegressor(**{'objective': 'RMSE',\n 'depth': 11,\n 'subsample': 0.9581878706329088,\n 'min_data_in_leaf': 43,\n 'l2_leaf_reg': 0.08249502036269224,\n 'verbose': 0,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 200,\n 'eval_metric': \"RMSE\",\n 'task_type': \"GPU\",\n \"iterations\" : 5000,\n 'learning_rate': 0.005,\n 'max_bin': 8000,\n 'bootstrap_type': 'Bernoulli'\n }),\n False],\n 'CAT6': [CatBoostRegressor(**{'objective': 'RMSE',\n 'logging_level': 'Silent',\n 'depth': 11,\n 'min_data_in_leaf': 62,\n 'l2_leaf_reg': 7.887761285633826,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 300,\n 'eval_metric': \"RMSE\",\n 'task_type': \"GPU\",\n \"iterations\" : 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'bootstrap_type': 'Poisson'\n }),\n False],\n 'CAT8': [CatBoostRegressor(**{'objective': 'RMSE',\n 'depth': 11, \n 'min_data_in_leaf': 51, \n 'l2_leaf_reg': 7.51216069323258,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 200,\n 'eval_metric': \"RMSE\",\n 'task_type': \"GPU\",\n \"iterations\" : 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'bootstrap_type': 'Bernoulli'\n }),\n False],\n 'CAT9': [CatBoostRegressor(**{'objective': 'RMSE',\n 'depth': 10, \n 'min_data_in_leaf': 18,\n 'l2_leaf_reg': 9.044302614615026,\n 'random_state': Config.state,\n 'cat_features': cat_features,\n 'early_stopping_rounds': 200,\n 'eval_metric': \"RMSE\",\n 'task_type': \"GPU\",\n \"iterations\" : 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'bootstrap_type': 'Bernoulli'\n }),\n False],\n 'XGB': [XGBRegressor(**{'tree_method': 'hist',\n 'n_estimators': 2000,\n 'objective': 'reg:squarederror',\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n }),\n False],\n 'XGB2': [XGBRegressor(**{'tree_method': 'hist',\n 'n_estimators': 2000,\n 'objective': 'reg:squarederror',\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n 'booster': 'gbtree',\n 'max_depth': 9,\n 'learning_rate': 0.0031740845858378402, \n 'min_child_weight': 16,\n 'subsample': 0.8666770334099294,\n 'reg_alpha': 0.191908421033219,\n 'reg_lambda': 0.7995814007464592,\n 'colsample_bytree': 0.9669446980405391,\n 'random_state': 42,\n 'objective': 'reg:squarederror',\n 'n_jobs': -1,\n }),\n False],\n 'XGB3': [XGBRegressor(**{'booster': 'gbtree', \n 'max_depth': 9,\n 'min_child_weight': 13,\n 'subsample': 0.8626964099130539,\n 'reg_alpha': 0.058057084885141746,\n 'reg_lambda': 0.6326297976784753,\n 'colsample_bytree': 0.8940282896063437,\n 'n_jobs': -1,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'tree_method': 'hist',\n 'n_estimators': 2000,\n 'objective': 'reg:squarederror',\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n }),\n False],\n 'XGB4': [XGBRegressor(**{'booster': 'gbtree',\n 'max_depth': 8,\n 'min_child_weight': 17,\n 'subsample': 0.8926967587245219,\n 'reg_alpha': 0.2968941900087133,\n 'reg_lambda': 0.8270727714379025,\n 'colsample_bytree': 0.9318890724874014,\n 'n_jobs': -1,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'tree_method': 'hist',\n 'n_estimators': 3000,\n 'objective': 'reg:squarederror',\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n }),\n False],\n 'XGB5': [XGBRegressor(**{'booster': 'gbtree', \n 'max_depth': 9,\n 'min_child_weight': 13,\n 'subsample': 0.8950190330733485, \n 'reg_alpha': 0.3530275785494063,\n 'reg_lambda': 0.9564526039891854, \n 'colsample_bytree': 0.8488129245455193,\n 'random_state': 42,\n 'objective': 'reg:squarederror',\n 'n_jobs': -1,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'tree_method': 'hist',\n 'n_estimators': 3000,\n 'objective': 'reg:squarederror',\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n }),\n False],\n 'XGB6': [XGBRegressor(**{'booster': 'gbtree',\n 'max_depth': 10, \n 'min_child_weight': 19,\n 'subsample': 0.8882395046974123,\n 'reg_alpha': 0.12361226000167053,\n 'reg_lambda': 0.7568174249367466,\n 'colsample_bytree': 0.83845902004046,\n 'objective': 'reg:squarederror',\n 'n_jobs': -1,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'tree_method': 'hist',\n 'n_estimators': 5000,\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n 'device': \"cuda\",\n }),\n False],\n 'XGB8': [XGBRegressor(**{'booster': 'gbtree',\n 'max_depth': 10,\n 'min_child_weight': 11,\n 'subsample': 0.8709310286076404, \n 'reg_alpha': 0.2429313136646974,\n 'reg_lambda': 0.9443233420006127,\n 'colsample_bytree': 0.8004450303825076,\n 'random_state': 42, \n 'objective': 'reg:squarederror', \n 'n_jobs': -1,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'tree_method': 'hist',\n 'n_estimators': 5000,\n 'random_state': Config.state,\n 'enable_categorical': True,\n 'verbosity': 0,\n 'early_stopping_rounds': Config.early_stop,\n 'eval_metric': 'rmse',\n 'device': \"cuda\",\n }),\n False],\n 'LGBM': [LGBMRegressor(**{'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'boosting_type': 'gbdt',\n 'n_estimators': 3000,\n 'eval_metric': 'rmse',\n 'objective': 'regression_l2',\n }),\n False],\n 'LGBM2': [LGBMRegressor(**{'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'boosting_type': 'gbdt',\n 'n_estimators': 3000,\n 'eval_metric': 'rmse',\n 'objective': 'regression_l2',\n 'max_depth': 9,\n 'num_leaves': 722,\n 'min_child_samples': 27,\n 'min_child_weight': 14,\n 'colsample_bytree': 0.49498912736255835,\n 'reg_alpha': 0.3428923649002299, \n 'reg_lambda': 0.6429288241470125,\n 'learning_rate': 0.01,\n }),\n False],\n 'LGBM3': [LGBMRegressor(**{'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'boosting_type': 'gbdt',\n 'n_estimators': 3000,\n 'eval_metric': 'rmse',\n 'objective': 'regression_l2',\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n 'max_depth': 8, \n 'num_leaves': 796,\n 'min_child_samples': 21,\n 'min_child_weight': 11,\n 'colsample_bytree': 0.48551477424501904, \n 'reg_alpha': 0.44307695050069845, \n 'reg_lambda': 0.9310703189038081, \n }),\n False],\n 'LGBM4': [LGBMRegressor(**{'objective': 'regression_l2',\n 'metric': 'rmse',\n 'max_depth': 10, \n 'num_leaves': 133,\n 'min_child_samples': 18, \n 'min_child_weight': 14, \n 'colsample_bytree': 0.4997666632394159, \n 'reg_alpha': 0.2813104886322961, \n 'reg_lambda': 0.8049716583639613,\n 'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'boosting_type': 'goss',\n 'n_estimators': 3000,\n 'learning_rate': 0.01,\n }),\n False],\n 'LGBM5': [LGBMRegressor(**{'objective': 'regression_l2',\n 'metric': 'rmse',\n 'max_depth': 12, \n 'num_leaves': 675, \n 'min_child_samples': 10,\n 'min_child_weight': 19,\n 'colsample_bytree': 0.49534353968335604,\n 'reg_alpha': 0.3747939308000606,\n 'reg_lambda': 0.5472625114112711, \n 'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'n_estimators': 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n }),\n False],\n 'LGBM6': [LGBMRegressor(**{'objective': 'regression_l2',\n 'metric': 'rmse',\n 'max_depth': 12,\n 'num_leaves': 246,\n 'min_child_samples': 10,\n 'min_child_weight': 24,\n 'colsample_bytree': 0.49704729723832064,\n 'reg_alpha': 0.2791624804454722,\n 'reg_lambda': 0.7306747260034717,\n 'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'n_estimators': 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000\n }),\n False],\n 'LGBM8': [LGBMRegressor(**{'objective': 'regression_l2',\n 'metric': 'rmse', \n 'max_depth': 7,\n 'num_leaves': 108,\n 'min_child_samples': 13,\n 'min_child_weight': 22,\n 'colsample_bytree': 0.4781013294503911,\n 'reg_alpha': 0.35450860719581045, \n 'reg_lambda': 0.6395930802901353, \n 'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'boosting_type': 'goss',\n 'n_estimators': 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n }),\n False],\n 'LGBM9': [LGBMRegressor(**{'objective': 'regression_l2',\n 'metric': 'rmse',\n 'max_depth': 11,\n 'num_leaves': 169, \n 'min_child_samples': 27,\n 'min_child_weight': 13,\n 'colsample_bytree': 0.4844468439076937,\n 'reg_alpha': 0.08086875774692211, \n 'reg_lambda': 0.9676995833820066,\n 'random_state': Config.state,\n 'early_stopping_round': Config.early_stop,\n 'categorical_feature': cat_features,\n 'verbose': -1,\n 'n_estimators': 5000,\n 'learning_rate': 0.01,\n 'max_bin': 8000,\n }),\n False],\n 'HGB3': [HistGradientBoostingRegressor(**{'max_depth': 14,\n 'learning_rate': 0.012284533985518386,\n 'loss': 'squared_error',\n 'l2_regularization': 4.037780478249763,\n 'min_samples_leaf': 40,\n 'max_leaf_nodes': 40,\n 'max_iter': 5000,\n 'random_state': Config.state,\n 'early_stopping': Config.early_stop\n }),\n False],\n 'HGB4': [HistGradientBoostingRegressor(**{'max_depth': 12,\n 'loss': 'squared_error',\n 'l2_regularization': 5.24776999351151e-09,\n 'min_samples_leaf': 26,\n 'max_leaf_nodes': 39,\n 'learning_rate': 0.005,\n 'max_iter': 5000,\n 'random_state': Config.state,\n 'early_stopping': Config.early_stop\n }),\n False],\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:11:18.018796Z","iopub.execute_input":"2024-12-26T11:11:18.019159Z","iopub.status.idle":"2024-12-26T11:11:18.065919Z","shell.execute_reply.started":"2024-12-26T11:11:18.019125Z","shell.execute_reply":"2024-12-26T11:11:18.064817Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Model(Config):\n \n def __init__(self, X, X_enc, y, test, test_enc, models):\n self.y = y\n self.models = models\n self.scores = pd.DataFrame(columns=['Score'])\n self.OOF_preds = pd.DataFrame()\n self.TEST_preds = pd.DataFrame()\n def train(self):\n \n folds = KFold(n_splits=self.n_splits, shuffle=True, random_state=self.state)\n \n for model_name, [model, training] in tqdm(self.models.items()):\n \n if training:\n print('='*20)\n print(model_name)\n if any(model in model_name for model in ['LGBM', 'CAT', 'XGB', 'YDF']):\n self.X = X\n self.test = test\n \n else:\n self.X = X_enc\n self.test = test_enc\n \n if 'NN' in model_name:\n for n_fold, (train_id, valid_id) in enumerate(folds.split(self.X, self.y)):\n\n X_train_cats = self.X.loc[train_id, cat_features]\n X_train_nums = self.X.loc[train_id, t.num_features]\n y_train = self.y.loc[train_id].values\n\n X_val_cats = self.X.loc[valid_id, cat_features]\n X_val_nums = self.X.loc[valid_id, t.num_features]\n y_val = self.y.loc[valid_id]\n\n X_test_cats = self.test[cat_features]\n X_test_nums = self.test[t.num_features]\n \n oof_preds = pd.DataFrame(columns=[model_name], index=X_val_cats.index)\n test_preds = pd.DataFrame(columns=[model_name], index=test.index)\n print(f'Fold {n_fold+1}')\n \n model = build_model() \n keras.utils.set_random_seed(self.state)\n optimizer = keras.optimizers.Adam(learning_rate=1e-3)\n model.compile(optimizer=optimizer, loss='mean_squared_error')\n model.fit([X_train_cats,X_train_nums], y_train, \n validation_data=([X_val_cats, X_val_nums], y_val),\n epochs=20,\n batch_size=1000,\n callbacks=[keras.callbacks.ReduceLROnPlateau(patience=1),\n keras.callbacks.EarlyStopping(patience=3)\n ])\n \n y_pred_val = model.predict([X_val_cats, X_val_nums])\n test_pred = model.predict([X_test_cats, X_test_nums])\n \n score = mean_squared_error(y_val, y_pred_val, squared=False)\n print(score)\n self.scores.loc[f'{model_name}', f'Fold {n_fold+1}'] = score\n \n oof_preds[model_name] = y_pred_val\n test_preds[model_name] = test_pred\n\n self.OOF_preds = pd.concat([self.OOF_preds, oof_preds], axis = 0, ignore_index = False)\n TEST_preds = pd.concat([self.TEST_preds, test_preds], axis = 0, ignore_index = False)\n \n else:\n for n_fold, (train_id, valid_id) in enumerate(folds.split(self.X, self.y)):\n X_train, y_train = self.X.iloc[train_id], self.y.iloc[train_id]\n X_val, y_val = self.X.iloc[valid_id], self.y.iloc[valid_id]\n\n oof_preds = pd.DataFrame(columns=[model_name], index=X_val.index)\n test_preds = pd.DataFrame(columns=[model_name], index=test.index)\n print(f'Fold {n_fold+1}')\n\n if \"XGB\" in model_name:\n model.fit(X_train, y_train, \n eval_set = [(X_val, y_val)], \n verbose = False\n )\n\n elif \"CAT\" in model_name:\n model.fit(X_train, y_train, \n eval_set = [(X_val, y_val)],\n verbose=False\n ) \n\n elif \"LGBM\" in model_name:\n model.fit(X_train, y_train, \n eval_set = [(X_val, y_val)], \n callbacks = [log_evaluation(0),\n early_stopping(self.early_stop, verbose = False)\n ]) \n\n else:\n model.fit(X_train, y_train)\n\n y_pred_val = model.predict(X_val)\n test_pred = model.predict(self.test)\n \n score = mean_squared_error(y_val, y_pred_val, squared=False)\n print(score)\n self.scores.loc[f'{model_name}', f'Fold {n_fold+1}'] = score\n\n oof_preds[model_name] = y_pred_val\n test_preds[model_name] = test_pred\n self.OOF_preds = pd.concat([self.OOF_preds, oof_preds], axis = 0, ignore_index = False)\n self.TEST_preds = pd.concat([self.TEST_preds, test_preds], axis = 0, ignore_index = False)\n\n self.OOF_preds = self.OOF_preds.groupby(level=0).mean()\n self.TEST_preds = self.TEST_preds.groupby(level=0).mean()\n\n self.OOF_preds[f'{model_name}'].to_csv(f'{model_name}_oof.csv', index=False)\n self.TEST_preds[f'{model_name}'].to_csv(f'{model_name}_test.csv', index=False)\n \n else:\n self.OOF_preds[f'{model_name}'] = pd.read_csv(f'/kaggle/input/scale-models/{model_name}_oof.csv')\n self.TEST_preds[f'{model_name}'] = pd.read_csv(f'/kaggle/input/scale-models/{model_name}_test.csv')\n \n for n_fold, (train_id, valid_id) in enumerate(folds.split(self.OOF_preds[f'{model_name}'], self.y)):\n y_pred_val, y_val = self.OOF_preds[f'{model_name}'].iloc[valid_id], self.y.iloc[valid_id]\n self.scores.loc[f'{model_name}', f'Fold {n_fold+1}'] = mean_squared_error(y_val, y_pred_val, squared=False)\n \n self.scores.loc[f'{model_name}', 'Score'] = self.scores.loc[f'{model_name}'][1:].mean()\n self.scores.loc['Ensemble'], self.OOF_preds[\"Ensemble\"], self.TEST_preds[\"Ensemble\"] = self.ensemble(self.OOF_preds, self.y, self.TEST_preds)\n self.scores = self.scores.sort_values('Score')\n\n self.result()\n\n return self.TEST_preds\n \n def ensemble(self, X, y, test):\n scores = []\n oof_pred = np.zeros(X.shape[0])\n test_pred = np.zeros(test.shape[0])\n model = LinearRegression()\n skf = KFold(n_splits=self.n_splits, random_state=self.state, shuffle=True)\n\n for fold_idx, (train_idx, val_idx) in enumerate(skf.split(X, y)):\n X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n model.fit(X_train, y_train)\n\n y_pred_probs = model.predict(X_val)\n oof_pred[val_idx] = y_pred_probs\n test_pred += model.predict(test) / self.n_splits\n \n score = mean_squared_error(y_val, y_pred_probs, squared=False)\n scores.append(score)\n \n return np.mean(scores), oof_pred, test_pred\n \n def result(self):\n \n plt.figure(figsize=(18, 6))\n hbars = plt.bar(self.scores.index, self.scores.Score, color='#3cb371', width=0.7)\n plt.bar_label(hbars, fmt='%.4f')\n plt.ylim(1.03,1.034)\n plt.xlabel('Models')\n plt.ylabel('Score') \n plt.show()\n\n y = np.expm1(self.y).sort_index()\n self.OOF_preds['Ensemble'] = np.expm1(self.OOF_preds['Ensemble']).sort_index()\n fig, axes = plt.subplots(1, 2, figsize=(14, 6))\n axes[0].scatter(y, self.OOF_preds['Ensemble'], alpha=0.5, s=15, edgecolors='#3cb371')\n axes[0].plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=2)\n axes[0].set_xlabel('Actual')\n axes[0].set_ylabel('Predicted')\n axes[0].set_title('Actual vs. Predicted')\n\n axes[1].scatter(self.OOF_preds['Ensemble'], y - self.OOF_preds['Ensemble'], alpha=0.5, s=15, edgecolors='#3cb371')\n axes[1].axhline(y=0, color='black', linestyle='--', lw=2)\n axes[1].set_xlabel('Predicted Values')\n axes[1].set_ylabel('Residuals')\n axes[1].set_title('Residual Plot')\n\n plt.tight_layout()\n plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:27:23.502122Z","iopub.execute_input":"2024-12-26T11:27:23.502544Z","iopub.status.idle":"2024-12-26T11:27:23.535535Z","shell.execute_reply.started":"2024-12-26T11:27:23.502509Z","shell.execute_reply":"2024-12-26T11:27:23.53423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = Model(X, X_enc, y, test, test_enc, models)\nTEST_preds = model.train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:27:26.887172Z","iopub.execute_input":"2024-12-26T11:27:26.887715Z","iopub.status.idle":"2024-12-26T11:28:08.177034Z","shell.execute_reply.started":"2024-12-26T11:27:26.887673Z","shell.execute_reply":"2024-12-26T11:28:08.172758Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#

Submission

","metadata":{}},{"cell_type":"code","source":"submission = Config.submission\nsubmission[Config.target] = np.expm1(TEST_preds['Ensemble'].values)\nsubmission.to_csv(\"submission.csv\", index=False)\n\ndisplay(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:36:55.620257Z","iopub.execute_input":"2024-12-26T11:36:55.620738Z","iopub.status.idle":"2024-12-26T11:36:57.363288Z","shell.execute_reply.started":"2024-12-26T11:36:55.620702Z","shell.execute_reply":"2024-12-26T11:36:57.362024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsubmission[Config.target].hist(color='#3cb371', bins=50)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-26T11:29:21.276847Z","iopub.execute_input":"2024-12-26T11:29:21.277298Z","iopub.status.idle":"2024-12-26T11:29:21.581633Z","shell.execute_reply.started":"2024-12-26T11:29:21.277262Z","shell.execute_reply":"2024-12-26T11:29:21.580451Z"}},"outputs":[],"execution_count":null}]}