{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## S4E12 Ensemble w/StackingRegressor","metadata":{}},{"cell_type":"markdown","source":"\nStackingRegressor is a specific implementation of the stacking ensemble technique for regression problems in machine learning. It is available in the scikit-learn library, a popular machine learning library in Python. StackingRegressor allows you to create a stacked ensemble by combining multiple regression models to improve predictive performance.","metadata":{}},{"cell_type":"markdown","source":"In each model, the parameter values ​​optimized using optuna were used.","metadata":{}},{"cell_type":"code","source":"# Misc\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nimport os\nfrom copy import deepcopy\nfrom functools import partial\nimport gc\nimport warnings\n\n# Import sklearn classes for model selection, cross validation, and performance evaluation\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, KFold\nfrom sklearn.metrics import accuracy_score, mean_squared_error, roc_auc_score, log_loss, f1_score, precision_score, recall_score\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, LabelEncoder, scale, PowerTransformer\nfrom category_encoders import OneHotEncoder, OrdinalEncoder, CountEncoder\nfrom imblearn.under_sampling import RandomUnderSampler\n\n# Import libraries for gradient boosting\nimport xgboost as xgb\nimport lightgbm as lgb\nimport catboost as catb\nfrom imblearn.ensemble import BalancedRandomForestClassifier\nfrom sklearn.impute import KNNImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.svm import NuSVC, SVC\nfrom sklearn.gaussian_process.kernels import RBF\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.ensemble import RandomForestRegressor, StackingRegressor\n","metadata":{"execution":{"iopub.status.busy":"2023-11-21T14:22:42.613508Z","iopub.execute_input":"2023-11-21T14:22:42.613792Z","iopub.status.idle":"2023-11-21T14:22:42.625359Z","shell.execute_reply.started":"2023-11-21T14:22:42.613758Z","shell.execute_reply":"2023-11-21T14:22:42.624574Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest0 = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-11-21T14:22:42.644967Z","iopub.execute_input":"2023-11-21T14:22:42.645202Z","iopub.status.idle":"2023-11-21T14:22:42.726485Z","shell.execute_reply.started":"2023-11-21T14:22:42.645172Z","shell.execute_reply":"2023-11-21T14:22:42.725735Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\ndef labelencoder(df_train, df_test):\n    combined_df = pd.concat([df_train, df_test])\n    for c in combined_df.columns:\n        if combined_df[c].dtype == 'object':\n            combined_df[c] = combined_df[c].fillna('N')\n            lbl = LabelEncoder()\n            lbl.fit(list(combined_df[c].values))\n            combined_df[c] = lbl.transform(combined_df[c].values)\n    return combined_df.iloc[:len(df_train)], combined_df.iloc[len(df_train):]\n\ntrain,test=labelencoder(train0,test0)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = train['Premium Amount']\ndata = train.drop(['Premium Amount'],axis=1)\n\ncolumns=data.columns.to_list()\nprint(columns)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import HistGradientBoostingRegressor\nhist_param={'learning_rate': 0.0893325347377967, 'max_iter': 168, 'max_leaf_nodes': 42, 'max_bins': 126, 'min_samples_leaf': 46, 'l2_regularization': 1.1467438645272217, 'early_stopping': True, 'validation_fraction': 0.2, 'random_state': 42, 'verbose': 1}\nhist_model = HistGradientBoostingRegressor(**hist_param)  \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import lightgbm as lgb\nlgb_param={'max_depth': 10, 'learning_rate': 0.04744795667113469, 'lambda_l1': 0.0005822312174849303, 'lambda_l2': 2.465950596637246e-07, 'bagging_freq': 2, 'bagging_fraction': 0.7850283770055242, 'feature_fraction': 0.9672449051424565, 'objective': 'rmse', 'num_leaves': 100, 'boosting': 'gbdt', 'verbosity': -1}\nlgb_model = lgb.LGBMRegressor(**lgb_param) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Splitting the data\nX_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.2, random_state=42)\n\n# Base learners (first layer models)\nbase_learners = [\n    ('hist', hist_model),\n    ('lgb', lgb_model)\n]\n\n# Meta-model (linear regression)\nmeta_model = LinearRegression()\n\n# Constructing the StackingRegressor\nstacked_model = StackingRegressor(\n    estimators=base_learners,  # First layer models\n    final_estimator=meta_model  # Meta-model\n)\n\n# Training the model\nstacked_model.fit(X_train, np.log1p(y_train))\n\n# Predictions on test data\n# y_pred = stacked_model.predict(X_test)\ny_pred = np.expm1(stacked_model.predict(X_test))\n\nfrom sklearn.metrics import mean_squared_log_error\nrmsle = np.sqrt(mean_squared_log_error(y_test, y_pred))   \nprint(f'RMSLE: {rmsle}')","metadata":{"execution":{"iopub.status.busy":"2023-11-21T14:22:42.728023Z","iopub.execute_input":"2023-11-21T14:22:42.728244Z","iopub.status.idle":"2023-11-21T14:22:42.73408Z","shell.execute_reply.started":"2023-11-21T14:22:42.728217Z","shell.execute_reply":"2023-11-21T14:22:42.733354Z"},"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model=stacked_model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = np.zeros((test.shape[0]))\nkf = KFold(n_splits=5, random_state=48, shuffle=True)\n\nfor trn_idx, test_idx in kf.split(train[columns], target):\n    X_tr, X_val = train[columns].iloc[trn_idx], train[columns].iloc[test_idx]\n    y_tr, y_val = target.iloc[trn_idx], target.iloc[test_idx]\n\n    model.fit(X_tr, np.log1p(y_tr))  \n\n    preds_fold = np.expm1(model.predict(test[columns])) \n    preds_fold = np.nan_to_num(preds_fold, nan=0, posinf=0, neginf=0)  \n    preds_fold = np.clip(preds_fold, 0, 1e10)\n    preds += preds_fold / kf.n_splits  \n\n    y_val_pred = np.expm1(model.predict(X_val))  \n    y_val_pred = np.nan_to_num(y_val_pred, nan=0, posinf=0, neginf=0)  \n    y_val_pred = np.clip(y_val_pred, 0, 1e10)\n    \n    y_val_actual = np.expm1(y_val)  \n    y_val_actual = np.nan_to_num(y_val_actual, nan=0, posinf=0, neginf=0) \n    \n    try:\n        rmsle = np.sqrt(mean_squared_log_error(y_val_actual, y_val_pred)) \n        print(f'RMSLE for fold: {rmsle}')\n        \n    except ValueError as e:\n        print(f\"Error in fold: {e}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"subm = pd.read_csv(\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nsubm['Premium Amount'] = preds.astype(int)\nsubm.to_csv('submission.csv',index=False)\nsubm","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}