{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libs","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport os\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\nfrom catboost import CatBoostRegressor\nimport xgboost as xgb\nfrom xgboost import XGBRegressor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:39:41.738785Z","iopub.execute_input":"2024-12-28T11:39:41.739151Z","iopub.status.idle":"2024-12-28T11:39:41.743647Z","shell.execute_reply.started":"2024-12-28T11:39:41.739114Z","shell.execute_reply":"2024-12-28T11:39:41.742807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"input = '/kaggle/input/playground-series-s4e12'\ntrain = pd.read_csv(os.path.join(input, 'train.csv'))\ntest = pd.read_csv(os.path.join(input, 'test.csv'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:39:41.744701Z","iopub.execute_input":"2024-12-28T11:39:41.744948Z","iopub.status.idle":"2024-12-28T11:39:46.971314Z","shell.execute_reply.started":"2024-12-28T11:39:41.744918Z","shell.execute_reply":"2024-12-28T11:39:46.970658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:39:46.972621Z","iopub.execute_input":"2024-12-28T11:39:46.972823Z","iopub.status.idle":"2024-12-28T11:39:46.993237Z","shell.execute_reply.started":"2024-12-28T11:39:46.972805Z","shell.execute_reply":"2024-12-28T11:39:46.992542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Features engineering","metadata":{}},{"cell_type":"code","source":"# Replace Policy Start Date\nfor i in [train, test]:\n    i['Policy Start Date'] = pd.to_datetime(i['Policy Start Date'])\n    i['Year'] = i['Policy Start Date'].dt.year\n    i['Day'] = i['Policy Start Date'].dt.day\n    i['Month'] = i['Policy Start Date'].dt.month\n    i['Month_name'] = i['Policy Start Date'].dt.month_name()\n    i['Day_of_week'] = i['Policy Start Date'].dt.day_name()\n    i['Week'] = i['Policy Start Date'].dt.isocalendar().week\n    \n    i['Year_sin'] = np.sin(2 * np.pi * i['Year'])\n    i['Year_cos'] = np.cos(2 * np.pi * i['Year'])\n    i['Month_sin'] = np.sin(2 * np.pi * i['Month'] / 12) \n    i['Month_cos'] = np.cos(2 * np.pi * i['Month'] / 12)\n    i['Day_sin'] = np.sin(2 * np.pi * i['Day'] / 31)  \n    i['Day_cos'] = np.cos(2 * np.pi * i['Day'] / 31)\n    i['Group']=(i['Year']-2020)*48+i['Month']*4+i['Day']//7\n    \n    i.drop('Policy Start Date', axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:17:14.934515Z","iopub.execute_input":"2024-12-31T11:17:14.934873Z","iopub.status.idle":"2024-12-31T11:17:14.947065Z","shell.execute_reply.started":"2024-12-31T11:17:14.934843Z","shell.execute_reply":"2024-12-31T11:17:14.945879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = 'Premium Amount'\nnumeric_cols = train.select_dtypes(include=[np.number]).columns.drop(target, errors='ignore').tolist()\ncat_cols = train.select_dtypes(exclude=[np.number, 'datetime64[ns]']).columns.tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:39:49.461037Z","iopub.execute_input":"2024-12-28T11:39:49.46129Z","iopub.status.idle":"2024-12-28T11:39:50.201172Z","shell.execute_reply.started":"2024-12-28T11:39:49.461269Z","shell.execute_reply":"2024-12-28T11:39:50.200251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill none\nfor col in numeric_cols:\n    median_value = train[col].median()\n    train[f\"is_{col}_na\"] = train[col].isna().astype(int)\n    train = train.loc[:, (train != 0).any()]\n    train[col] = train[col].fillna(median_value)\n    \n    test[f\"is_{col}_na\"] = test[col].isna().astype(int)\n    test = test.loc[:, (test != 0).any()]\n    test[col] = test[col].fillna(median_value)\n\n\nfor col in cat_cols:\n    train[col] = train[col].astype(str).fillna(\"Unknown\")\n    test[col] = test[col].astype(str).fillna(\"Unknown\")\n\n\nnumeric_cols = train.select_dtypes(include=[np.number]).columns.drop(target, errors='ignore').tolist()\ncat_cols = train.select_dtypes(exclude=[np.number]).columns.drop(target, errors='ignore').tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:17:49.006227Z","iopub.execute_input":"2024-12-31T11:17:49.006564Z","iopub.status.idle":"2024-12-31T11:17:49.018402Z","shell.execute_reply.started":"2024-12-31T11:17:49.006533Z","shell.execute_reply":"2024-12-31T11:17:49.017168Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train test split","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX = train.drop(columns=target)\ny = train[target]\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.2, random_state=42\n)\n\n# Identify categorical feature indices for CatBoost\ncat_features_indices = [X_train.columns.get_loc(c) for c in cat_cols if c in X_train.columns]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:40:30.647756Z","iopub.execute_input":"2024-12-28T11:40:30.647988Z","iopub.status.idle":"2024-12-28T11:40:32.041234Z","shell.execute_reply.started":"2024-12-28T11:40:30.647968Z","shell.execute_reply":"2024-12-28T11:40:32.040599Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Catboost","metadata":{}},{"cell_type":"code","source":"params = {\n    'iterations': 1100,\n    'learning_rate': 0.05,\n    'depth': 8,\n    'loss_function': 'RMSE',\n    'random_seed': 42,\n    'task_type': 'GPU',\n    'verbose': 0\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:40:32.042468Z","iopub.execute_input":"2024-12-28T11:40:32.042717Z","iopub.status.idle":"2024-12-28T11:40:32.046311Z","shell.execute_reply.started":"2024-12-28T11:40:32.042684Z","shell.execute_reply":"2024-12-28T11:40:32.045514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 15\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\nfold_scores = []\nmodels = []\n\nfor i, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n    X_train_fold, X_val_fold = X_train.iloc[train_idx], X_train.iloc[val_idx]\n    y_train_fold, y_val_fold = y_train.iloc[train_idx], y_train.iloc[val_idx]\n\n    model = CatBoostRegressor(**params, cat_features=cat_features_indices)\n    \n    model.fit(X_train_fold, np.log1p(y_train_fold),\n              eval_set=(X_val_fold, np.log1p(y_val_fold)),\n              early_stopping_rounds=50,\n              use_best_model=True)\n    \n    y_pred_fold = np.expm1(model.predict(X_val_fold))\n    \n    # RMSLE this fold\n    fold_rmsle = np.sqrt(mean_squared_log_error(y_val_fold, y_pred_fold))\n    fold_scores.append(fold_rmsle)\n    models.append(model)\n    print(f\"Fold {i+1} RMSLE: {fold_rmsle}\")\n\nmean_rmsle = np.mean(fold_scores)\nprint(f\"Mean RMSLE: {mean_rmsle:.5f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:40:32.047025Z","iopub.execute_input":"2024-12-28T11:40:32.047207Z","iopub.status.idle":"2024-12-28T11:50:10.877913Z","shell.execute_reply.started":"2024-12-28T11:40:32.047188Z","shell.execute_reply":"2024-12-28T11:50:10.876902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds_list = []\nfor i, model in enumerate(models):\n    preds = model.predict(test)\n    test_preds_list.append(np.expm1(preds))\n\ntest_preds_array = np.column_stack(test_preds_list)\ntest_final_preds = np.mean(test_preds_array, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:53:30.819871Z","iopub.execute_input":"2024-12-28T11:53:30.820162Z","iopub.status.idle":"2024-12-28T11:54:17.277431Z","shell.execute_reply.started":"2024-12-28T11:53:30.820141Z","shell.execute_reply":"2024-12-28T11:54:17.276689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output= pd.DataFrame(test['id'])\ncat_output = test_final_preds\noutput['Premium Amount']= cat_output\noutput.to_csv(\"submission_cat.csv\", index = None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:54:32.004005Z","iopub.execute_input":"2024-12-28T11:54:32.004316Z","iopub.status.idle":"2024-12-28T11:54:33.278047Z","shell.execute_reply.started":"2024-12-28T11:54:32.004292Z","shell.execute_reply":"2024-12-28T11:54:33.277426Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGBoost","metadata":{}},{"cell_type":"code","source":"# One hot encoding cat cols for XGB\nfor idx, dataset in enumerate([train, test]):\n    i_dummy = pd.get_dummies(dataset[cat_cols], drop_first=True).astype(int)\n\n    dataset.drop(columns=[\n        'Gender', 'Marital Status', 'Education Level', 'Occupation',\n        'Location', 'Policy Type', 'Customer Feedback', 'Smoking Status',\n        'Exercise Frequency', 'Property Type', 'Month_name', 'Day_of_week'\n    ], inplace=True)\n\n    dataset = pd.concat([dataset, i_dummy], axis=1)\n\n    if idx == 0:\n        train = dataset\n    else:\n        test = dataset","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T11:20:36.046363Z","iopub.execute_input":"2024-12-31T11:20:36.046694Z","iopub.status.idle":"2024-12-31T11:20:36.058189Z","shell.execute_reply.started":"2024-12-31T11:20:36.046668Z","shell.execute_reply":"2024-12-31T11:20:36.056678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = train.drop(columns=target)\ny = train[target]\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X, y, test_size=0.2, random_state=42\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:55:12.15153Z","iopub.execute_input":"2024-12-28T11:55:12.151816Z","iopub.status.idle":"2024-12-28T11:55:13.188671Z","shell.execute_reply.started":"2024-12-28T11:55:12.151795Z","shell.execute_reply":"2024-12-28T11:55:13.187972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {'n_estimators': 1100,\n          'colsample_bytree': 1.0,\n          'learning_rate': 0.02,\n          'max_depth': 10,\n          'subsample': 0.8,\n          'random_state':42,\n          'eval_metric':'rmsle',\n          'tree_method':'hist',\n          'device':'gpu',\n           'verbose':200}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:55:21.991222Z","iopub.execute_input":"2024-12-28T11:55:21.991553Z","iopub.status.idle":"2024-12-28T11:55:21.99551Z","shell.execute_reply.started":"2024-12-28T11:55:21.991524Z","shell.execute_reply":"2024-12-28T11:55:21.994584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_splits = 14\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\nfold_scores = []\nmodels = []\n\nfor i, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n    X_train_fold, X_val_fold = X_train.iloc[train_idx], X_train.iloc[val_idx]\n    y_train_fold, y_val_fold = y_train.iloc[train_idx], y_train.iloc[val_idx]\n\n    # Initialize XGB\n    model = XGBRegressor(**params)\n    \n    model.fit(X_train_fold, np.log1p(y_train_fold), eval_set=[(X_val_fold, np.log1p(y_val_fold))], early_stopping_rounds=50,verbose=200)\n    \n    \n    y_pred_fold = np.expm1(model.predict(X_val_fold))\n\n    fold_rmsle = np.sqrt(mean_squared_log_error(y_val_fold, y_pred_fold))\n    fold_scores.append(fold_rmsle)\n    models.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:55:25.507669Z","iopub.execute_input":"2024-12-28T11:55:25.50797Z","iopub.status.idle":"2024-12-28T11:57:38.287035Z","shell.execute_reply.started":"2024-12-28T11:55:25.507944Z","shell.execute_reply":"2024-12-28T11:57:38.286345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds_list = []\nfor i, model in enumerate(models):\n    preds = model.predict(X_val)\n    val_preds_list.append(np.expm1(preds))\n\nval_preds_array = np.column_stack(val_preds_list)\nval_final_preds = np.mean(val_preds_array, axis=1)\n\nprint(\"AVG Val Pred RMSLE:\", np.sqrt(mean_squared_log_error(y_val, val_final_preds)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:58:42.935238Z","iopub.execute_input":"2024-12-28T11:58:42.935609Z","iopub.status.idle":"2024-12-28T11:58:46.234065Z","shell.execute_reply.started":"2024-12-28T11:58:42.935557Z","shell.execute_reply":"2024-12-28T11:58:46.23317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_preds_list = []\nfor i, model in enumerate(models):\n    preds = model.predict(test)\n    test_preds_list.append(np.expm1(preds))\n\ntest_preds_array = np.column_stack(test_preds_list)\ntest_final_preds = np.mean(test_preds_array, axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:58:49.63766Z","iopub.execute_input":"2024-12-28T11:58:49.637961Z","iopub.status.idle":"2024-12-28T11:59:10.287595Z","shell.execute_reply.started":"2024-12-28T11:58:49.637934Z","shell.execute_reply":"2024-12-28T11:59:10.286898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output= pd.DataFrame(test['id'])\nxgb_output = test_final_preds\noutput['Premium Amount']= xgb_output\noutput.to_csv(\"submission_xgb.csv\", index = None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:59:22.528215Z","iopub.execute_input":"2024-12-28T11:59:22.528547Z","iopub.status.idle":"2024-12-28T11:59:23.483238Z","shell.execute_reply.started":"2024-12-28T11:59:22.528519Z","shell.execute_reply":"2024-12-28T11:59:23.482596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Combining results","metadata":{}},{"cell_type":"code","source":"submission_cat = pd.read_csv(\"submission_cat.csv\")\nsubmission_xgb = pd.read_csv(\"submission_xgb.csv\")\n\n\nsubmission_avg = submission_cat.copy()\nsubmission_avg['Premium Amount'] = (\n    0.55 * submission_cat['Premium Amount'] +\n    0.45 * submission_xgb['Premium Amount']\n)\n\n\nsubmission_avg.to_csv(\"submission.csv\", index=False)\n\nprint(\"'submission.csv' done!\")\nprint('Happy New Year!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T11:59:28.023671Z","iopub.execute_input":"2024-12-28T11:59:28.023957Z","iopub.status.idle":"2024-12-28T11:59:29.610256Z","shell.execute_reply.started":"2024-12-28T11:59:28.023934Z","shell.execute_reply":"2024-12-28T11:59:29.609563Z"}},"outputs":[],"execution_count":null}]}