{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch, numpy as np, pandas as pd, matplotlib.pyplot as plt\nimport xgboost as xgb, lightgbm as lgb\nfrom pathlib import Path\nfrom fastai.tabular.all import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:36.571694Z","iopub.execute_input":"2024-12-31T22:32:36.571892Z","iopub.status.idle":"2024-12-31T22:32:43.348702Z","shell.execute_reply.started":"2024-12-31T22:32:36.571874Z","shell.execute_reply":"2024-12-31T22:32:43.347979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:43.349478Z","iopub.execute_input":"2024-12-31T22:32:43.349909Z","iopub.status.idle":"2024-12-31T22:32:43.353412Z","shell.execute_reply.started":"2024-12-31T22:32:43.349888Z","shell.execute_reply":"2024-12-31T22:32:43.352619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = Path('/kaggle/input/playground-series-s4e12')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:43.354403Z","iopub.execute_input":"2024-12-31T22:32:43.354713Z","iopub.status.idle":"2024-12-31T22:32:43.367961Z","shell.execute_reply.started":"2024-12-31T22:32:43.354684Z","shell.execute_reply":"2024-12-31T22:32:43.367321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:53.604594Z","iopub.execute_input":"2024-12-31T22:32:53.604935Z","iopub.status.idle":"2024-12-31T22:32:58.672351Z","shell.execute_reply.started":"2024-12-31T22:32:53.604909Z","shell.execute_reply":"2024-12-31T22:32:58.671432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_features(df):\n    df.columns = df.columns.str.replace(' ', '_')\n    df['Policy_Start_Date'] = pd.to_datetime(df.Policy_Start_Date)\n    date = df.Policy_Start_Date.dt\n    df.drop(['Policy_Start_Date'], axis=1)\n    \n    df['Year'] = date.year.astype(float)\n    df['Month'] = date.month.astype(float)\n    df['Day'] = date.day.astype(float)\n    df['Week']  = date.isocalendar().week.astype(float)\n    df['Weekday'] = date.weekday.astype(float)\n    df['Epoch'] = df.Policy_Start_Date.astype(np.int64) / 10**9\n    \n    df['Year_sin'] = np.sin(2 * np.pi * df.Year)\n    df['Year_cos'] = np.cos(2 * np.pi * df.Year)\n    df['Month_sin'] = np.sin(2 * np.pi * df.Month / 12) \n    df['Month_cos'] = np.cos(2 * np.pi * df.Month / 12)\n    \n    df['Income_Age_Ratio'] = df.Annual_Income / df.Age\n    df['Log_Annual_Income'] = np.log1p(df.Annual_Income)\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:08:18.817538Z","iopub.execute_input":"2024-12-31T23:08:18.817862Z","iopub.status.idle":"2024-12-31T23:08:18.82466Z","shell.execute_reply.started":"2024-12-31T23:08:18.81784Z","shell.execute_reply":"2024-12-31T23:08:18.823516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fill_null(df):\n    for col in df:\n        if df[col].isna().sum() > 0: df[col + '_na'] = df[col].isna().astype(float)\n    df = df.fillna(df.mode().iloc[0])\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:33:45.528976Z","iopub.execute_input":"2024-12-31T22:33:45.529263Z","iopub.status.idle":"2024-12-31T22:33:45.534074Z","shell.execute_reply.started":"2024-12-31T22:33:45.529241Z","shell.execute_reply":"2024-12-31T22:33:45.53288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def categorize_cats(df, encode=False):\n    conts,cats = cont_cat_split(df)\n    for col in cats:\n        if encode:\n            df[col] = pd.Categorical(df[col])\n            df[col] = df[col].cat.codes\n        else: df[col] = df[col].astype('category')\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:43.394981Z","iopub.execute_input":"2024-12-31T22:32:43.395238Z","iopub.status.idle":"2024-12-31T22:32:43.404877Z","shell.execute_reply.started":"2024-12-31T22:32:43.395218Z","shell.execute_reply":"2024-12-31T22:32:43.404012Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def split_indeps_deps(df, log=False):\n    indeps = df\n    deps = None\n    if 'id' in df.columns: indeps = df.drop(['id'], axis=1)\n    if 'Premium_Amount' in df.columns:\n        indeps = indeps.drop('Premium_Amount', axis=1)\n        deps = df.Premium_Amount\n        if log: deps = np.log1p(deps)\n    return indeps,deps","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:43.405731Z","iopub.execute_input":"2024-12-31T22:32:43.405954Z","iopub.status.idle":"2024-12-31T22:32:43.415213Z","shell.execute_reply.started":"2024-12-31T22:32:43.405936Z","shell.execute_reply":"2024-12-31T22:32:43.414617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def proc_data(df, encode=False, log=False):\n    df = extract_features(df)\n    df = fill_null(df)\n    df = categorize_cats(df, encode=encode)\n    indeps,deps = split_indeps_deps(df, log=log)\n    return indeps,deps","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:32:43.415983Z","iopub.execute_input":"2024-12-31T22:32:43.416246Z","iopub.status.idle":"2024-12-31T22:32:43.427093Z","shell.execute_reply.started":"2024-12-31T22:32:43.416221Z","shell.execute_reply":"2024-12-31T22:32:43.426321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"indeps,deps = proc_data(df, encode=True)\ndeps_log = np.log1p(deps)\nindeps.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:08:21.317627Z","iopub.execute_input":"2024-12-31T23:08:21.317912Z","iopub.status.idle":"2024-12-31T23:08:28.707241Z","shell.execute_reply.started":"2024-12-31T23:08:21.317892Z","shell.execute_reply":"2024-12-31T23:08:28.7062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.data.transforms import RandomSplitter\ntrain_split,valid_split = RandomSplitter(valid_pct=0.2)(df)\ntrain_indeps,train_deps,train_deps_log = indeps.iloc[train_split],deps.iloc[train_split],deps_log.iloc[train_split]\nvalid_indeps,valid_deps,valid_deps_log = indeps.iloc[valid_split],deps.iloc[valid_split],deps_log.iloc[valid_split]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:08:28.708722Z","iopub.execute_input":"2024-12-31T23:08:28.709044Z","iopub.status.idle":"2024-12-31T23:08:29.510346Z","shell.execute_reply.started":"2024-12-31T23:08:28.70902Z","shell.execute_reply":"2024-12-31T23:08:29.509398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error\ndef rmsle(preds, targs):\n    preds = np.maximum(0, preds)\n    targs = np.maximum(0, targs)\n    return np.sqrt(mean_squared_log_error(targs, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:33:56.131672Z","iopub.execute_input":"2024-12-31T22:33:56.131974Z","iopub.status.idle":"2024-12-31T22:33:56.136282Z","shell.execute_reply.started":"2024-12-31T22:33:56.131947Z","shell.execute_reply":"2024-12-31T22:33:56.135411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import skopt\nfrom sklearn.model_selection import ShuffleSplit\n\ndef bayesian_optimization(model, search_spaces=None, n_iter=10, verbose=1):\n    start = time.time()\n    if search_spaces is None:\n        search_spaces = {\n            'n_estimators': (10, 200),      \n            'max_depth': (3, 10),\n            'learning_rate': (0.01, 0.3, 'log-uniform'),\n            'reg_alpha': (0.001, 0.2, 'log-uniform'),  \n            'reg_lambda': (0.1, 100, 'log-uniform')     \n        }\n    \n    splitter = ShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n    \n    search = skopt.BayesSearchCV(\n        estimator=model,\n        search_spaces=search_spaces,\n        n_iter=n_iter,\n        verbose=verbose,\n        scoring='neg_root_mean_squared_error',\n        random_state=42,\n        n_jobs=-1,\n        cv=splitter\n    )\n    \n    search.fit(indeps, deps_log)\n    end = time.time()\n    print(f'elapsed time: {(end - start):.2f}s')\n    return search","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:41:27.473625Z","iopub.execute_input":"2024-12-31T22:41:27.473927Z","iopub.status.idle":"2024-12-31T22:41:27.48004Z","shell.execute_reply.started":"2024-12-31T22:41:27.473905Z","shell.execute_reply":"2024-12-31T22:41:27.479082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_model = xgb.XGBRegressor(random_state=42)\nxgb_search = bayesian_optimization(xgb_model, n_iter=50)\nprint(\"Best params: \", xgb_search.best_params_)\nprint(\"Best RMSLE: \", -xgb_search.best_score_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:41:37.923288Z","iopub.execute_input":"2024-12-31T22:41:37.923631Z","iopub.status.idle":"2024-12-31T23:01:20.269683Z","shell.execute_reply.started":"2024-12-31T22:41:37.923602Z","shell.execute_reply":"2024-12-31T23:01:20.268557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dict(xgb_search.best_params_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:23:56.208752Z","iopub.execute_input":"2024-12-31T23:23:56.209056Z","iopub.status.idle":"2024-12-31T23:23:56.214586Z","shell.execute_reply.started":"2024-12-31T23:23:56.209031Z","shell.execute_reply":"2024-12-31T23:23:56.213661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(model, encode=False):\n    test_df = pd.read_csv(path/'test.csv')\n    submit_df = pd.read_csv(path/'sample_submission.csv')\n    test_indeps,_ = proc_data(test_df, encode=True)\n    preds = np.expm1(model.predict(test_indeps))\n    submit_df['Premium Amount'] = preds\n    return submit_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T22:34:45.937312Z","iopub.execute_input":"2024-12-31T22:34:45.937742Z","iopub.status.idle":"2024-12-31T22:34:45.944472Z","shell.execute_reply.started":"2024-12-31T22:34:45.937707Z","shell.execute_reply":"2024-12-31T22:34:45.943073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_params = xgb_search.best_params_\nxgb_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:01:23.047588Z","iopub.execute_input":"2024-12-31T23:01:23.047999Z","iopub.status.idle":"2024-12-31T23:01:23.054734Z","shell.execute_reply.started":"2024-12-31T23:01:23.047966Z","shell.execute_reply":"2024-12-31T23:01:23.053821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_model = xgb.XGBRegressor(**xgb_params)\nxgb_model.fit(indeps, deps_log)\n#submit_df = submit(xgb_model, encode=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T20:18:02.864544Z","iopub.execute_input":"2024-12-31T20:18:02.864842Z","iopub.status.idle":"2024-12-31T20:18:21.291238Z","shell.execute_reply.started":"2024-12-31T20:18:02.86482Z","shell.execute_reply":"2024-12-31T20:18:21.290537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit_df.to_csv('insurance_v10.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T19:23:53.74546Z","iopub.execute_input":"2024-12-31T19:23:53.745785Z","iopub.status.idle":"2024-12-31T19:23:54.677972Z","shell.execute_reply.started":"2024-12-31T19:23:53.74576Z","shell.execute_reply":"2024-12-31T19:23:54.677311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_model = lgb.LGBMRegressor(random_state=42, verbose=-1)\nlgb_search = bayesian_optimization(lgb_model, n_iter=50)\nprint(\"Best params: \", lgb_search.best_params_)\nprint(\"Best RMSLE: \", -lgb_search.best_score_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:08:45.828001Z","iopub.execute_input":"2024-12-31T23:08:45.828333Z","iopub.status.idle":"2024-12-31T23:21:18.928134Z","shell.execute_reply.started":"2024-12-31T23:08:45.828304Z","shell.execute_reply":"2024-12-31T23:21:18.927313Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dict(lgb_search.best_params_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:24:35.722834Z","iopub.execute_input":"2024-12-31T23:24:35.72313Z","iopub.status.idle":"2024-12-31T23:24:35.729675Z","shell.execute_reply.started":"2024-12-31T23:24:35.723109Z","shell.execute_reply":"2024-12-31T23:24:35.728661Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_model = lgb.LGBMRegressor(**lgb_search.best_params_, verbose=-1)\nlgb_model.fit(indeps, deps_log)\nsubmit_df = submit(lgb_model, encode=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit_df.to_csv('insurance_v11.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T19:59:47.498016Z","iopub.execute_input":"2024-12-31T19:59:47.498417Z","iopub.status.idle":"2024-12-31T19:59:48.819372Z","shell.execute_reply.started":"2024-12-31T19:59:47.498386Z","shell.execute_reply":"2024-12-31T19:59:48.818393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def test_drop_cols(model, arch, params):\n    params = params.copy()\n    if arch == 'XGB': importances = model.get_booster().get_score()\n    elif arch == 'LGB': importances = dict(zip(model.feature_name_, model.feature_importances_))\n    losses = []\n    for i in range(len(importances)):\n        remove_feature = min(importances, key=importances.get)\n        importances.pop(remove_feature)\n        included_features = list(importances.keys())\n        if arch == 'XGB': m = xgb.XGBRegressor(**params)\n        elif arch == 'LGB': m = lgb.LGBMRegressor(**params, verbose=-1)\n        m.fit(train_indeps[included_features], train_deps_log)\n        preds = np.expm1(m.predict(valid_indeps[included_features]))\n        losses.append(rmsle(preds, valid_deps))\n        print(f'Removed {remove_feature:<30} Loss: {losses[-1]}')\n    plt.plot(losses)\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:25:29.791415Z","iopub.execute_input":"2024-12-31T23:25:29.79176Z","iopub.status.idle":"2024-12-31T23:25:29.798125Z","shell.execute_reply.started":"2024-12-31T23:25:29.791733Z","shell.execute_reply":"2024-12-31T23:25:29.797316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_params = {\n    'learning_rate': 0.11040568023673476,\n    'max_depth': 10,\n    'n_estimators': 100,\n    'reg_alpha': 0.03833403913374451,\n    'reg_lambda': 0.508018478116339\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:25:44.062304Z","iopub.execute_input":"2024-12-31T23:25:44.062707Z","iopub.status.idle":"2024-12-31T23:25:44.067394Z","shell.execute_reply.started":"2024-12-31T23:25:44.062671Z","shell.execute_reply":"2024-12-31T23:25:44.066298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_drop_cols(xgb_model, arch='XGB', params=lgb_params)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb.plot_importance(xgb_model)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:31:58.740893Z","iopub.execute_input":"2024-12-31T23:31:58.741294Z","iopub.status.idle":"2024-12-31T23:31:59.299377Z","shell.execute_reply.started":"2024-12-31T23:31:58.741256Z","shell.execute_reply":"2024-12-31T23:31:59.298525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb.plot_importance(lgb_model)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T23:32:01.12868Z","iopub.execute_input":"2024-12-31T23:32:01.129107Z","iopub.status.idle":"2024-12-31T23:32:01.643184Z","shell.execute_reply.started":"2024-12-31T23:32:01.129072Z","shell.execute_reply":"2024-12-31T23:32:01.642292Z"}},"outputs":[],"execution_count":null}]}