{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch, numpy as np, pandas as pd, matplotlib.pyplot as plt\nimport xgboost as xgb, lightgbm as lgb\nfrom pathlib import Path\nfrom fastai.tabular.all import *\nimport sklearn\nprint(F'{sklearn.__version__=}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:32.939529Z","iopub.execute_input":"2024-12-27T18:44:32.939831Z","iopub.status.idle":"2024-12-27T18:44:39.642602Z","shell.execute_reply.started":"2024-12-27T18:44:32.939804Z","shell.execute_reply":"2024-12-27T18:44:39.641864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:39.643675Z","iopub.execute_input":"2024-12-27T18:44:39.644281Z","iopub.status.idle":"2024-12-27T18:44:39.647753Z","shell.execute_reply.started":"2024-12-27T18:44:39.644255Z","shell.execute_reply":"2024-12-27T18:44:39.646827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ndef clear():\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:39.649279Z","iopub.execute_input":"2024-12-27T18:44:39.649563Z","iopub.status.idle":"2024-12-27T18:44:39.662088Z","shell.execute_reply.started":"2024-12-27T18:44:39.649536Z","shell.execute_reply":"2024-12-27T18:44:39.661371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path = Path('/kaggle/input/playground-series-s4e12')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:45.409467Z","iopub.execute_input":"2024-12-27T18:44:45.409743Z","iopub.status.idle":"2024-12-27T18:44:45.413429Z","shell.execute_reply.started":"2024-12-27T18:44:45.409722Z","shell.execute_reply":"2024-12-27T18:44:45.41253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(path/'train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:45.829685Z","iopub.execute_input":"2024-12-27T18:44:45.830074Z","iopub.status.idle":"2024-12-27T18:44:51.057648Z","shell.execute_reply.started":"2024-12-27T18:44:45.830038Z","shell.execute_reply":"2024-12-27T18:44:51.056701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def proc_data(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    df['Month'] = df['Policy Start Date'].dt.month.astype(float)\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Week']  = df['Policy Start Date'].dt.isocalendar().week\n    df['Weekday'] = df['Policy Start Date'].dt.weekday.astype(float)\n    df['Income-Age Ratio'] = df['Annual Income'] / df['Age']\n    for col in df:\n        if(df[col].isna().sum() > 0): df[col + '_na'] = df[col].isna().astype(float)\n    df = df.fillna(df.mode().iloc[0])\n    conts,cats = cont_cat_split(df)\n    for col in cats:\n        df[col] = pd.Categorical(df[col])\n        df[col] = df[col].cat.codes\n    indeps = df.drop(['id', 'Policy Start Date'], axis=1)\n    deps = None\n    if 'Premium Amount' in df.columns:\n        indeps = indeps.drop('Premium Amount', axis=1)\n        deps = df['Premium Amount']\n    return indeps,deps","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:51.059005Z","iopub.execute_input":"2024-12-27T18:44:51.059327Z","iopub.status.idle":"2024-12-27T18:44:51.06584Z","shell.execute_reply.started":"2024-12-27T18:44:51.059298Z","shell.execute_reply":"2024-12-27T18:44:51.064912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"indeps,deps = proc_data(df)\nindeps.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:51.067499Z","iopub.execute_input":"2024-12-27T18:44:51.067835Z","iopub.status.idle":"2024-12-27T18:44:58.248313Z","shell.execute_reply.started":"2024-12-27T18:44:51.067805Z","shell.execute_reply":"2024-12-27T18:44:58.24742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_log_error\ndef rmsle(preds, targs):\n    #preds = preds.cpu().numpy() if preds.is_cuda else preds.numpy()\n    #targs = targs.cpu().numpy() if targs.is_cuda else targs.numpy()\n    preds = np.maximum(0, preds)\n    targs = np.maximum(0, targs)\n    return np.sqrt(mean_squared_log_error(targs, preds))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:58.249687Z","iopub.execute_input":"2024-12-27T18:44:58.250062Z","iopub.status.idle":"2024-12-27T18:44:58.254259Z","shell.execute_reply.started":"2024-12-27T18:44:58.250037Z","shell.execute_reply":"2024-12-27T18:44:58.25345Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.data.transforms import RandomSplitter\ntrain_split,valid_split = RandomSplitter(valid_pct=0.2)(df)\ntrain_indeps,train_deps = indeps.iloc[train_split],deps.iloc[train_split]\nvalid_indeps,valid_deps = indeps.iloc[valid_split],deps.iloc[valid_split]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:44:58.255494Z","iopub.execute_input":"2024-12-27T18:44:58.255706Z","iopub.status.idle":"2024-12-27T18:44:58.875491Z","shell.execute_reply.started":"2024-12-27T18:44:58.255689Z","shell.execute_reply":"2024-12-27T18:44:58.874559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(arch, params):\n    if(arch == 'LGB'): return lgb.LGBMRegressor(**params, n_jobs=-1, verbose=-1)\n    if(arch == 'XGB'): return xgb.XGBRFRegressor(**params, n_jobs=-1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:45:01.066383Z","iopub.execute_input":"2024-12-27T18:45:01.066701Z","iopub.status.idle":"2024-12-27T18:45:01.071101Z","shell.execute_reply.started":"2024-12-27T18:45:01.066675Z","shell.execute_reply":"2024-12-27T18:45:01.070193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial, arch):\n    start = time.time()\n    params = {\n        'random_state': 42,\n        'metric': 'rmse',\n        #'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.6, 1), # almost always says 1\n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 1),\n        'num_leaves': trial.suggest_int('num_leaves', 31, 512),\n        'n_estimators': trial.suggest_int('n_estimators', 1, 200),\n        #'max_bins': trial.suggest_int('max_bins', 10, 1e5),\n        'max_depth': trial.suggest_int('max_depth', 2, 30),\n        #'min_data_in_leaf': trial.suggest_int('min_data_in_leaf', 5, 50),\n        #'min_split_gain': trial.suggest_uniform('min_split_gain', 0, 1),\n        #'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-5, 1),\n        #'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-5, 1),\n        #'subsample': trial.suggest_uniform('subsample', 0.5, 1),\n    }\n    model = get_model(arch, params)\n    model.fit(train_indeps, train_deps)\n    preds = model.predict(valid_indeps)\n    score = rmsle(valid_deps, preds)\n    end = time.time()\n    print(f'RMSLE: {score:<8.4f} elapsed time: {(end - start):.2f}s')\n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:11:25.687268Z","iopub.execute_input":"2024-12-27T18:11:25.687702Z","iopub.status.idle":"2024-12-27T18:11:25.716727Z","shell.execute_reply.started":"2024-12-27T18:11:25.687663Z","shell.execute_reply":"2024-12-27T18:11:25.715372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import partial\nobjective_lgb = partial(objective, arch='LGB')\nobjective_xgb = partial(objective, arch='XGB')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:09:27.640859Z","iopub.execute_input":"2024-12-27T18:09:27.641264Z","iopub.status.idle":"2024-12-27T18:09:27.645468Z","shell.execute_reply.started":"2024-12-27T18:09:27.641226Z","shell.execute_reply":"2024-12-27T18:09:27.644473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import optuna\noptuna.logging.set_verbosity(optuna.logging.ERROR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:09:29.125072Z","iopub.execute_input":"2024-12-27T18:09:29.125525Z","iopub.status.idle":"2024-12-27T18:09:29.130181Z","shell.execute_reply.started":"2024-12-27T18:09:29.125491Z","shell.execute_reply":"2024-12-27T18:09:29.12897Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"start = time.time()\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective_lgb, n_trials=50, n_jobs=-1)\n\nprint(f'best hyperparameters: {study.best_params}')\nprint(f'total elapsed time: {(time.time() - start):.2f}s')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def test_params(arch, params):\n    model = get_model(arch, params)\n    model.fit(train_indeps, train_deps)\n    preds = model.predict(valid_indeps)\n    return rmsle(valid_deps, preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:25:51.888603Z","iopub.execute_input":"2024-12-27T01:25:51.88892Z","iopub.status.idle":"2024-12-27T01:25:51.893072Z","shell.execute_reply.started":"2024-12-27T01:25:51.888892Z","shell.execute_reply":"2024-12-27T01:25:51.89213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimal_params = {'colsample_bytree': 0.9997851104353842, 'learning_rate': 0.046642320150933586, 'num_leaves': 463, 'n_estimators': 151, 'max_depth': 20}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:28:34.474629Z","iopub.execute_input":"2024-12-27T01:28:34.474941Z","iopub.status.idle":"2024-12-27T01:28:34.478993Z","shell.execute_reply.started":"2024-12-27T01:28:34.474918Z","shell.execute_reply":"2024-12-27T01:28:34.477996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(arch, params, name=''):\n    test_df = pd.read_csv(path/'test.csv')\n    test_indeps,_ = proc_data(test_df)\n    submit_df = pd.read_csv(path/'sample_submission.csv')\n    model = get_model(arch, params)\n    model.fit(indeps, deps)\n    preds = model.predict(test_indeps)\n    submit_df['Premium Amount'] = preds\n    if name == '': name = 'submission'\n    return submit_df.to_csv(f'{name}.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:28:28.047427Z","iopub.execute_input":"2024-12-27T01:28:28.047774Z","iopub.status.idle":"2024-12-27T01:28:28.0525Z","shell.execute_reply.started":"2024-12-27T01:28:28.047745Z","shell.execute_reply":"2024-12-27T01:28:28.051468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit_csv = submit('LGB', optimal_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T01:29:59.961373Z","iopub.execute_input":"2024-12-27T01:29:59.961675Z","iopub.status.idle":"2024-12-27T01:30:29.091373Z","shell.execute_reply.started":"2024-12-27T01:29:59.96164Z","shell.execute_reply":"2024-12-27T01:30:29.090635Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submit_csv","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install smac ConfigSpace","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ConfigSpace import Configuration","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:45:36.603695Z","iopub.execute_input":"2024-12-27T18:45:36.604178Z","iopub.status.idle":"2024-12-27T18:45:36.656936Z","shell.execute_reply.started":"2024-12-27T18:45:36.604127Z","shell.execute_reply":"2024-12-27T18:45:36.656064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(config: Configuration, arch):\n    params = {\n        'random_state': 42,\n        'metric': 'rmsle',\n    }\n    params.update(config.get_dictionary())\n    model = get_model(arch, params)\n    model.fit(train_indeps, train_deps)\n    preds = model.predict(valid_indeps)\n    score = rmsle(valid_deps, preds)\n    end = time.time()\n    print(f'RMSLE: {score:<8.4f} elapsed time: {(end - start):.2f}s')\n    return score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:45:37.186627Z","iopub.execute_input":"2024-12-27T18:45:37.186911Z","iopub.status.idle":"2024-12-27T18:45:37.192532Z","shell.execute_reply.started":"2024-12-27T18:45:37.186888Z","shell.execute_reply":"2024-12-27T18:45:37.191632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import partial\nobjective_lgb = partial(objective, arch='LGB')\nobjective_xgb = partial(objective, arch='XGB')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:45:40.301761Z","iopub.execute_input":"2024-12-27T18:45:40.302196Z","iopub.status.idle":"2024-12-27T18:45:40.307405Z","shell.execute_reply.started":"2024-12-27T18:45:40.302157Z","shell.execute_reply":"2024-12-27T18:45:40.306379Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from ConfigSpace import ConfigurationSpace\nfrom ConfigSpace.hyperparameters import UniformFloatHyperparameter, UniformIntegerHyperparameter\n\nconfig_space = ConfigurationSpace()\n\nlearning_rate = UniformFloatHyperparameter(\"learning_rate\", lower=0.001, upper=0.1, default_value=0.01)\nnum_leaves = UniformIntegerHyperparameter(\"num_leaves\", lower=31, upper=500, default_value=31)\nmax_depth = UniformIntegerHyperparameter(\"max_depth\", lower=1, upper=30, default_value=6)\nmin_data_in_leaf = UniformIntegerHyperparameter(\"min_data_in_leaf\", lower=10, upper=100, default_value=20)\n\nconfig_space.add_hyperparameters([learning_rate, num_leaves, max_depth, min_data_in_leaf])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:45:46.09152Z","iopub.execute_input":"2024-12-27T18:45:46.0918Z","iopub.status.idle":"2024-12-27T18:45:46.101664Z","shell.execute_reply.started":"2024-12-27T18:45:46.091778Z","shell.execute_reply":"2024-12-27T18:45:46.100821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from smac import Scenario\nfrom smac.facade.abstract_facade import SMAC4HPO\n\n# Define scenario\nscenario = Scenario({\n    \"run_obj\": \"quality\",  # Optimize for \"quality\" (minimization)\n    \"runcount-limit\": 50,  # Number of trials\n    \"cs\": config_space,    # Configuration space\n    \"deterministic\": True, # Deterministic objective function\n    'n_workers': -1\n})\n\nsmac = SMAC4HPO(scenario=scenario, tae_runner=objective_lgb)\nbest_config = smac.optimize()\n\nprint(\"Best hyperparameters:\", best_config)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-27T18:48:52.964273Z","iopub.execute_input":"2024-12-27T18:48:52.964586Z","iopub.status.idle":"2024-12-27T18:48:52.980713Z","shell.execute_reply.started":"2024-12-27T18:48:52.964559Z","shell.execute_reply":"2024-12-27T18:48:52.979468Z"}},"outputs":[],"execution_count":null}]}