{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"df2ad90b-31f6-4979-96ae-19beed14175a","cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom zipfile import ZipFile\nimport polars as pl\nimport pandas as pd\n\nfrom sklearn.preprocessing import OrdinalEncoder, RobustScaler, OneHotEncoder\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import cross_val_score\n\nfrom xgboost import XGBRegressor\nimport optuna\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:17.088458Z","iopub.execute_input":"2024-12-06T15:50:17.088935Z","iopub.status.idle":"2024-12-06T15:50:17.095368Z","shell.execute_reply.started":"2024-12-06T15:50:17.088898Z","shell.execute_reply":"2024-12-06T15:50:17.094025Z"}},"outputs":[],"execution_count":null},{"id":"56713ef1-e694-4d84-a216-41fdb1d7518d","cell_type":"code","source":"class CFG:\n    root = Path(r'/kaggle/input/playground-series-s4e12')\n    work = Path(r'/kaggle/working/')\n    target = 'Premium Amount'\n    trainyn = False\n    random_state = 42\ncfg = CFG()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:17.097395Z","iopub.execute_input":"2024-12-06T15:50:17.097762Z","iopub.status.idle":"2024-12-06T15:50:17.11155Z","shell.execute_reply.started":"2024-12-06T15:50:17.097727Z","shell.execute_reply":"2024-12-06T15:50:17.11026Z"}},"outputs":[],"execution_count":null},{"id":"b964bd2d-6388-46bd-bb23-4ddf80a1ef6f","cell_type":"code","source":"def getData():\n    df_train = pl.read_csv(cfg.root / 'train.csv')\n    df_test = pl.read_csv(cfg.root / 'test.csv')\n    df_merged = pl.concat([df_train.select(pl.all().exclude(cfg.target)), df_test])\n    return df_train, df_test, df_merged","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:17.113395Z","iopub.execute_input":"2024-12-06T15:50:17.113867Z","iopub.status.idle":"2024-12-06T15:50:17.131379Z","shell.execute_reply.started":"2024-12-06T15:50:17.113801Z","shell.execute_reply":"2024-12-06T15:50:17.130213Z"}},"outputs":[],"execution_count":null},{"id":"69ecac35-00f1-4463-8bef-2f962b48de12","cell_type":"code","source":"def preprocessData():\n    df_train, df_test, df_merged = getData()\n\n    columns_hot = ['Marital Status','Gender']\n\n    df_train = df_train.with_columns(\n        (pl.col('Policy Start Date').str.to_datetime().dt.year()).alias('Year'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.month()).alias('Month'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.day()).alias('Day')\n    )\n\n    df_test = df_test.with_columns(\n        (pl.col('Policy Start Date').str.to_datetime().dt.year()).alias('Year'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.month()).alias('Month'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.day()).alias('Day')\n    )\n\n    df_merged = df_merged.with_columns(\n        (pl.col('Policy Start Date').str.to_datetime().dt.year()).alias('Year'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.month()).alias('Month'),\n        (pl.col('Policy Start Date').str.to_datetime().dt.day()).alias('Day')\n    )\n\n    #scikit-learn is an old version therefore unable to use polars dataframe so I had to convert ouputs to pandas\n    \n    tr_hot = OneHotEncoder(sparse_output=False)\n    tr_hot.set_output(transform='pandas')\n    \n    # df_merged = df_merged.with_columns(tr_hot.fit_transform(df_merged.select(pl.col(columns_hot)))).select(pl.all().exclude(columns_hot))\n    # df_train = df_train.with_columns(tr_hot.transform(df_train.select(pl.col(columns_hot)))).select(pl.all().exclude(columns_hot))\n    # df_test = df_test.with_columns(tr_hot.transform(df_test.select(pl.col(columns_hot)))).select(pl.all().exclude(columns_hot))\n    \n    \n    df_train_ids = df_train.select(pl.col('id'))\n    df_test_ids = df_test.select(pl.col('id'))\n    columns = df_merged.columns\n    \n    columns_str = [x for x in df_merged.columns if df_merged.select(pl.col(x)).dtypes[0] == pl.String]\n    \n    columns_num = [x for x in df_merged.columns if df_merged.select(pl.col(x)).dtypes[0] != pl.String]\n    columns_num.remove('id')\n\n    df_merged = df_merged.with_columns(df_merged.select(pl.col(columns_str)).fill_null('Unknown'))\n\n    \n    \n    tr_num_imp = SimpleImputer(strategy='median')\n    tr_num_imp.set_output(transform='pandas')\n    pl_num_imp = Pipeline([('num_imp',tr_num_imp)])\n    pl_num_imp.set_output(transform='pandas')\n    tr_col = ColumnTransformer([\n        ('num_imp',pl_num_imp, columns_num)\n    ], verbose_feature_names_out=False, remainder='passthrough')\n    tr_col.set_output(transform='pandas')\n    tr_rob = RobustScaler()\n    tr_rob.set_output(transform='pandas')\n    tr_ord = OrdinalEncoder()\n    tr_ord.set_output(transform='pandas')\n\n\n    #unable to use polars dataframe in kaggle\n    \n    df_merged = df_merged.to_pandas()\n    df_merged = tr_col.fit_transform(df_merged)\n    \n    \n    # df_merged = df_merged.with_columns([df_merged.select(pl.col(columns_str)).fill_null('Unknown')])\n    \n    df_merged[columns_str] = tr_ord.fit_transform(df_merged[columns_str])\n    df_merged[columns_str + columns_num] = tr_rob.fit_transform(df_merged[columns_str + columns_num])\n    \n    df_merged = pl.from_pandas(df_merged)\n    \n    return df_train_ids.join(df_merged, left_on=['id'], right_on=['id'], how='inner').select(pl.col(columns)).with_columns(df_train.select(pl.col(cfg.target))),df_test_ids.join(df_merged, left_on=['id'], right_on=['id'], how='inner').select(pl.col(columns)),df_merged.select(pl.col(columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:17.132754Z","iopub.execute_input":"2024-12-06T15:50:17.13318Z","iopub.status.idle":"2024-12-06T15:50:17.15004Z","shell.execute_reply.started":"2024-12-06T15:50:17.133145Z","shell.execute_reply":"2024-12-06T15:50:17.1488Z"}},"outputs":[],"execution_count":null},{"id":"6186a8d6-fe66-4c75-bb52-328c73af0d30","cell_type":"code","source":"df_train, df_test, df_merged = preprocessData()\nX = df_train.select(pl.all().exclude('id', cfg.target)).to_numpy()\ny = df_train.select(pl.col(cfg.target)).to_numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:17.152192Z","iopub.execute_input":"2024-12-06T15:50:17.152546Z","iopub.status.idle":"2024-12-06T15:50:38.15896Z","shell.execute_reply.started":"2024-12-06T15:50:17.152514Z","shell.execute_reply":"2024-12-06T15:50:38.157758Z"}},"outputs":[],"execution_count":null},{"id":"619d1ac8-00ff-4589-b7bc-b7897d4a455c","cell_type":"code","source":"def objective_xgb(trial):\n    param_xgb = {\n    'max_depth': trial.suggest_int('max_depth', 3, 10),\n    'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1),\n    'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n    # 'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n    'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n    'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n    'gamma': trial.suggest_float('gamma', 0.0, 1.0),\n    'reg_alpha': trial.suggest_float('reg_alpha', 0.0, 1.0),\n    'reg_lambda': trial.suggest_float('reg_lambda', 0.0, 1.0),\n    'random_state': cfg.random_state,\n    'n_jobs': -1\n    }\n    \n    clf = XGBRegressor(**param_xgb) \n    return cross_val_score(clf, X, y, cv=3, scoring='neg_root_mean_squared_error').mean()\n\nif cfg.trainyn == True:\n    study_xgb = optuna.create_study(direction='maximize', study_name='XGB')\n    study_xgb.optimize(objective_xgb, n_trials=500)\n    \n    print('Best hyperparameters:', study_xgb.best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:38.161287Z","iopub.execute_input":"2024-12-06T15:50:38.161776Z","iopub.status.idle":"2024-12-06T15:50:38.170931Z","shell.execute_reply.started":"2024-12-06T15:50:38.161726Z","shell.execute_reply":"2024-12-06T15:50:38.169393Z"}},"outputs":[],"execution_count":null},{"id":"0d16baa9-54d4-4d34-8cf5-39060d2ac528","cell_type":"code","source":"params_xgb = {'n_jobs': -1, 'random_state': cfg.random_state, 'max_depth': 10, 'learning_rate': 0.014317743951304335, 'n_estimators': 884, 'subsample': 0.8524592737747381, 'colsample_bytree': 0.9582643287745425, 'gamma': 0.43154220359434237, 'reg_alpha': 0.2641652654906262, 'reg_lambda': 0.31789200776235}\nmodel_xgb = XGBRegressor(**params_xgb)\ncross_val_score(model_xgb, X, y, cv=5, scoring='neg_root_mean_squared_error').mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:50:38.172379Z","iopub.execute_input":"2024-12-06T15:50:38.172734Z","iopub.status.idle":"2024-12-06T15:57:17.000975Z","shell.execute_reply.started":"2024-12-06T15:50:38.1727Z","shell.execute_reply":"2024-12-06T15:57:16.999513Z"}},"outputs":[],"execution_count":null},{"id":"3f2e13a3-ba5b-4f83-b8f4-41e41d3a369b","cell_type":"code","source":"model_xgb.fit(X, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:57:17.002676Z","iopub.execute_input":"2024-12-06T15:57:17.003019Z","iopub.status.idle":"2024-12-06T15:58:43.72047Z","shell.execute_reply.started":"2024-12-06T15:57:17.002984Z","shell.execute_reply":"2024-12-06T15:58:43.719308Z"}},"outputs":[],"execution_count":null},{"id":"a56c95ab-42d1-4429-8442-bc0988a37c90","cell_type":"code","source":"preds = model_xgb.predict(df_test.select(pl.all().exclude('id')))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:58:43.7216Z","iopub.execute_input":"2024-12-06T15:58:43.721954Z","iopub.status.idle":"2024-12-06T15:59:06.065268Z","shell.execute_reply.started":"2024-12-06T15:58:43.721919Z","shell.execute_reply":"2024-12-06T15:59:06.064305Z"}},"outputs":[],"execution_count":null},{"id":"26958141-3d67-4e31-8aff-0f72a176b30b","cell_type":"code","source":"df_sub = df_test.with_columns((pl.Series(preds).alias(cfg.target))).select(pl.col('id',cfg.target))\ndf_sub.write_csv(cfg.work / 'submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-06T15:59:06.066256Z","iopub.execute_input":"2024-12-06T15:59:06.066584Z","iopub.status.idle":"2024-12-06T15:59:06.169312Z","shell.execute_reply.started":"2024-12-06T15:59:06.066551Z","shell.execute_reply":"2024-12-06T15:59:06.167857Z"}},"outputs":[],"execution_count":null}]}