{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\nimport numpy as np\nimport pandas as pd \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:41:12.629319Z","iopub.execute_input":"2024-12-10T07:41:12.630769Z","iopub.status.idle":"2024-12-10T07:41:12.639186Z","shell.execute_reply.started":"2024-12-10T07:41:12.630689Z","shell.execute_reply":"2024-12-10T07:41:12.637456Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"root_path = '/kaggle/input'\nsample_sub = pd.read_csv(f'{root_path}/playground-series-s4e12/sample_submission.csv')\ntrain = pd.read_csv(f'{root_path}/playground-series-s4e12/train.csv')\ntest = pd.read_csv(f'{root_path}/playground-series-s4e12/test.csv')\nOriginal = pd.read_csv(\"/kaggle/input/insurance-premium-prediction/Insurance Premium Prediction Dataset.csv\")\nOriginal = Original.dropna()\n\ntest_ids = test['id'].values\n\ntrain.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)\n\ntrain = pd.concat([train, Original], ignore_index=True)\ndef removeNoise(df, columns, threshold=100):\n    \n    for col in columns:\n        df.loc[df[col].value_counts(dropna=False)[df[col]].values < threshold, col] = \"noise\"\n    \n    return df\n\ncols = test.select_dtypes(exclude=[\"float64\",\"int32\"]).columns\ntrain = removeNoise(train, cols)\ntest = removeNoise(test, cols)\n\ncat_c = test.select_dtypes(include=\"object\").columns\ntrain[cat_c] = train[cat_c].fillna('None').astype(\"category\")\ntest[cat_c] = test[cat_c].fillna('None').astype(\"category\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:41:12.642086Z","iopub.execute_input":"2024-12-10T07:41:12.642661Z","iopub.status.idle":"2024-12-10T07:41:33.357369Z","shell.execute_reply.started":"2024-12-10T07:41:12.642605Z","shell.execute_reply":"2024-12-10T07:41:33.355723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport optuna\nimport pandas as pd\nfrom optuna.samplers import TPESampler\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold\nfrom typing import Dict, List, Tuple, Union\nfrom xgboost import XGBRegressor\n\nclass xgboost_model:\n    def __init__(self, train: pd.DataFrame, test: pd.DataFrame, target: str, base_params=None):\n        # Initialize the model with train and test data, target column, and optional base parameters\n        self.train = train\n        self.test = test\n        self.model_dict: Dict[str, XGBRegressor] = {}\n        self.test_predict_list: List[np.ndarray] = []\n        self.target = target\n        if base_params is None:\n            base_params = {}\n        self.base_params = base_params\n\n    def objective(self, trial: optuna.Trial) -> float:\n        # Define the objective function for Optuna optimization\n        params = {\n            \"max_depth\": trial.suggest_int(\"max_depth\", 2, 10),\n            \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-3, 0.3, log=True),\n            \"n_estimators\": trial.suggest_int(\"n_estimators\", 50, 1500),\n            \"subsample\": trial.suggest_float(\"subsample\", 0.5, 1.0),\n            \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.5, 1.0),\n            \"colsample_bylevel\": trial.suggest_float(\"colsample_bylevel\", 0.5, 1.0),\n            \"colsample_bynode\": trial.suggest_float(\"colsample_bynode\", 0.5, 1.0),\n            \"reg_alpha\": trial.suggest_float(\"reg_alpha\", 0, 1),\n            \"reg_lambda\": trial.suggest_float(\"reg_lambda\", 0, 1),\n            \"gamma\": trial.suggest_float(\"gamma\", 0, 1),\n            \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 20),\n            \"max_delta_step\": trial.suggest_int(\"max_delta_step\", 0, 10),\n            \"grow_policy\": trial.suggest_categorical(\"grow_policy\", [\"depthwise\", \"lossguide\"]),\n            'max_cat_to_onehot': trial.suggest_int(\"max_cat_to_onehot\", 4, 32),\n            'max_cat_threshold': trial.suggest_int(\"max_cat_threshold\", 32, 32),\n            **self.base_params\n        }\n        scores, _, _ = self.fit(params)\n        return np.mean(scores)\n\n    def fit(self, params: Dict[str, Union[int, float, str, bool]]) -> Tuple[List[float], List[np.ndarray], np.ndarray]:\n        # Fit the model using the given parameters\n        label_columns = [self.target]\n        train_cols = [col for col in self.train.columns.to_list() if col not in label_columns]\n        scores = []\n        mskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n        oof_valid_preds = np.zeros((self.train[train_cols].shape[0], 1))\n        for fold, (train_idx, valid_idx) in enumerate(mskf.split(self.train[train_cols], self.train[label_columns])):\n            # Perform 5-fold cross-validation\n            X_train, y_train = self.train[train_cols].iloc[train_idx], self.train[label_columns].iloc[train_idx]\n            X_valid, y_valid = self.train[train_cols].iloc[valid_idx], self.train[label_columns].iloc[valid_idx]\n            params[\"enable_categorical\"] = True\n            model = XGBRegressor(**params)\n            model.fit(X_train, y_train, eval_set=[(X_valid, y_valid)], verbose=False)\n            valid_preds = model.predict(X_valid)\n            oof_valid_preds[valid_idx] = valid_preds.reshape(-1, 1)\n            test_predict = model.predict(self.test[train_cols])\n            self.test_predict_list.append(test_predict)\n            \n            mse = mean_squared_error(y_valid, valid_preds)\n            scores.append(mse)\n            self.model_dict[f'fold_{fold}'] = model\n           \n        oof_score = mean_squared_error(self.train[label_columns], oof_valid_preds)\n        scores.append(oof_score)\n        print(f'The average mean square error is {np.mean(scores)}')\n        return scores, self.test_predict_list, oof_valid_preds\n\n    def optimize(self, n_trials: int = 100) -> Dict[str, Union[int, float, str, bool]]:\n        # Perform hyperparameter optimization using Optuna\n        study = optuna.create_study(direction=\"maximize\", sampler=TPESampler(seed=42))\n        study.optimize(self.objective, timeout=n_trials, show_progress_bar=True)\n        print(\"Best trial:\")\n        trial = study.best_trial\n        print(\" Value:\", trial.value)\n        print(\" Params:\")\n        for key, value in trial.params.items():\n            print(f\" {key}: {value}\")\n        return study.best_params","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:41:33.359089Z","iopub.execute_input":"2024-12-10T07:41:33.359461Z","iopub.status.idle":"2024-12-10T07:41:33.383017Z","shell.execute_reply.started":"2024-12-10T07:41:33.359424Z","shell.execute_reply":"2024-12-10T07:41:33.381688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = xgboost_model(train, test, 'Premium Amount')\nbest_params = model.optimize(n_trials=100)\nscores, test_preds, oof_preds = model.fit(best_params)\ntest_preds = np.mean(test_preds, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:42:23.869823Z","iopub.execute_input":"2024-12-10T07:42:23.870299Z","iopub.status.idle":"2024-12-10T07:42:23.875705Z","shell.execute_reply.started":"2024-12-10T07:42:23.870248Z","shell.execute_reply":"2024-12-10T07:42:23.874526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_sub[\"Premium Amount\"] = test_preds\nsample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-10T07:42:23.877006Z","iopub.execute_input":"2024-12-10T07:42:23.877322Z","iopub.status.idle":"2024-12-10T07:42:25.193133Z","shell.execute_reply.started":"2024-12-10T07:42:23.877291Z","shell.execute_reply":"2024-12-10T07:42:25.191761Z"}},"outputs":[],"execution_count":null}]}