{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":10220676,"sourceType":"datasetVersion","datasetId":6318248},{"sourceId":10220771,"sourceType":"datasetVersion","datasetId":6318310}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\n\nimport numpy as np\nimport polars as pl\nimport pandas as pd\n\nfrom sklearn.base import clone\nimport optuna\nimport os\n\nfrom tqdm import tqdm\nimport category_encoders as ce\nfrom IPython.display import clear_output\n\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n!pip install -qq pytorch_tabnet\n\nimport warnings\nwarnings.filterwarnings('ignore')\npd.options.display.max_columns = None\n\nfrom lightgbm import LGBMRegressor\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor, Pool\n\n\nfrom sklearn.model_selection import *\nfrom sklearn.metrics import *\n\nimport joblib","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-12-21T14:18:37.717478Z","iopub.execute_input":"2024-12-21T14:18:37.717929Z","iopub.status.idle":"2024-12-21T14:18:55.226027Z","shell.execute_reply.started":"2024-12-21T14:18:37.71786Z","shell.execute_reply":"2024-12-21T14:18:55.224544Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Carga de datos","metadata":{}},{"cell_type":"code","source":"correr_nonlog = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:18:55.229026Z","iopub.execute_input":"2024-12-21T14:18:55.229804Z","iopub.status.idle":"2024-12-21T14:18:55.235632Z","shell.execute_reply.started":"2024-12-21T14:18:55.229747Z","shell.execute_reply":"2024-12-21T14:18:55.234455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ntrain = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\nsample = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\nif not correr_nonlog:\n    nonlog_fe , nonlog = joblib.load(\"/kaggle/input/non-log-cat-pk/cat_non_loged.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:18:55.240979Z","iopub.execute_input":"2024-12-21T14:18:55.241324Z","iopub.status.idle":"2024-12-21T14:19:06.354305Z","shell.execute_reply.started":"2024-12-21T14:18:55.241291Z","shell.execute_reply":"2024-12-21T14:19:06.353127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['is_train'] = 1\ntest['is_train'] = 0\ndata = pd.concat([train, test], ignore_index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:19:06.356577Z","iopub.execute_input":"2024-12-21T14:19:06.356998Z","iopub.status.idle":"2024-12-21T14:19:06.759317Z","shell.execute_reply.started":"2024-12-21T14:19:06.356964Z","shell.execute_reply":"2024-12-21T14:19:06.758197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ingeniería de datos","metadata":{}},{"cell_type":"code","source":"def data_ing(data):\n    data['Policy Start Date'] = pd.to_datetime(data['Policy Start Date'])\n    data['Year'] = data['Policy Start Date'].dt.year\n    data['Day'] = data['Policy Start Date'].dt.day\n    data['Month'] = data['Policy Start Date'].dt.month\n    data['Month_name'] = data['Policy Start Date'].dt.month_name()\n    data['Day_of_week'] = data['Policy Start Date'].dt.day_name()\n    data['Week'] = data['Policy Start Date'].dt.isocalendar().week\n    data['Year_sin'] = np.sin(2 * np.pi * data['Year'])\n    data['Year_cos'] = np.cos(2 * np.pi * data['Year'])\n    data['Year_sin'] = np.sin(2 * np.pi * data['Year']) \n    data['Year_cos'] = np.cos(2 * np.pi * data['Year']) \n    data['Month_sin'] = np.sin(2 * np.pi * data['Month'] / 12) \n    data['Month_cos'] = np.cos(2 * np.pi * data['Month'] / 12)\n    data['Day_sin'] = np.sin(2 * np.pi * data['Day'] / 31)  \n    data['Day_cos'] = np.cos(2 * np.pi * data['Day'] / 31)\n    data['Group']=(data['Year']-2020)*48+data['Month']*4+data['Day']//7\n    #data[\"Policy_age_yrs\"] = (datetime.now() - data['Policy_Start_Date']).dt.total_seconds()\n    data[\"seconds since 1970\"] = data['Policy Start Date'].astype(\"int64\")\n\n    # Counter for missing values per row\n    data['MissingValuesCount'] = data.isna().sum(axis=1)\n    data['MissingHealth'] = data['Health Score'].isna().astype(int)\n\n    data['Days Passed'] = (data['Policy Start Date'].max() - data['Policy Start Date']).dt.days\n\n\n    # Creating Ratio variables, as well as categorizing continuous ones\n    data['Claims v Duration'] = data['Previous Claims'] / data['Insurance Duration']\n    data['Health vs Claims'] = data['Health Score'] / data['Previous Claims']\n    data['Cat Credit Score'] = data['Credit Score'].copy()\n    data['Int Credit Score'] = data['Credit Score'].apply(lambda x: int(x) if pd.notna(x) else x)\n    data['Int Annual Income'] = data['Annual Income'].apply(lambda x: int(x) if pd.notna(x) else x)\n\n\n    #data['Annual Income Log'] = np.log1p(data['Annual Income'])\n    #data['Annual Income SQRT'] = np.sqrt(data['Annual Income'])\n    #data['Annual Income cbrt'] = np.cbrt(data['Annual Income'])\n    #data['Annual Income family'] = data['Annual Income']/(data['Number of Dependents'] + 1)\n    #data['Annual Income family Log'] = np.log1p(data['Annual Income family'])\n    #data['Annual Income family SQRT'] = np.sqrt(data['Annual Income family'])\n    #data['Annual Income family cbrt'] = np.cbrt(data['Annual Income family'])\n\n    #data['Annual Income Log family'] = data['Annual Income Log']/(data['Number of Dependents'] + 1)\n    #data['Annual Income SQRT family'] = data['Annual Income SQRT']/(data['Number of Dependents'] + 1)\n    #data['Annual Income cbrt family'] = data['Annual Income cbrt']/(data['Number of Dependents'] + 1)\n\n\n    #data['score'] = data['Credit Score'] + data['Health Score']\n    #data['score avg'] = data['Credit Score']*0.5 + data['Health Score']*0.5\n    #data['Annual Income per score'] = data['Annual Income']/data['score']\n    #data['Annual Income log per score'] = data['Annual Income Log']/data['score']\n    #data['Annual Income sqrt per score'] = data['Annual Income SQRT']/data['score']\n    return data\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:19:06.760485Z","iopub.execute_input":"2024-12-21T14:19:06.760824Z","iopub.status.idle":"2024-12-21T14:19:06.77352Z","shell.execute_reply.started":"2024-12-21T14:19:06.760788Z","shell.execute_reply":"2024-12-21T14:19:06.772268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data_ing(data)\ndata.drop('Policy Start Date', axis=1, inplace=True)\ndata.drop('id', axis=1, inplace=True) \n\n\ncolumns_to_convert = data.columns.difference(['Premium Amount'])\ndata[columns_to_convert] = data[columns_to_convert].fillna('None').astype('string')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:19:06.775001Z","iopub.execute_input":"2024-12-21T14:19:06.775383Z","iopub.status.idle":"2024-12-21T14:19:51.141678Z","shell.execute_reply.started":"2024-12-21T14:19:06.775348Z","shell.execute_reply":"2024-12-21T14:19:51.140439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_to_convert = data.columns.difference(['Premium Amount'])\ndata[columns_to_convert] = data[columns_to_convert].fillna('None').astype('string')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:19:51.143187Z","iopub.execute_input":"2024-12-21T14:19:51.143633Z","iopub.status.idle":"2024-12-21T14:20:06.28473Z","shell.execute_reply.started":"2024-12-21T14:19:51.143585Z","shell.execute_reply":"2024-12-21T14:20:06.283646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndata.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:06.286192Z","iopub.execute_input":"2024-12-21T14:20:06.286708Z","iopub.status.idle":"2024-12-21T14:20:06.328696Z","shell.execute_reply.started":"2024-12-21T14:20:06.286659Z","shell.execute_reply":"2024-12-21T14:20:06.327518Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# CATBOOST NONLOG","metadata":{}},{"cell_type":"code","source":"data_train = data[data['is_train']== '1' ]\ndata_test = data[data['is_train']== '0' ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:06.330267Z","iopub.execute_input":"2024-12-21T14:20:06.330749Z","iopub.status.idle":"2024-12-21T14:20:09.220711Z","shell.execute_reply.started":"2024-12-21T14:20:06.330699Z","shell.execute_reply":"2024-12-21T14:20:09.21951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = data_train.drop(['Premium Amount'], axis=1)\ny = data_train['Premium Amount']\n\ndata_test = data_test.drop(['Premium Amount'], axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:09.221994Z","iopub.execute_input":"2024-12-21T14:20:09.222294Z","iopub.status.idle":"2024-12-21T14:20:12.050112Z","shell.execute_reply.started":"2024-12-21T14:20:09.222266Z","shell.execute_reply":"2024-12-21T14:20:12.048739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = X.columns.values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.051846Z","iopub.execute_input":"2024-12-21T14:20:12.052208Z","iopub.status.idle":"2024-12-21T14:20:12.056707Z","shell.execute_reply.started":"2024-12-21T14:20:12.052177Z","shell.execute_reply":"2024-12-21T14:20:12.055599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainCATBOOST_NONLOG(params, n_splits, SEED):\n     kfold = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=SEED)\n     train_mse_scores = []\n     val_mse_scores = []\n     models = []\n     oof = np.zeros(len(X))\n\n     for fold, (train_idx, val_idx) in enumerate(tqdm(kfold.split(X, y), desc=\"Training Folds\", total=n_splits)):\n         X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n         y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n         print(f\"-------- Woking on fold {fold} --------\")\n        \n         model = CatBoostRegressor(**params, random_state=SEED, verbose=200, eval_metric='RMSE', task_type='GPU')\n         model.fit(X_train, y_train, \n                   eval_set=[(X_val, y_val)], \n                   early_stopping_rounds=300,\n                  cat_features=cat_features)\n         \n         models.append(model)\n         y_train_pred = model.predict(X_train)\n         y_val_pred = model.predict(X_val)\n\n         y_train_pred = np.clip(y_train_pred, 20, 4999)\n         y_val_pred = np.clip(y_val_pred, 20, 4999)\n         \n         train_mse = np.sqrt(mean_squared_log_error(y_train, y_train_pred))\n         val_mse = np.sqrt(mean_squared_log_error(y_val, y_val_pred))\n\n         train_mse_scores.append(train_mse)\n         val_mse_scores.append(val_mse)\n         \n         oof[val_idx] = y_val_pred\n         \n         print(f\"\\n Scores Fold: {fold}\")\n         print(f\"Train RMSE: {train_mse:.4f}\")\n         print(f\"Validation RMSE: {val_mse:.4f}\")\n    \n     mean_train_mse = np.mean(train_mse_scores)\n     mean_val_mse = np.mean(val_mse_scores)\n     print(\"\\n Final Mean Scores:\")\n     print(f\"Mean Train RMSE: {mean_train_mse:.4f}\")\n     print(f\"Mean Validation RMSE: {mean_val_mse:.4f}\")\n\n     return mean_train_mse, mean_val_mse, models, oof","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.058301Z","iopub.execute_input":"2024-12-21T14:20:12.059018Z","iopub.status.idle":"2024-12-21T14:20:12.071552Z","shell.execute_reply.started":"2024-12-21T14:20:12.058971Z","shell.execute_reply":"2024-12-21T14:20:12.07025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objectiveCAT_NOLOG(trial):\n  params = {\n        \"loss_function\": \"RMSE\",\n        \"task_type\": \"GPU\",\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 1e-4, 1e-1),\n        \"iterations\": trial.suggest_int(\"iterations\", 100, 2000),\n        \"max_bin\": trial.suggest_int(\"max_bin\", 128, 512),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 4, 15),\n        \"colsample_bylevel\": trial.suggest_uniform(\"colsample_bylevel\", 0.5, 1.0),\n        \"l2_leaf_reg\": trial.suggest_loguniform(\"l2_leaf_reg\", 1e-3, 10.0),\n        \"random_strength\": trial.suggest_uniform(\"random_strength\", 0.1, 1.0),\n        \"verbose\": 0\n    }\n   \n\n  SEED = 42\n  n_splits = 5\n\n  mean_train_mse, mean_val_mse, models, oof = TrainCATBOOST_NONLOG(params, n_splits, SEED)\n  trial.set_user_attr('train_rmse', mean_train_mse)\n  trial.set_user_attr('val_rmse', mean_val_mse)\n  return mean_val_mse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.0771Z","iopub.execute_input":"2024-12-21T14:20:12.077474Z","iopub.status.idle":"2024-12-21T14:20:12.089352Z","shell.execute_reply.started":"2024-12-21T14:20:12.077441Z","shell.execute_reply":"2024-12-21T14:20:12.088106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RETUNE_CAT_NOLOG= False \n\ncat_params_mse = {'iterations': 600, 'max_depth': 6, 'learning_rate': 0.1, 'l2_leaf_reg': 0.7}\n\n\nif RETUNE_CAT_NOLOG:\n    study = optuna.create_study(direction='minimize')\n    study.enqueue_trial(cat_params_mse)\n    study.optimize(objectiveCAT_NOLOG, n_trials=50,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.091261Z","iopub.execute_input":"2024-12-21T14:20:12.09161Z","iopub.status.idle":"2024-12-21T14:20:12.108628Z","shell.execute_reply.started":"2024-12-21T14:20:12.091575Z","shell.execute_reply":"2024-12-21T14:20:12.107396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if correr_nonlog:\n    SEED = 42\n    n_splits = 5\n    mean_train_mse, mean_val_mse, models_mse, oof_mse = TrainCATBOOST_NONLOG(cat_params_mse, n_splits, SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.110147Z","iopub.execute_input":"2024-12-21T14:20:12.11108Z","iopub.status.idle":"2024-12-21T14:20:12.119128Z","shell.execute_reply.started":"2024-12-21T14:20:12.111028Z","shell.execute_reply":"2024-12-21T14:20:12.118195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if correr_nonlog:\n    test_predictions_mse = np.zeros(len(data_test))\n\n    for model in models_mse:\n        test_predictions_mse += np.clip(model.predict(data_test), 20, 4999) / len(models_mse)\n\n    sample['Premium Amount'] = test_predictions_mse\n    sample.to_csv('CAT_NONLOG_V1.csv', index=False)\n    joblib.dump([oof_mse,test_predictions_mse],\"cat_non_loged.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.120769Z","iopub.execute_input":"2024-12-21T14:20:12.121143Z","iopub.status.idle":"2024-12-21T14:20:12.13311Z","shell.execute_reply.started":"2024-12-21T14:20:12.121111Z","shell.execute_reply":"2024-12-21T14:20:12.13195Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensamble CATBOOST NONLOG: CATBOOST NONLOG AS INPUT","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\nsample = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:12.134525Z","iopub.execute_input":"2024-12-21T14:20:12.134968Z","iopub.status.idle":"2024-12-21T14:20:19.767847Z","shell.execute_reply.started":"2024-12-21T14:20:12.134924Z","shell.execute_reply":"2024-12-21T14:20:19.766803Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['is_train'] = 1\ntest['is_train'] = 0\ndata = pd.concat([train, test], ignore_index=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:19.769202Z","iopub.execute_input":"2024-12-21T14:20:19.769509Z","iopub.status.idle":"2024-12-21T14:20:20.019092Z","shell.execute_reply.started":"2024-12-21T14:20:19.769479Z","shell.execute_reply":"2024-12-21T14:20:20.017985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = data_ing(data)\ndata.drop('Policy Start Date', axis=1, inplace=True)\ndata.drop('id', axis=1, inplace=True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:20.020841Z","iopub.execute_input":"2024-12-21T14:20:20.021318Z","iopub.status.idle":"2024-12-21T14:20:29.225993Z","shell.execute_reply.started":"2024-12-21T14:20:20.021272Z","shell.execute_reply":"2024-12-21T14:20:29.224646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:29.227207Z","iopub.execute_input":"2024-12-21T14:20:29.227551Z","iopub.status.idle":"2024-12-21T14:20:29.274514Z","shell.execute_reply.started":"2024-12-21T14:20:29.227512Z","shell.execute_reply":"2024-12-21T14:20:29.273211Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train = data[data['is_train']== 1 ]\ndata_test = data[data['is_train']== 0 ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:29.276323Z","iopub.execute_input":"2024-12-21T14:20:29.276695Z","iopub.status.idle":"2024-12-21T14:20:30.221065Z","shell.execute_reply.started":"2024-12-21T14:20:29.27666Z","shell.execute_reply":"2024-12-21T14:20:30.220066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if not correr_nonlog:\n    data_train['premium_amount_mse'] = nonlog_fe\n    data_test['premium_amount_mse'] = nonlog\nelse:\n    data_train['premium_amount_mse'] = oof_mse\n    data_test['premium_amount_mse'] = test_predictions_mse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.222275Z","iopub.execute_input":"2024-12-21T14:20:30.222685Z","iopub.status.idle":"2024-12-21T14:20:30.232416Z","shell.execute_reply.started":"2024-12-21T14:20:30.222651Z","shell.execute_reply":"2024-12-21T14:20:30.231096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.233833Z","iopub.execute_input":"2024-12-21T14:20:30.234186Z","iopub.status.idle":"2024-12-21T14:20:30.28724Z","shell.execute_reply.started":"2024-12-21T14:20:30.234155Z","shell.execute_reply":"2024-12-21T14:20:30.285996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols = [col for col in data_train.columns if data_train[col].dtype == 'object']\nfeature_cols = list(data_test.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.288727Z","iopub.execute_input":"2024-12-21T14:20:30.2891Z","iopub.status.idle":"2024-12-21T14:20:30.297016Z","shell.execute_reply.started":"2024-12-21T14:20:30.289066Z","shell.execute_reply":"2024-12-21T14:20:30.295436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CategoricalEncoder:\n    def __init__(self, train, test):\n        self.train = train\n        self.test = test\n\n    def frequency_encode(self, cat_cols, feature_cols, drop_org=False):\n        combined = pd.concat([self.train, self.test], axis=0, ignore_index=True)\n\n        new_cat_cols = [] \n        for col in cat_cols:\n            freq_encoding = combined[col].value_counts().to_dict()\n            \n            self.train[f\"{col}_freq\"] = self.train[col].map(freq_encoding).astype('float')\n            self.test[f\"{col}_freq\"] = self.test[col].map(freq_encoding).astype('float')\n\n            new_col_name = f\"{col}_freq\"\n            new_cat_cols.append(new_col_name)\n            feature_cols.append(new_col_name)\n            if drop_org:\n                feature_cols.remove(col)\n\n        return self.train, self.test, new_cat_cols, feature_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.298839Z","iopub.execute_input":"2024-12-21T14:20:30.299456Z","iopub.status.idle":"2024-12-21T14:20:30.312081Z","shell.execute_reply.started":"2024-12-21T14:20:30.299413Z","shell.execute_reply":"2024-12-21T14:20:30.310751Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_cols","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.313497Z","iopub.execute_input":"2024-12-21T14:20:30.313859Z","iopub.status.idle":"2024-12-21T14:20:30.326596Z","shell.execute_reply.started":"2024-12-21T14:20:30.313825Z","shell.execute_reply":"2024-12-21T14:20:30.325437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder = CategoricalEncoder(data_train, data_test)\ntrain, test, cat_cols, feature_cols = encoder.frequency_encode(cat_cols, feature_cols, drop_org=True)\n\ndata_train = train[feature_cols]\ndata_test = test[feature_cols]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:30.328185Z","iopub.execute_input":"2024-12-21T14:20:30.328541Z","iopub.status.idle":"2024-12-21T14:20:34.833201Z","shell.execute_reply.started":"2024-12-21T14:20:30.328508Z","shell.execute_reply":"2024-12-21T14:20:34.832033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:34.834338Z","iopub.execute_input":"2024-12-21T14:20:34.834647Z","iopub.status.idle":"2024-12-21T14:20:34.895494Z","shell.execute_reply.started":"2024-12-21T14:20:34.834618Z","shell.execute_reply":"2024-12-21T14:20:34.89405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = data_train.drop(['Premium Amount'], axis=1)\ny = data_train['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:34.896899Z","iopub.execute_input":"2024-12-21T14:20:34.897237Z","iopub.status.idle":"2024-12-21T14:20:35.444328Z","shell.execute_reply.started":"2024-12-21T14:20:34.897205Z","shell.execute_reply":"2024-12-21T14:20:35.443089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TrainCAT(params, n_splits, SEED):\n     kfold = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=SEED)\n     train_rmse_scores = []\n     val_rmse_scores = []\n     models = []\n     oof = np.zeros(len(X))\n\n     for fold, (train_idx, val_idx) in enumerate(tqdm(kfold.split(X, y), desc=\"Training Folds\", total=n_splits)):\n         X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n         y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n         print(f\"-------- Woking on fold {fold} --------\")\n         \n         y_train_log = np.log1p(y_train)\n         y_val_log = np.log1p(y_val)\n        \n         model = CatBoostRegressor(**params, random_state=SEED, verbose=400, eval_metric='RMSE', task_type='GPU')\n         model.fit(X_train, y_train_log, \n                   eval_set=[(X_val, y_val_log)], \n                   early_stopping_rounds=300,\n                   #cat_features=feature_cols\n                  )\n         \n         models.append(model)\n         \n         y_train_log_pred = model.predict(X_train)\n         y_val_log_pred = model.predict(X_val)\n        \n         y_train_pred = np.expm1(y_train_log_pred)\n         y_val_pred = np.expm1(y_val_log_pred)\n\n         y_train_pred = np.clip(y_train_pred, 20, 4999)\n         y_val_pred = np.clip(y_val_pred, 20, 4999)\n         \n         train_rmse = np.sqrt(mean_squared_log_error(y_train, y_train_pred))\n         val_rmse = np.sqrt(mean_squared_log_error(y_val, y_val_pred))\n\n         train_rmse_scores.append(train_rmse)\n         val_rmse_scores.append(val_rmse)\n         \n         oof[val_idx] = y_val_pred\n         \n         print(f\"\\n Scores Fold: {fold}\")\n         print(f\"Train RMSLE: {train_rmse:.4f}\")\n         print(f\"Validation RMSLE: {val_rmse:.4f}\")\n    \n     mean_train_rmse = np.mean(train_rmse_scores)\n     mean_val_rmse = np.mean(val_rmse_scores)\n     print(\"\\n Final Mean Scores:\")\n     print(f\"Mean Train RMSLE: {mean_train_rmse:.4f}\")\n     print(f\"Mean Validation RMSLE: {mean_val_rmse:.4f}\")\n\n     return mean_train_rmse, mean_val_rmse, models, oof","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:35.445754Z","iopub.execute_input":"2024-12-21T14:20:35.446153Z","iopub.status.idle":"2024-12-21T14:20:35.457099Z","shell.execute_reply.started":"2024-12-21T14:20:35.446121Z","shell.execute_reply":"2024-12-21T14:20:35.455948Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objectiveCAT(trial):\n  params = {\n        \"learning_rate\": trial.suggest_loguniform(\"learning_rate\", 1e-4, 1e-1),\n        \"iterations\": trial.suggest_int(\"iterations\", 100, 3000),\n        \"max_depth\": trial.suggest_int(\"max_depth\", 4, 15),\n        #\"colsample_bylevel\": trial.suggest_uniform(\"colsample_bylevel\", 0.5, 1.0),\n        \"l2_leaf_reg\": trial.suggest_loguniform(\"l2_leaf_reg\", 1e-3, 10.0),\n        #\"random_strength\": trial.suggest_uniform(\"random_strength\", 0.1, 1.0),\n    }\n\n  SEED = 42\n  n_splits = 5\n\n  mean_train_rmse, mean_val_rmse, models, oof = TrainCAT(params, n_splits, SEED)\n  trial.set_user_attr('train_rmse', mean_train_rmse)\n  trial.set_user_attr('val_rmse', mean_val_rmse)\n  return mean_val_rmse","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:35.458938Z","iopub.execute_input":"2024-12-21T14:20:35.459269Z","iopub.status.idle":"2024-12-21T14:20:35.479209Z","shell.execute_reply.started":"2024-12-21T14:20:35.459238Z","shell.execute_reply":"2024-12-21T14:20:35.477805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"RETUNE_CAT = False  \n\ncat_params = {'learning_rate': 0.003827706197381081, 'iterations': 2886, 'max_depth': 13, 'l2_leaf_reg': 0.009320279216092816}\n#Trial 10 finished with value: 1.03422577475209 and Mean Train RMSLE: 1.0308 and parameters: {'learning_rate': 0.001017472258778599, 'iterations': 2872, 'max_depth': 13, 'l2_leaf_reg': 6.7793282802205335}.\n#Trial 12 finished with value: 1.034004921795855 and Mean Train RMSLE: 1.0313 and parameters: {'learning_rate': 0.01084256335097527, 'iterations': 744, 'max_depth': 9, 'l2_leaf_reg': 1.0863882746073976}\n#Trial 0 finished with value: 1.0333455167344339 and Mean Train RMSLE: 1.0179 and parameters: {'learning_rate': 0.00788523477755652, 'iterations': 1671, 'max_depth': 11, 'l2_leaf_reg': 1.1562877233639617}. Best is trial 0 with value: 1.0333455167344339.\n#Trial 14 finished with value: 1.0333420205236927 and Mean Train RMSLE: 1.0123 and parameters: {'learning_rate': 0.014246704153134504, 'iterations': 820, 'max_depth': 12, 'l2_leaf_reg': 0.7028954226047814}\n#Trial 18 finished with value: 1.033262731188275 and Mean Train RMSLE: 0.9986 and parameters: {'learning_rate': 0.003827706197381081, 'iterations': 2886, 'max_depth': 13, 'l2_leaf_reg': 0.009320279216092816}\n\nif RETUNE_CAT:\n    study = optuna.create_study(direction='minimize')\n    study.enqueue_trial(cat_params)\n    study.optimize(objectiveCAT, n_trials=50,)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:35.480557Z","iopub.execute_input":"2024-12-21T14:20:35.480986Z","iopub.status.idle":"2024-12-21T14:20:35.493467Z","shell.execute_reply.started":"2024-12-21T14:20:35.480951Z","shell.execute_reply":"2024-12-21T14:20:35.492197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nn_splits = 5\nmean_train_rmse, mean_val_rmse, models, oof = TrainCAT(cat_params, n_splits, SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:35.494919Z","iopub.execute_input":"2024-12-21T14:20:35.495357Z","iopub.status.idle":"2024-12-21T14:20:36.986217Z","shell.execute_reply.started":"2024-12-21T14:20:35.495311Z","shell.execute_reply":"2024-12-21T14:20:36.984659Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for model in models:\n    # Obtener la importancia de las características\n    feature_importances = model.get_feature_importance(prettified=True)\n\n    # Mostrar las importancias de las características\n    print(feature_importances)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:36.987205Z","iopub.status.idle":"2024-12-21T14:20:36.987573Z","shell.execute_reply.started":"2024-12-21T14:20:36.987396Z","shell.execute_reply":"2024-12-21T14:20:36.987413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions = np.zeros(len(data_test))\n\nfor model in models:\n    test_predictions += np.clip(np.expm1(model.predict(data_test)), 20, 4999) / len(models)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:36.990303Z","iopub.status.idle":"2024-12-21T14:20:36.990786Z","shell.execute_reply.started":"2024-12-21T14:20:36.990587Z","shell.execute_reply":"2024-12-21T14:20:36.990611Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample['Premium Amount'] = test_predictions\nsample.to_csv('CATBOOST_ENSEMBLE_V2_BEST_TRIAL_CV_5_SPLITS.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:36.992205Z","iopub.status.idle":"2024-12-21T14:20:36.992573Z","shell.execute_reply.started":"2024-12-21T14:20:36.992399Z","shell.execute_reply":"2024-12-21T14:20:36.992416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-21T14:20:36.995286Z","iopub.status.idle":"2024-12-21T14:20:36.995902Z","shell.execute_reply.started":"2024-12-21T14:20:36.995583Z","shell.execute_reply":"2024-12-21T14:20:36.995613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"joblib.dump([oof,test_predictions],\"CATBOOST_ENSEMBLE_V2_BEST_TRIAL_CV_5_SPLITS.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T03:12:13.761004Z","iopub.execute_input":"2024-12-17T03:12:13.761342Z","iopub.status.idle":"2024-12-17T03:12:13.78291Z","shell.execute_reply.started":"2024-12-17T03:12:13.761311Z","shell.execute_reply":"2024-12-17T03:12:13.782183Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ENSEMBLE CATBOOST: OUTPUT WITH 1 SPLITS","metadata":{}},{"cell_type":"code","source":"y_train_log = np.log1p(y)\n\n        \nmodel = CatBoostRegressor(**cat_params, random_state=SEED, verbose=400, eval_metric='RMSE', task_type='GPU')\nmodel.fit(X, y_train_log, early_stopping_rounds=300)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:24:42.721904Z","iopub.execute_input":"2024-12-17T15:24:42.722245Z","iopub.status.idle":"2024-12-17T15:26:55.612726Z","shell.execute_reply.started":"2024-12-17T15:24:42.722213Z","shell.execute_reply":"2024-12-17T15:26:55.611805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions = np.zeros(len(data_test))\ntest_predictions += np.clip(np.expm1(model.predict(data_test)), 20, 4999)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:26:59.610809Z","iopub.execute_input":"2024-12-17T15:26:59.611121Z","iopub.status.idle":"2024-12-17T15:27:07.441149Z","shell.execute_reply.started":"2024-12-17T15:26:59.611095Z","shell.execute_reply":"2024-12-17T15:27:07.440191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample['Premium Amount'] = test_predictions\nsample.to_csv('CATBOOST_ENSEMBLE_V2_BEST_TRIAL_CV_1_SPLITS.csv', index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:27:10.141049Z","iopub.execute_input":"2024-12-17T15:27:10.141895Z","iopub.status.idle":"2024-12-17T15:27:11.552228Z","shell.execute_reply.started":"2024-12-17T15:27:10.141861Z","shell.execute_reply":"2024-12-17T15:27:11.551541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:27:12.312427Z","iopub.execute_input":"2024-12-17T15:27:12.313068Z","iopub.status.idle":"2024-12-17T15:27:12.322478Z","shell.execute_reply.started":"2024-12-17T15:27:12.313032Z","shell.execute_reply":"2024-12-17T15:27:12.321385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"joblib.dump([oof,test_predictions],\"CATBOOST_ENSEMBLE_V2_BEST_TRIAL_CV_1_SPLITS.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-17T15:29:23.686918Z","iopub.execute_input":"2024-12-17T15:29:23.68778Z","iopub.status.idle":"2024-12-17T15:29:23.708853Z","shell.execute_reply.started":"2024-12-17T15:29:23.687743Z","shell.execute_reply":"2024-12-17T15:29:23.708056Z"}},"outputs":[],"execution_count":null}]}