{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#pip install pingouin\n#pip install statstests","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:06:31.106881Z","iopub.execute_input":"2025-01-04T20:06:31.107359Z","iopub.status.idle":"2025-01-04T20:06:35.400206Z","shell.execute_reply.started":"2025-01-04T20:06:31.107323Z","shell.execute_reply":"2025-01-04T20:06:35.398874Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% FRAMEWORKS\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pingouin as pg\nfrom statsmodels.api import OLS\nfrom scipy.stats import zscore\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV\nfrom sklearn.tree import DecisionTreeRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_log_error, make_scorer\nfrom statstests.process import stepwise\nfrom statstests.tests import shapiro_francia\nfrom scipy.stats import boxcox\nfrom datetime import datetime\nfrom itertools import product\nimport time as tt\nimport warnings as ww\nww.filterwarnings(\"ignore\")\npd.options.display.float_format = '{:.2f}'.format\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:25:04.393696Z","iopub.execute_input":"2025-01-04T20:25:04.394135Z","iopub.status.idle":"2025-01-04T20:25:04.407168Z","shell.execute_reply.started":"2025-01-04T20:25:04.394104Z","shell.execute_reply":"2025-01-04T20:25:04.40573Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\", index_col=\"id\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\", index_col=\"id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:13:24.080201Z","iopub.execute_input":"2025-01-04T20:13:24.080604Z","iopub.status.idle":"2025-01-04T20:13:33.109492Z","shell.execute_reply.started":"2025-01-04T20:13:24.080573Z","shell.execute_reply":"2025-01-04T20:13:33.108358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% FUNCTIONS\n\n# RMSE\ndef root_mean_squared_log_error(y_true, y_pred):\n   \n    # Adiciona uma pequena constante (1e-10) para evitar log(0)\n    epsilon = 1e-10\n    y_true = np.maximum(y_true, epsilon)  # Evita log de 0\n    y_pred = np.maximum(y_pred, epsilon)  # Evita log de 0\n\n    # Calcular o erro logarítmico quadrático\n    log_error = np.log(y_pred + 1) - np.log(y_true + 1)\n    \n    return np.sqrt(np.mean(np.square(log_error)))\n\n# FUNCTION TO INVERSE BOXCOX\ndef inverse_boxcox(y, lamb):\n    if lamb == 0:\n        return np.exp(y)  # Caso especial: lambda = 0\n    else:\n        return np.power(y * lamb + 1, 1 / lamb)\n    \n# FUNCTION TO INVERSE LOG1P\ndef inverse_log1p(y):\n    return np.expm1(y)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:09:44.085614Z","iopub.execute_input":"2025-01-04T20:09:44.086075Z","iopub.status.idle":"2025-01-04T20:09:44.093426Z","shell.execute_reply.started":"2025-01-04T20:09:44.08604Z","shell.execute_reply":"2025-01-04T20:09:44.091841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% INFOS\nprint(train.info(), \"\\n\")\nprint(train.isna().mean().sort_values(ascending=False), \"\\n\")\nprint(train.describe().T)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:14:50.543759Z","iopub.execute_input":"2025-01-04T20:14:50.544171Z","iopub.status.idle":"2025-01-04T20:14:52.463992Z","shell.execute_reply.started":"2025-01-04T20:14:50.544143Z","shell.execute_reply":"2025-01-04T20:14:52.462464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% COLUMNS ADJUSTING\ntrain.columns = train.columns.str.replace(\"'s\", \"\", regex=False)\ntest.columns = test.columns.str.replace(\"'s\", \"\", regex=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:15:14.742592Z","iopub.execute_input":"2025-01-04T20:15:14.74304Z","iopub.status.idle":"2025-01-04T20:15:14.751013Z","shell.execute_reply.started":"2025-01-04T20:15:14.743007Z","shell.execute_reply":"2025-01-04T20:15:14.748473Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% ANALISYS\nplt.figure(figsize=(10,7))\ntrain.hist(grid=False, xlabelsize=6, ylabelsize=6, bins=20)\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:15:17.717676Z","iopub.execute_input":"2025-01-04T20:15:17.718106Z","iopub.status.idle":"2025-01-04T20:15:19.86972Z","shell.execute_reply.started":"2025-01-04T20:15:17.718067Z","shell.execute_reply":"2025-01-04T20:15:19.868406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% INPUT MEDIAN IN NAN VALUES\ncols_quanti = train.select_dtypes(include=\"number\").columns\ncols_quanti2 = test.select_dtypes(include=\"number\").columns\ncols_qualy = train.select_dtypes(exclude=\"number\").columns\ntrain[cols_quanti] = train[cols_quanti].apply(lambda col: col.fillna(col.median()), axis=0)\ntest[cols_quanti2] = test[cols_quanti2].apply(lambda col: col.fillna(col.median()), axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:15:38.340883Z","iopub.execute_input":"2025-01-04T20:15:38.34127Z","iopub.status.idle":"2025-01-04T20:15:39.097538Z","shell.execute_reply.started":"2025-01-04T20:15:38.341241Z","shell.execute_reply":"2025-01-04T20:15:39.096316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% POLICY START DATE\ndef days_difference(train, test):\n    \n    reference_date = datetime.now()\n    \n    train[\"Policy Start Date\"] = train[\"Policy Start Date\"].str.split(\" \").str[0]\n    train[\"Policy Start Date\"] = pd.to_datetime(train[\"Policy Start Date\"], format='%Y-%m-%d', dayfirst=True)\n    train[\"Days\"] = (reference_date - train[\"Policy Start Date\"]).dt.days\n    train.drop(columns=\"Policy Start Date\", inplace=True)\n    \n    test[\"Policy Start Date\"] = test[\"Policy Start Date\"].str.split(\" \").str[0]\n    test[\"Policy Start Date\"] = pd.to_datetime(test[\"Policy Start Date\"], format='%Y-%m-%d', dayfirst=True)\n    test[\"Days\"] = (reference_date - test[\"Policy Start Date\"]).dt.days\n    test.drop(columns=\"Policy Start Date\", inplace=True)\n\ndays_difference(train, test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:16:02.403915Z","iopub.execute_input":"2025-01-04T20:16:02.404317Z","iopub.status.idle":"2025-01-04T20:16:08.170108Z","shell.execute_reply.started":"2025-01-04T20:16:02.404286Z","shell.execute_reply":"2025-01-04T20:16:08.169068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% DUMMIN THE QUALY FEATURES\ncols_qualy = train.select_dtypes(exclude=\"number\").columns\ntrain = pd.get_dummies(train, columns=cols_qualy, dtype=\"int\")\ntest = pd.get_dummies(test, columns=cols_qualy, dtype=\"int\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:16:40.536871Z","iopub.execute_input":"2025-01-04T20:16:40.537332Z","iopub.status.idle":"2025-01-04T20:16:43.368476Z","shell.execute_reply.started":"2025-01-04T20:16:40.537298Z","shell.execute_reply":"2025-01-04T20:16:43.367269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% CORRELATIONS ANALISYS\npg.rcorr(train, \n         method=\"pearson\",\n         upper='pval',\n         decimals=4,\n         pval_stars={0.01:'***',\n                     0.05:'**',\n                     0.10:'*'})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:17:03.470816Z","iopub.execute_input":"2025-01-04T20:17:03.471356Z","iopub.status.idle":"2025-01-04T20:17:36.803472Z","shell.execute_reply.started":"2025-01-04T20:17:03.471316Z","shell.execute_reply":"2025-01-04T20:17:36.800837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% OUTLIERS W/ ZSCORES\nsns.histplot(train[\"Premium Amount\"], kde=True)\n\ntrain_zscores = train.apply(zscore)\noutliers = (train_zscores.abs() > 3).any(axis=1)\n\nprint(f\"outliers detected: {len(train[outliers]) / len(train) * 100:.2f}%\")\n\ntrain = train[~outliers]\n\ntrain[[\"Annual Income\",\"Previous Claims\",\"Premium Amount\"]].describe().T\n\nplt.figure(figsize=(10,7))\nsns.boxplot(zscore(train[cols_quanti]), palette=\"viridis\")\nplt.xticks(rotation=75)\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:18:04.717221Z","iopub.execute_input":"2025-01-04T20:18:04.717676Z","iopub.status.idle":"2025-01-04T20:18:13.755525Z","shell.execute_reply.started":"2025-01-04T20:18:04.717641Z","shell.execute_reply":"2025-01-04T20:18:13.754408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% COLUMNS REGEX TO DISCART PROBLEMS W/ OLS MODELS\ntrain.columns = train.columns.str.replace(r\"[^A-Za-z0-9_]\", \"_\", regex=True).str.strip()\ntest.columns = test.columns.str.replace(r\"[^A-Za-z0-9_]\", \"_\", regex=True).str.strip()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:18:40.270652Z","iopub.execute_input":"2025-01-04T20:18:40.271059Z","iopub.status.idle":"2025-01-04T20:18:40.277353Z","shell.execute_reply.started":"2025-01-04T20:18:40.27103Z","shell.execute_reply":"2025-01-04T20:18:40.276074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% BOXCOX TO NORMALIZE THE TARGET\ntrain2 = train.copy()\ntransformed_data, lambdA = boxcox(train[\"Premium_Amount\"])\ntrain2[\"Premium_Amount_Box\"] = transformed_data","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:18:53.069007Z","iopub.execute_input":"2025-01-04T20:18:53.069396Z","iopub.status.idle":"2025-01-04T20:18:58.581447Z","shell.execute_reply.started":"2025-01-04T20:18:53.069365Z","shell.execute_reply":"2025-01-04T20:18:58.580423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% FEATURES AND OLS MODELS ANALISYS\nfeatures = train2.drop(columns=[\"Premium_Amount\",\"Premium_Amount_Box\"]).columns\nfeatures = \" + \".join(features)\nform1 = \"Premium_Amount ~ \" + features\nform2 = \"Premium_Amount_Box ~ \" + features\n\nmodelOls1 = OLS.from_formula(formula=form1, data=train2).fit()\nmodelOls2 = OLS.from_formula(formula=form2, data=train2).fit()\n\nprint(modelOls1.summary())\nprint(modelOls2.summary())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:19:38.328666Z","iopub.execute_input":"2025-01-04T20:19:38.329086Z","iopub.status.idle":"2025-01-04T20:19:51.938425Z","shell.execute_reply.started":"2025-01-04T20:19:38.329054Z","shell.execute_reply":"2025-01-04T20:19:51.935377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %% STEPWISE\nmodel_stepwise = stepwise(modelOls2)\n\n# ATRIBUTES DISCARTED AFTER STEPWISE PROCESS\nfeatures_discarted = [\"Insurance_Duration\",\"Number_of_Dependents\",\"Vehicle_Age\"]\n\ntrain2.drop(columns=features_discarted, inplace=True)\ntest.drop(columns=features_discarted, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:20:31.452878Z","iopub.execute_input":"2025-01-04T20:20:31.453262Z","iopub.status.idle":"2025-01-04T20:20:58.192118Z","shell.execute_reply.started":"2025-01-04T20:20:31.453233Z","shell.execute_reply":"2025-01-04T20:20:58.191178Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% DATA SPLIT TO TRAINING MODELS\nX = train2.drop(columns=[\"Premium_Amount_Box\", \"Premium_Amount\"])\ny = np.log1p(train2[\"Premium_Amount_Box\"])\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=88)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:21:04.726265Z","iopub.execute_input":"2025-01-04T20:21:04.726633Z","iopub.status.idle":"2025-01-04T20:21:05.396385Z","shell.execute_reply.started":"2025-01-04T20:21:04.7266Z","shell.execute_reply":"2025-01-04T20:21:05.395219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% DECISIONTREE\n\n# params\nparam_grid = {\n    'max_depth': [5, 10],  \n    'min_samples_split': [2, 5], \n    'min_samples_leaf': [1, 2], \n    'criterion': ['squared_error']} \n\n# modelin\nmodel = DecisionTreeRegressor(random_state=88)\n\n# gridSearchCV\nn_iter_search = 12\nrmsle_scorer = make_scorer(root_mean_squared_log_error, greater_is_better=False) \ngrid_search = RandomizedSearchCV(estimator=model, \n                                 param_distributions=param_grid, \n                                 n_iter=n_iter_search, \n                                 cv=5, \n                                 scoring=rmsle_scorer, \n                                 n_jobs=-1, \n                                 random_state=99)\n\n# training\ngrid_search.fit(X_train, y_train)\n\n# best params\nprint(\"Best params of gs tree:\", grid_search.best_params_)\n\n# evaluating the model\ny_pred = grid_search.predict(X_test)\nrmsle_value = root_mean_squared_log_error(y_test, y_pred)\nprint(\"Raiz Quadrada do Erro Logarítmico Médio (RMSLE) no teste:\", rmsle_value)\n\n# predictions to sub\ntest[\"Premium Amount\"] = grid_search.predict(test)\ntest[\"Premium Amount\"] = inverse_log1p(test[\"Premium Amount\"])\ntest[\"Premium Amount\"] = inverse_boxcox(test[\"Premium Amount\"], lamb=lambdA)\nsubmission = test[\"Premium Amount\"].reset_index()\nsubmission.to_csv(\"/kaggle/working/sub_tree.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:21:36.361143Z","iopub.execute_input":"2025-01-04T20:21:36.361545Z","iopub.status.idle":"2025-01-04T20:23:20.105745Z","shell.execute_reply.started":"2025-01-04T20:21:36.361512Z","shell.execute_reply":"2025-01-04T20:23:20.104505Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#%% XGBOOSTREGRESSOR\n\n# params\nparam_grid = {\n    'n_estimators': [100, 200, 300],        # Número de árvores\n    'learning_rate': [0.01, 0.05, 0.1],     # Taxa de aprendizado\n    'max_depth': [3, 5, 7],                 # Profundidade máxima das árvores\n    'subsample': [0.8, 1.0],                # Amostragem de observações\n    'colsample_bytree': [0.8, 1.0],         # Amostragem de colunas\n}\n\nparam_combinations = list(product(*param_grid.values()))\nbest_params = None\nbest_rmsle = float('inf')\n\n# Loop in params\nfor params in param_combinations:   \n    model = XGBRegressor(\n        n_estimators=params[0],\n        learning_rate=params[1],\n        max_depth=params[2],\n        subsample=params[3],\n        colsample_bytree=params[4],\n        random_state=88\n    )\n    \n    # model training\n    model.fit(X_train, y_train)\n    \n    # predicts\n    y_pred = model.predict(X_test)\n    \n    # RMSLE\n    rmsle = root_mean_squared_log_error(y_test, y_pred)\n    print(f\"Parâmetros: {params} -> RMSLE: {rmsle}\")\n    \n    # best params\n    if rmsle < best_rmsle:\n        best_rmsle = rmsle\n        best_params = params\n\n# results\nprint(\"\\nBest params:\")\nprint(f\"n_estimators: {best_params[0]}, learning_rate: {best_params[1]}, max_depth: {best_params[2]}, subsample: {best_params[3]}, colsample_bytree: {best_params[4]}\")\nprint(f\"Best RMSLE: {best_rmsle}\")\n\n# predictions to sub\ndel test[\"Premium Amount\"]\ntest[\"Premium Amount\"] = model.predict(test)\ntest[\"Premium Amount\"] = inverse_log1p(test[\"Premium Amount\"])\ntest[\"Premium Amount\"] = inverse_boxcox(test[\"Premium Amount\"], lambdA)\nsubmission = test[\"Premium Amount\"].reset_index()\nsubmission.to_csv(\"/kaggle/working/sub_xgboost.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:25:13.982993Z","iopub.execute_input":"2025-01-04T20:25:13.983421Z","iopub.status.idle":"2025-01-04T20:44:34.049533Z","shell.execute_reply.started":"2025-01-04T20:25:13.98339Z","shell.execute_reply":"2025-01-04T20:44:34.047766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# %% LIGHTGBM\n\n# params\nparam_grid = {\n    'n_estimators': [100, 200],   # Número de árvores\n    'max_depth': [5, 10],          # Profundidade máxima\n    'learning_rate': [0.01, 0.1],  # Taxa de aprendizado\n    'num_leaves': [31, 50],        # Número de folhas por árvore\n    'colsample_bytree': [0.8, 1.0] # Amostragem de colunas\n}\n\n# LightGBM\nmodel = LGBMRegressor(random_state=88)\n\n# RandomizedSearchCV\nrmsle_scorer = make_scorer(root_mean_squared_log_error, greater_is_better=False)\ngrid_search = RandomizedSearchCV(\n    estimator=model, \n    param_distributions=param_grid, \n    n_iter=12, \n    cv=5, \n    scoring=rmsle_scorer, \n    n_jobs=-1, \n    random_state=99\n)\n\n# training\ngrid_search.fit(X_train, y_train)\n\n# Melhores parâmetros\nprint(\"Melhores parâmetros:\", grid_search.best_params_)\n\n# evaluating the model\ny_pred = grid_search.predict(X_test)\nrmsle_value = root_mean_squared_log_error(y_test, y_pred)\nprint(\"RMSLE no teste:\", rmsle_value)\n\n# predictions to sub\ndel test[\"Premium Amount\"]\ntest[\"Premium Amount\"] = grid_search.predict(test)\ntest[\"Premium Amount\"] = inverse_log1p(test[\"Premium Amount\"])\ntest[\"Premium Amount\"] = inverse_boxcox(test[\"Premium Amount\"], lambdA)\nsubmission = test[\"Premium Amount\"].reset_index()\nsubmission.to_csv(\"/kaggle/working/sub_lightgbm.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-04T20:45:24.342743Z","iopub.execute_input":"2025-01-04T20:45:24.343212Z","iopub.status.idle":"2025-01-04T20:53:16.960103Z","shell.execute_reply.started":"2025-01-04T20:45:24.343178Z","shell.execute_reply":"2025-01-04T20:53:16.958569Z"}},"outputs":[],"execution_count":null}]}