{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing Libs","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.ensemble import VotingRegressor\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder, OrdinalEncoder\n\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor, early_stopping\nfrom catboost import CatBoostRegressor\n\nimport optuna","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"random.seed(42)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Reading data","metadata":{}},{"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\ntest_data = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\ntrain_data = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Exploring data","metadata":{}},{"cell_type":"markdown","source":"Here we will be making 2 steps:  \n- Check data shape for both train and test data;\n- Change column names for lower and add underscore instead of space for better coding.","metadata":{}},{"cell_type":"code","source":"print('Train data:')\nprint(f'Columns: {train_data.shape[1]} | Rows: {train_data.shape[0]}')\nprint('-------------------------------')\nprint('Test data:')\nprint(f'Columns: {test_data.shape[1]} | Rows: {test_data.shape[0]}')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_cols = []\nfor col in train_data.columns:\n    new_cols.append(col.lower().replace(' ', '_'))\n\ntrain_data.columns = new_cols","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_cols = []\nfor col in test_data.columns:\n    new_cols.append(col.lower().replace(' ', '_'))\n\ntest_data.columns = new_cols","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Treating data","metadata":{}},{"cell_type":"markdown","source":"On treatment step, we will be making as follow:  \n- Fill the NaN values, for object with \"Unknown\" and for numeric with median;\n- Transform binary columns on binary data;\n- Transform categorical columns on dummy data;\n- Transform ordinal columns in int data;\n- Transform the datetime column in seconds to add spatial time to our regression.","metadata":{}},{"cell_type":"code","source":"train_data[train_data.select_dtypes(include=['number']).columns].describe().apply(lambda s: s.apply('{0:.5f}'.format))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fill_nan_columns(data):\n    numeric_columns = [\n        'age', 'annual_income', 'number_of_dependents', 'health_score',\n        'previous_claims', 'vehicle_age', 'credit_score', 'insurance_duration'\n    ]\n    for col in numeric_columns:\n        if col in data.columns:\n            data[col].fillna(data[col].median(), inplace=True)\n            data[col] = np.log1p(data[col])\n\n    object_columns = data.select_dtypes(include=['object']).columns\n    for col in object_columns:\n        if col in data.columns:\n            data[col].fillna(\"Unknown\", inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_new_features(df):\n    df['contract_length'] = pd.cut(\n        df['insurance_duration'].fillna(99),  \n        bins=[-float('inf'), 1, 3, float('inf')],  \n        labels=[0, 1, 2]\n    ).astype(int)\n\n    df['income_per_dependent'] = df['annual_income'] / [1 if x == 0 else x for x in df['number_of_dependents']]\n    df['credit_score_per_insurance_duration'] = df['credit_score'] / df['insurance_duration']\n    df['credit_per_health_score'] = df['credit_score'] / df['health_score']\n    df['credit_per_age'] = df['credit_score'] * df['age']\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"binary_columns = ['smoking_status']\ndummy_columns = ['gender', 'marital_status', 'occupation', 'location', 'property_type', 'education_level', 'policy_type', 'customer_feedback']\nordinal_columns = {\n    'exercise_frequency' : [\n        'Rarely',\n        'Monthly',\n        'Weekly',\n        'Daily'\n    ]\n}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_feature(train_data, test_data):\n    le = LabelEncoder()\n    for feature in binary_columns:\n        train_data[feature] = le.fit_transform(train_data[feature])\n        test_data[feature] = le.transform(test_data[feature])\n\n    for feature, order in ordinal_columns.items():\n        oe = OrdinalEncoder(categories=[order])\n        train_data[feature] = oe.fit_transform(train_data[[feature]]).flatten()\n        test_data[feature] = oe.transform(test_data[[feature]]).flatten()\n\n    train_data = pd.get_dummies(train_data, columns=dummy_columns, drop_first=True)\n    test_data = pd.get_dummies(test_data, columns=dummy_columns, drop_first=True)\n    return train_data, test_data","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def treat_datetime_columns(df):\n    df['policy_start_date'] = pd.to_datetime(df['policy_start_date'])\n    df['year'] = df['policy_start_date'].dt.year\n    df['day'] = df['policy_start_date'].dt.day\n    df['month'] = df['policy_start_date'].dt.month\n    df['week'] = df['policy_start_date'].dt.isocalendar().week\n    df['year_sin'] = np.sin(2 * np.pi * df['year'])\n    df['year_cos'] = np.cos(2 * np.pi * df['year'])\n    df['month_sin'] = np.sin(2 * np.pi * df['month'] / 12) \n    df['month_cos'] = np.cos(2 * np.pi * df['month'] / 12)\n    df['day_sin'] = np.sin(2 * np.pi * df['day'] / 31)  \n    df['day_cos'] = np.cos(2 * np.pi * df['day'] / 31)\n    df['group']=(df['year']-2020)*48+df['month']*4+df['day']//7\n    df['policy_start_date'] = df['policy_start_date'].astype(np.int64) / 10**9\n    return df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def treat_data(train_data, test_data):\n    df_train = train_data.copy()\n    df_test = test_data.copy()\n\n    fill_nan_columns(df_train)\n    fill_nan_columns(df_test)\n\n    df_train = add_new_features(df_train)\n    df_test = add_new_features(df_test)\n\n    encoded_train, encoded_test = encode_feature(df_train, df_test)\n\n    encoded_train = treat_datetime_columns(encoded_train)\n    encoded_test = treat_datetime_columns(encoded_test)\n    return encoded_train, encoded_test","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"treated_train_data, treated_test_data = treat_data(train_data, test_data)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"treated_train_data.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"treated_test_data.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ML","metadata":{}},{"cell_type":"markdown","source":"On the ML steps, we will train and test our models as such:  \n- The y will be transformed in log scale in order to better training;\n- We will be using optuna for hyperparameter tunning on LGM, XGBoost and CatBoost regressors;\n- We will be using KFold for model selection;","metadata":{}},{"cell_type":"code","source":"feature_df = treated_train_data.copy()\nfeature_df.drop('id', axis=1, inplace=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target = 'premium_amount'\n\nX = feature_df.drop(target, axis=1)\ny = feature_df[target]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_log = np.log1p(y) ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y_log, test_size=0.2, random_state=42)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef objective_lgbm(trial):\n    params = {\n        'boosting_type': 'gbdt',\n        'device': 'gpu',\n        'colsample_bytree':trial.suggest_float('colsample_bytree', 0.5, 1),\n        'learning_rate':trial.suggest_float('learning_rate', 0.01, 0.1),\n        'max_depth':trial.suggest_int('max_depth', 10, 25),\n        'min_child_samples':trial.suggest_int('min_child_samples', 70, 200),\n        'n_estimators':trial.suggest_int('n_estimators', 500, 2000),\n        'num_leaves':trial.suggest_int('num_leaves', 20, 200),\n        'reg_alpha':trial.suggest_float('reg_alpha', 1e-8, 100),\n        'reg_lambda':trial.suggest_float('reg_lambda', 1e-8, 500),\n        'subsample':trial.suggest_float('subsample', 0.5, 1),\n        'metric':'rmse',\n        'objective':'regression',\n        'verbose':-1,\n    }\n    \n    model = LGBMRegressor(\n        **params\n    )\n    \n    model.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_test, y_test)],\n        callbacks=[early_stopping(stopping_rounds=30, verbose=False)]\n    )\n    \n    y_pred = model.predict(X_test)\n    rmsle = mean_squared_log_error(np.expm1(y_test), np.expm1(y_pred), squared=False)\n    return rmsle\n\nstudy_lgbm = optuna.create_study(direction='minimize')\nstudy_lgbm.optimize(objective_lgbm, n_trials=50)\n\nprint(\"Best parameters:\", study_lgbm.best_params)\nprint(\"Best RMSLE:\", study_lgbm.best_value)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef objective_cat(trial):\n    params = {\n        'iterations': trial.suggest_int('iterations', 2000, 6000),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n        'depth': trial.suggest_int('depth', 4, 16),\n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-8, 5.0),\n        'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 1.0),\n        'task_type': 'GPU',\n        'devices': '0',\n        'eval_metric' : 'RMSE'\n    }\n    \n    model = CatBoostRegressor(\n        **params\n    )\n    \n    model.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_test, y_test)],\n        early_stopping_rounds=30,\n        verbose=False\n    )\n    \n    y_pred = model.predict(X_test)\n    rmsle = mean_squared_log_error(np.expm1(y_test), np.expm1(y_pred), squared=False)\n    return rmsle\n\nstudy_cat = optuna.create_study(direction='minimize')\nstudy_cat.optimize(objective_cat, n_trials=50)\n\nprint(\"Best parameters:\", study_cat.best_params)\nprint(\"Best RMSLE:\", study_cat.best_value)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef objective_xgb(trial):\n    params = {\n        'n_estimators': trial.suggest_int('n_estimators', 500, 3000),\n        'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.1),\n        'max_depth': trial.suggest_int('max_depth', 5, 18),\n        'min_child_weight': trial.suggest_int('min_child_weight', 5, 20),\n        'subsample': trial.suggest_float('subsample', 0.5, 1),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1),\n        'reg_alpha': trial.suggest_float('reg_alpha', 1, 20),\n        'reg_lambda': trial.suggest_float('reg_lambda', 1, 15),\n        'tree_method': 'hist',\n        'device': 'cuda',\n        'early_stopping_rounds':30\n    }\n    \n    model = XGBRegressor(\n        **params\n    )\n    \n    model.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_test, y_test)],\n        verbose=False\n    )\n    \n    y_pred = model.predict(X_test)\n    rmsle = mean_squared_log_error(np.expm1(y_test), np.expm1(y_pred), squared=False)\n    return rmsle\n\nstudy_xgb = optuna.create_study(direction='minimize')\nstudy_xgb.optimize(objective_xgb, n_trials=50)\n\nprint(\"Best parameters:\", study_xgb.best_params)\nprint(\"Best RMSLE:\", study_xgb.best_value)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Best params\n# lgbm_params = {\n#     'colsample_bytree': 0.9567079501977234, \n#     'learning_rate': 0.03097721111297455, \n#     'max_depth': 17, \n#     'min_child_samples': 112, \n#     'n_estimators': 1318, \n#     'num_leaves': 76, \n#     'reg_alpha': 6.282057237782206, \n#     'reg_lambda': 85.79210780843442, \n#     'subsample': 0.6422559055151422\n# }\n\n# cat_params = {\n#     'iterations': 3829, \n#     'learning_rate': 0.08007079092734848, \n#     'depth': 9, \n#     'l2_leaf_reg': 1.567404271751824, \n#     'bagging_temperature': 0.2317460985087944\n# }\n\n# xgb_params = {\n#     'n_estimators': 1001, \n#     'learning_rate': 0.022923784529572888, \n#     'max_depth': 8, \n#     'min_child_weight': 15, \n#     'subsample': 0.6702495107426214, \n#     'colsample_bytree': 0.9831382664215965, \n#     'reg_alpha': 12.244203187112658, \n#     'reg_lambda': 5.98033120501897\n# }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndef train_model():\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    oof = np.zeros(len(X))\n    models = []\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n        print(f\"Fold {fold + 1}\")\n        X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n        y_train, y_valid = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n\n        # LightGBM model\n        lgbm_model = LGBMRegressor(\n            **study_lgbm.best_params, \n            # **lgbm_params,\n            device='gpu', \n            gpu_platform_id=0, \n            gpu_device_id=0, \n            verbose=-1\n        )\n\n        # CatBoost model\n        cat_model = CatBoostRegressor(\n            **study_cat.best_params, \n            # **cat_params,\n            verbose=500, \n            task_type='GPU'\n        )\n\n        # XGBoost model\n        xgb_model = XGBRegressor(\n            **study_xgb.best_params, \n            # **xgb_params,\n            tree_method=\"hist\", \n            device='cuda'\n        )\n\n        # Voting Regressor\n        voting_model = VotingRegressor(\n            estimators=[\n                ('lgbm', lgbm_model),\n                ('cat', cat_model),\n                ('xgb', xgb_model)\n            ]\n        )\n\n        voting_model.fit(X_train, y_train)\n        oof[valid_idx] = np.maximum(0, voting_model.predict(X_valid))\n        fold_rmsle = mean_squared_log_error(np.expm1(y_valid), np.expm1(oof[valid_idx]), squared=False)\n        print(f\"Fold {fold + 1} RMSLE: {fold_rmsle}\")\n        models.append(voting_model)\n        \n    return models, oof\n\nmodels,oof = train_model()","metadata":{"trusted":true,"execution":{"execution_failed":"2024-12-28T21:18:36.84Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predicting and submiting","metadata":{}},{"cell_type":"markdown","source":"And, finally, predicting the better results based on models voting.","metadata":{}},{"cell_type":"code","source":"y_to_submit = np.zeros(len(treated_test_data))\n\nfor model in models:\n    y_to_submit += np.maximum(0, np.expm1(model.predict(treated_test_data.drop('id', axis=1)))) / len(models)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"to_submit = pd.DataFrame(\n    data={\n        'id' : treated_test_data['id'],\n        'Premium Amount' : y_to_submit\n    }\n)\n\nto_submit.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"to_submit.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}