{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:53:59.366837Z","iopub.execute_input":"2024-12-02T06:53:59.367894Z","iopub.status.idle":"2024-12-02T06:53:59.375376Z","shell.execute_reply.started":"2024-12-02T06:53:59.367823Z","shell.execute_reply":"2024-12-02T06:53:59.374611Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Loading the directory","metadata":{}},{"cell_type":"code","source":"base_file_dir = \"/kaggle/input/playground-series-s4e12/\"\n\ntrain_df = pd.read_csv(f\"{base_file_dir}/train.csv\")\ntest_df = pd.read_csv(f\"{base_file_dir}/test.csv\")\nsub_df = pd.read_csv(f\"{base_file_dir}/sample_submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:53:59.741224Z","iopub.execute_input":"2024-12-02T06:53:59.741607Z","iopub.status.idle":"2024-12-02T06:54:06.835016Z","shell.execute_reply.started":"2024-12-02T06:53:59.741574Z","shell.execute_reply":"2024-12-02T06:54:06.834175Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing Value","metadata":{}},{"cell_type":"code","source":"train_df = train_df.fillna(np.nan)\ntest_df = test_df.fillna(np.nan)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:06.836681Z","iopub.execute_input":"2024-12-02T06:54:06.837044Z","iopub.status.idle":"2024-12-02T06:54:08.554747Z","shell.execute_reply.started":"2024-12-02T06:54:06.837012Z","shell.execute_reply":"2024-12-02T06:54:08.55388Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Categorical Value Handling","metadata":{}},{"cell_type":"code","source":"train_df['Gender'].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:08.556145Z","iopub.execute_input":"2024-12-02T06:54:08.556454Z","iopub.status.idle":"2024-12-02T06:54:08.627344Z","shell.execute_reply.started":"2024-12-02T06:54:08.556423Z","shell.execute_reply":"2024-12-02T06:54:08.626275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.get_dummies(train_df, columns=['Gender','Marital Status','Education Level','Occupation','Location','Policy Type','Smoking Status','Exercise Frequency','Property Type','Customer Feedback'])\ntest_df = pd.get_dummies(test_df, columns=['Gender','Marital Status','Education Level','Occupation','Location','Policy Type','Smoking Status','Exercise Frequency','Property Type','Customer Feedback'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:08.630294Z","iopub.execute_input":"2024-12-02T06:54:08.630717Z","iopub.status.idle":"2024-12-02T06:54:10.529266Z","shell.execute_reply.started":"2024-12-02T06:54:08.630671Z","shell.execute_reply":"2024-12-02T06:54:10.52824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df['Policy Start Date'] = pd.to_datetime(train_df['Policy Start Date'])\ntest_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'])\n\ntrain_df['Year'] = train_df['Policy Start Date'].dt.year\ntrain_df['Month'] = train_df['Policy Start Date'].dt.month\ntrain_df['Day'] = train_df['Policy Start Date'].dt.day\ntrain_df['Hour'] = train_df['Policy Start Date'].dt.hour\ntrain_df['Minute'] = train_df['Policy Start Date'].dt.minute\ntrain_df['Weekday'] = train_df['Policy Start Date'].dt.weekday\n\ntest_df['Year'] = test_df['Policy Start Date'].dt.year\ntest_df['Month'] = test_df['Policy Start Date'].dt.month\ntest_df['Day'] = test_df['Policy Start Date'].dt.day\ntest_df['Hour'] = test_df['Policy Start Date'].dt.hour\ntest_df['Minute'] = test_df['Policy Start Date'].dt.minute\ntest_df['Weekday'] = test_df['Policy Start Date'].dt.weekday\n\n\ntrain_df = train_df.drop('Policy Start Date', axis=1)\ntest_df = test_df.drop('Policy Start Date', axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:10.530406Z","iopub.execute_input":"2024-12-02T06:54:10.530696Z","iopub.status.idle":"2024-12-02T06:54:11.983026Z","shell.execute_reply.started":"2024-12-02T06:54:10.530667Z","shell.execute_reply":"2024-12-02T06:54:11.982077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bool_columns = train_df.select_dtypes(include='bool').columns\ntrain_df[bool_columns] = train_df[bool_columns].astype(int)\n\nbool_columns = test_df.select_dtypes(include='bool').columns\ntest_df[bool_columns] = test_df[bool_columns].astype(int)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:11.985294Z","iopub.execute_input":"2024-12-02T06:54:11.985583Z","iopub.status.idle":"2024-12-02T06:54:12.288817Z","shell.execute_reply.started":"2024-12-02T06:54:11.985554Z","shell.execute_reply":"2024-12-02T06:54:12.287903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:12.290116Z","iopub.execute_input":"2024-12-02T06:54:12.290419Z","iopub.status.idle":"2024-12-02T06:54:12.343459Z","shell.execute_reply.started":"2024-12-02T06:54:12.290389Z","shell.execute_reply":"2024-12-02T06:54:12.342498Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train the model","metadata":{}},{"cell_type":"code","source":"# import xgboost as xgb\n# import numpy as np\n# from sklearn.model_selection import train_test_split, GridSearchCV\n# from sklearn.metrics import mean_squared_error, r2_score\n\n# X = train_df.drop(columns=['Premium Amount'])\n# y = train_df['Premium Amount']\n\n# X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# model = xgb.XGBRegressor(missing=np.nan, objective='reg:squarederror')\n\n# # Defin hyperparameter grid\n# param_grid = {\n#     'n_estimators': [100, 200, 300],\n#     'max_depth': [3, 6, 9],\n#     'learning_rate': [0.01, 0.05, 0.1],\n#     'subsample': [0.8, 1.0],\n#     'colsample_bytree': [0.8, 1.0],\n#     'gamma': [0, 0.1, 0.2],\n# }\n\n# # Perform Grid Search\n# grid_search = GridSearchCV(estimator=model, param_grid=param_grid, \n#                            scoring='neg_mean_squared_error', \n#                            cv=3, verbose=1, n_jobs=-1)\n\n# # Fit the model with hyperparameter tuning\n# grid_search.fit(X_train, y_train)\n\n# # Best model and parameters\n# best_model = grid_search.best_estimator_\n# best_params = grid_search.best_params_\n\n# print(f\"Best Hyperparameters: {best_params}\")\n\n# # Predict with the tuned model\n# y_pred = best_model.predict(X_val)\n\n# # Evaluate the model\n# mse = mean_squared_error(y_val, y_pred)\n# rmse = np.sqrt(mse)\n# r2 = r2_score(y_val, y_pred)\n\n# def rmsle(y_true, y_pred):\n#     return np.sqrt(np.mean((np.log1p(y_pred) - np.log1p(y_true))**2))\n\n# # Output the evaluation metrics\n# print(f\"Mean Squared Error: {mse}\")\n# print(f\"R² Score: {r2}\")\n# print(f\"Root Mean Squared Log Error: {rmsle(y_val, y_pred)}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T07:37:59.407367Z","iopub.execute_input":"2024-12-02T07:37:59.407767Z","iopub.status.idle":"2024-12-02T07:41:41.159239Z","shell.execute_reply.started":"2024-12-02T07:37:59.407723Z","shell.execute_reply":"2024-12-02T07:41:41.157561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%pip -q install optuna\nimport optuna\nimport xgboost as xgb\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error, r2_score\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)  # Suppress UserWarnings\n\n# Prepare your data\nX = train_df.drop(columns=['Premium Amount'])\ny = train_df['Premium Amount']\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Objective function for Optuna optimization\ndef objective(trial):\n    # Defining the hyperparameter search space\n    params = {\n        'n_estimators': trial.suggest_categorical('n_estimators', [100, 200, 300]),\n        'max_depth': trial.suggest_int('max_depth', 3, 9),\n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.1),\n        'subsample': trial.suggest_uniform('subsample', 0.8, 1.0),\n        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.8, 1.0),\n        'gamma': trial.suggest_uniform('gamma', 0, 0.2),\n        'objective': 'reg:squarederror',  # Regression objective\n        'missing': np.nan  # Handle missing values\n    }\n\n    # Initialize XGBoost model with suggested parameters\n    model = xgb.XGBRegressor(**params)\n    \n    # Fit the model\n    model.fit(X_train, y_train)\n    \n    # Predict the values\n    y_pred = model.predict(X_val)\n    \n    # Evaluate the model using Mean Squared Error (MSE)\n    mse = mean_squared_error(y_val, y_pred)\n    return mse  # Minimize MSE\n\n# Create a study object for Optuna and optimize the objective function\nstudy = optuna.create_study(direction='minimize')  # Minimize MSE\nstudy.optimize(objective, n_trials=50)  # Perform 50 trials\n\n# Get the best hyperparameters and best model\nbest_params = study.best_params\nprint(f\"Best Hyperparameters: {best_params}\")\n\n# Using the best hyperparameters to train the final model\nbest_model = xgb.XGBRegressor(**best_params)\nbest_model.fit(X_train, y_train)\n\n# Predict with the tuned model\ny_pred = best_model.predict(X_val)\n\n# Evaluate the model\nmse = mean_squared_error(y_val, y_pred)\nrmse = np.sqrt(mse)\nr2 = r2_score(y_val, y_pred)\n\n# Define RMSLE function\ndef rmsle(y_true, y_pred):\n    return np.sqrt(np.mean((np.log1p(y_pred) - np.log1p(y_true))**2))\n\n# Output the evaluation metrics\nprint(f\"Mean Squared Error: {mse}\")\nprint(f\"R² Score: {r2}\")\nprint(f\"Root Mean Squared Log Error: {rmsle(y_val, y_pred)}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGB:\nMean Squared Error: 707631.7254956178\\\nR² Score: 0.05306386388958062\\\nRoot Mean Squared Log Error: 1.1404765369442305\\","metadata":{}},{"cell_type":"code","source":"test_predictions = best_model.predict(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:54:37.565413Z","iopub.execute_input":"2024-12-02T06:54:37.565764Z","iopub.status.idle":"2024-12-02T06:54:38.726605Z","shell.execute_reply.started":"2024-12-02T06:54:37.565735Z","shell.execute_reply":"2024-12-02T06:54:38.725733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ids = test_df['id'] \n\nsubmission_df = pd.DataFrame({\n    'id': test_ids,              \n    'Premium Amount': test_predictions  \n})\n\n# Save the submission to a CSV file\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-02T06:56:20.492337Z","iopub.execute_input":"2024-12-02T06:56:20.492709Z","iopub.status.idle":"2024-12-02T06:56:21.82819Z","shell.execute_reply.started":"2024-12-02T06:56:20.492661Z","shell.execute_reply":"2024-12-02T06:56:21.826916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}