{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport xgboost as xgb\nimport optuna\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import (\n    mean_squared_error, \n    mean_absolute_error, \n    r2_score\n)\n\n# Suppress warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:12.14818Z","iopub.execute_input":"2024-12-16T11:32:12.148486Z","iopub.status.idle":"2024-12-16T11:32:13.150724Z","shell.execute_reply.started":"2024-12-16T11:32:12.148459Z","shell.execute_reply":"2024-12-16T11:32:13.150101Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:13.151972Z","iopub.execute_input":"2024-12-16T11:32:13.152317Z","iopub.status.idle":"2024-12-16T11:32:18.728745Z","shell.execute_reply.started":"2024-12-16T11:32:13.152291Z","shell.execute_reply":"2024-12-16T11:32:18.728035Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"# Feature engineering for date-based features\ndef prepare_date_features(df):\n    df['Policy Start Date'] = pd.to_datetime(df['Policy Start Date'])\n    \n    # Extract date-related features\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['Day'] = df['Policy Start Date'].dt.day\n    \n    # Create cyclic features for months and days\n    df['Month_sin'] = np.sin(2 * np.pi * df['Month'] / 12)\n    df['Month_cos'] = np.cos(2 * np.pi * df['Month'] / 12)\n    df['Day_sin'] = np.sin(2 * np.pi * df['Day'] / 31)\n    df['Day_cos'] = np.cos(2 * np.pi * df['Day'] / 31)\n    \n    df.drop('Policy Start Date', axis=1, inplace=True)\n    return df\n\n# Apply feature engineering\ntrain_df = prepare_date_features(train_df)\ntest_df = prepare_date_features(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:18.729664Z","iopub.execute_input":"2024-12-16T11:32:18.729905Z","iopub.status.idle":"2024-12-16T11:32:20.104247Z","shell.execute_reply.started":"2024-12-16T11:32:18.729881Z","shell.execute_reply":"2024-12-16T11:32:20.103511Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Target Distribution Before and After Log Transformation","metadata":{}},{"cell_type":"code","source":"# Compare Target Distribution Before and After Log Transformation\ndef compare_target_distribution(y, target_name=\"Target\"):\n    y_log = np.log1p(y)\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    sns.histplot(y, kde=True, bins=30, color=\"blue\")\n    plt.title(f\"Original Target Distribution: {target_name}\")\n    plt.subplot(1, 2, 2)\n    sns.histplot(y_log, kde=True, bins=30, color=\"green\")\n    plt.title(f\"Log-Transformed Target Distribution: log1p({target_name})\")\n    plt.tight_layout()\n    plt.show()\n\ny = train_df['Premium Amount']\ncompare_target_distribution(y, target_name=\"Premium Amount\")\n\n# Apply log transformation to the target\ny_log = np.log1p(y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:20.105187Z","iopub.execute_input":"2024-12-16T11:32:20.105453Z","iopub.status.idle":"2024-12-16T11:32:29.495333Z","shell.execute_reply.started":"2024-12-16T11:32:20.105407Z","shell.execute_reply":"2024-12-16T11:32:29.494485Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Numerical and categorical features","metadata":{}},{"cell_type":"code","source":"# Define numerical and categorical features\nnumerical_features = [\n    'Age', 'Annual Income', 'Number of Dependents', \n    'Health Score', 'Previous Claims', 'Vehicle Age', \n    'Credit Score', 'Insurance Duration', \n    'Month_sin', 'Month_cos', 'Day_sin', 'Day_cos'\n]\n\ncategorical_features = [\n    'Gender', 'Marital Status', 'Education Level', \n    'Occupation', 'Location', 'Policy Type', \n    'Customer Feedback', 'Smoking Status', \n    'Exercise Frequency', 'Property Type'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:29.497908Z","iopub.execute_input":"2024-12-16T11:32:29.498531Z","iopub.status.idle":"2024-12-16T11:32:29.503165Z","shell.execute_reply.started":"2024-12-16T11:32:29.498489Z","shell.execute_reply":"2024-12-16T11:32:29.502331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Mssing value","metadata":{}},{"cell_type":"code","source":"# Check missing values and categorical encoding before preprocessing\ndef check_data_before_preprocessing(df, numerical_features, categorical_features):\n    print(\"=== Missing Values BEFORE Preprocessing ===\")\n    print(df[numerical_features + categorical_features].isnull().sum())\n\n    print(\"\\n=== Data Types ===\")\n    print(df.dtypes)\n\ncheck_data_before_preprocessing(train_df, numerical_features, categorical_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:29.504312Z","iopub.execute_input":"2024-12-16T11:32:29.504643Z","iopub.status.idle":"2024-12-16T11:32:30.132714Z","shell.execute_reply.started":"2024-12-16T11:32:29.504606Z","shell.execute_reply":"2024-12-16T11:32:30.13184Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessing pipeline","metadata":{}},{"cell_type":"code","source":"# Preprocessing pipelines for numerical and categorical data\nnum_pipeline = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='median')),\n    ('scaler', StandardScaler())\n])\n\ncat_pipeline = Pipeline(steps=[\n    ('imputer', SimpleImputer(strategy='constant', fill_value='Unknown')),\n    ('onehot', OneHotEncoder(handle_unknown='ignore'))\n])\n\n# Combine preprocessing steps\npreprocessor = ColumnTransformer(\n    transformers=[\n        ('num', num_pipeline, numerical_features),\n        ('cat', cat_pipeline, categorical_features)\n    ])\n\n# Separate features\nX = train_df.drop(columns=['Premium Amount', 'id'])\nX_processed = preprocessor.fit_transform(X)\ntest_processed = preprocessor.transform(test_df.drop(columns=['id']))\n\n# Confirm transformed features\nprint(f\"Transformed dataset shape: {X_processed.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:30.133884Z","iopub.execute_input":"2024-12-16T11:32:30.134562Z","iopub.status.idle":"2024-12-16T11:32:39.518152Z","shell.execute_reply.started":"2024-12-16T11:32:30.13452Z","shell.execute_reply":"2024-12-16T11:32:39.517224Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Missing value after pipline","metadata":{}},{"cell_type":"code","source":"# Check transformed data after preprocessing\ndef check_data_after_preprocessing(preprocessor, X, numerical_features, categorical_features):\n    # Get transformed feature names\n    feature_names = (\n        preprocessor.named_transformers_['num'].get_feature_names_out(numerical_features).tolist() +\n        preprocessor.named_transformers_['cat'].named_steps['onehot'].get_feature_names_out(categorical_features).tolist()\n    )\n    \n    print(\"\\n Missing Values AFTER Preprocessing# Check if any missing values remain\n    \n    print(\"\\n Transformed Feature Names\")\n    print(feature_names)\n\ncheck_data_after_preprocessing(preprocessor, X_processed, numerical_features, categorical_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:39.519472Z","iopub.execute_input":"2024-12-16T11:32:39.519852Z","iopub.status.idle":"2024-12-16T11:32:39.635879Z","shell.execute_reply.started":"2024-12-16T11:32:39.519813Z","shell.execute_reply":"2024-12-16T11:32:39.634977Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train/val split","metadata":{}},{"cell_type":"code","source":"# Split the data into training and validation sets\nX_train, X_val, y_train_log, y_val_log = train_test_split(X_processed, y_log, test_size=0.2, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:39.636936Z","iopub.execute_input":"2024-12-16T11:32:39.637234Z","iopub.status.idle":"2024-12-16T11:32:39.873038Z","shell.execute_reply.started":"2024-12-16T11:32:39.637205Z","shell.execute_reply":"2024-12-16T11:32:39.872344Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hyperparameter optimization with Optuna","metadata":{}},{"cell_type":"code","source":"# Optuna Hyperparameter Optimization\ndef objective(trial):\n    params = {\n        'tree_method': 'gpu_hist',\n        'objective': 'reg:squaredlogerror',\n        'learning_rate': trial.suggest_loguniform('learning_rate', 0.01, 0.3),\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'subsample': trial.suggest_uniform('subsample', 0.5, 1.0),\n        'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.5, 1.0),\n        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),\n        'random_state': 42\n    }\n    model = xgb.XGBRegressor(**params)\n    model.fit(\n        X_train, y_train_log, \n        eval_set=[(X_val, y_val_log)],\n        early_stopping_rounds=50, verbose=False\n    )\n    y_pred_log = model.predict(X_val)\n    rmsle = np.sqrt(mean_squared_error(y_val_log, y_pred_log))\n    return rmsle\n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:32:39.87385Z","iopub.execute_input":"2024-12-16T11:32:39.874098Z","iopub.status.idle":"2024-12-16T11:40:05.307276Z","shell.execute_reply.started":"2024-12-16T11:32:39.874073Z","shell.execute_reply":"2024-12-16T11:40:05.306439Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Final model training","metadata":{}},{"cell_type":"code","source":"# Extract the best parameters\nbest_params = study.best_params\nbest_params['random_state'] = 42\n\n# Train the final model with the best parameters\nfinal_model = xgb.XGBRegressor(**best_params)\nfinal_model.fit(X_processed, y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:40:05.308241Z","iopub.execute_input":"2024-12-16T11:40:05.308501Z","iopub.status.idle":"2024-12-16T11:41:19.895544Z","shell.execute_reply.started":"2024-12-16T11:40:05.308475Z","shell.execute_reply":"2024-12-16T11:41:19.894666Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"code","source":"# Train Final Model\nbest_params = study.best_params\nfinal_model = xgb.XGBRegressor(**best_params)\nfinal_model.fit(X_processed, y_log)\n\n# Evaluate Model\ny_pred_log = final_model.predict(X_processed)\ny_pred = np.expm1(y_pred_log)  # Transform back to original scale\n\nprint(\"Model Metrics (log-transformed):\")\nprint(f\"RMSLE: {np.sqrt(mean_squared_error(y_log, y_pred_log)):.4f}\")\nprint(f\"RMSE: {np.sqrt(mean_squared_error(y_log, y_pred_log)):.4f}\")\nprint(f\"MAE: {mean_absolute_error(y_log, y_pred_log):.4f}\")\nprint(f\"R²: {r2_score(y_log, y_pred_log):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:41:19.896756Z","iopub.execute_input":"2024-12-16T11:41:19.897146Z","iopub.status.idle":"2024-12-16T11:42:50.014983Z","shell.execute_reply.started":"2024-12-16T11:41:19.897104Z","shell.execute_reply":"2024-12-16T11:42:50.014117Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature importance visualization","metadata":{}},{"cell_type":"code","source":"# Extract feature names\nfeature_names = (\n    preprocessor.named_transformers_['num'].get_feature_names_out(numerical_features).tolist() +\n    preprocessor.named_transformers_['cat'].named_steps['onehot'].get_feature_names_out(categorical_features).tolist()\n)\n\n# Plot feature importance\nplt.figure(figsize=(10, 6))\nfeature_importance = final_model.feature_importances_\nsorted_idx = np.argsort(feature_importance)\nplt.barh(range(len(sorted_idx)), feature_importance[sorted_idx])\nplt.yticks(range(len(sorted_idx)), [feature_names[i] for i in sorted_idx])\nplt.title(\"Feature Importance in XGBoost Model\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:42:50.015964Z","iopub.execute_input":"2024-12-16T11:42:50.016313Z","iopub.status.idle":"2024-12-16T11:42:50.654773Z","shell.execute_reply.started":"2024-12-16T11:42:50.016278Z","shell.execute_reply":"2024-12-16T11:42:50.653889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission preparation","metadata":{}},{"cell_type":"code","source":"# Submission Preparation\ntest_predictions_log = final_model.predict(test_processed)\ntest_predictions = np.expm1(test_predictions_log)\n\nsubmission = pd.DataFrame({\n    'id': test_df['id'], \n    'Premium Amount': test_predictions.round(3)\n})\nsubmission.to_csv('xgboost_submission_log1p.csv', index=False)\nprint(\"Submission file saved: 'xgboost_submission_log1p.csv'\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-16T11:42:50.656985Z","iopub.execute_input":"2024-12-16T11:42:50.6573Z","iopub.status.idle":"2024-12-16T11:43:02.099995Z","shell.execute_reply.started":"2024-12-16T11:42:50.657266Z","shell.execute_reply":"2024-12-16T11:43:02.099116Z"}},"outputs":[],"execution_count":null}]}