{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"jupytext":{"cell_metadata_filter":"-all","main_language":"python","notebook_metadata_filter":"-all"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 📊🏆 Welcome ! 🎉","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:27:31.205076Z","iopub.execute_input":"2024-12-04T06:27:31.206109Z","iopub.status.idle":"2024-12-04T06:27:31.233049Z","shell.execute_reply.started":"2024-12-04T06:27:31.206068Z","shell.execute_reply":"2024-12-04T06:27:31.231978Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📥 Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install feature_engine\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# For model building\nimport optuna\nfrom sklearn.model_selection import RepeatedKFold\nfrom tqdm import tqdm\nimport lightgbm as lgb\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom datetime import datetime\nfrom feature_engine.creation import CyclicalFeatures\n\n# To ignore warnings\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(\"✅ Libraries imported successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:27:35.015184Z","iopub.execute_input":"2024-12-04T06:27:35.016116Z","iopub.status.idle":"2024-12-04T06:27:55.329835Z","shell.execute_reply.started":"2024-12-04T06:27:35.01605Z","shell.execute_reply":"2024-12-04T06:27:55.328412Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📂 Loading the Data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\nsample_submission = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\n\nprint(\"✅ Data loaded successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:28:22.333793Z","iopub.execute_input":"2024-12-04T06:28:22.334177Z","iopub.status.idle":"2024-12-04T06:28:30.952891Z","shell.execute_reply.started":"2024-12-04T06:28:22.334141Z","shell.execute_reply":"2024-12-04T06:28:30.95165Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔍 Exploring the Data","metadata":{}},{"cell_type":"code","source":"print(\"Training Data Overview:\")\ndisplay(train.head())\n\nprint(\"Test Data Overview:\")\ndisplay(test.head())\n\nprint(f\"Training data shape: {train.shape}\")\nprint(f\"Test data shape: {test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:29:12.563502Z","iopub.execute_input":"2024-12-04T06:29:12.563912Z","iopub.status.idle":"2024-12-04T06:29:12.630733Z","shell.execute_reply.started":"2024-12-04T06:29:12.563876Z","shell.execute_reply":"2024-12-04T06:29:12.629573Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🧹 Data Preprocessing 🛠️\n\nLet's clean up our data! 🧽","metadata":{}},{"cell_type":"code","source":"# Filling missing 'Age' with mean\nmean_age = train['Age'].mean()\ntrain['Age'] = train['Age'].fillna(mean_age)\ntest['Age'] = test['Age'].fillna(mean_age)\n\n# Filling missing 'Annual Income' with median\nmedian_annual_income = train['Annual Income'].median()\ntrain['Annual Income'] = train['Annual Income'].fillna(median_annual_income)\ntest['Annual Income'] = test['Annual Income'].fillna(median_annual_income)\n\n# Filling missing 'Marital Status' with random choice\ntrain['Marital Status'] = train['Marital Status'].fillna(np.random.choice(['Single', 'Married', 'Divorced']))\ntest['Marital Status'] = test['Marital Status'].fillna(np.random.choice(['Single', 'Married', 'Divorced']))\n\n# Filling missing 'Number of Dependents' with mean\nmean_number_dependents = train['Number of Dependents'].mean()\ntrain['Number of Dependents'] = train['Number of Dependents'].fillna(mean_number_dependents)\ntest['Number of Dependents'] = test['Number of Dependents'].fillna(mean_number_dependents)\n\n# Filling missing 'Occupation' with 'Missing'\ntrain['Occupation'] = train['Occupation'].fillna('Missing')\ntest['Occupation'] = test['Occupation'].fillna('Missing')\n\n# Filling missing 'Health Score' with mean\nmean_health_score = train['Health Score'].mean()\ntrain['Health Score'] = train['Health Score'].fillna(mean_health_score)\ntest['Health Score'] = test['Health Score'].fillna(mean_health_score)\n\n# Filling missing 'Previous Claims' with median\nmedian_previous_claims = train['Previous Claims'].median()\ntrain['Previous Claims'] = train['Previous Claims'].fillna(median_previous_claims)\ntest['Previous Claims'] = test['Previous Claims'].fillna(median_previous_claims)\n\n# Filling missing 'Vehicle Age' with mean\nmean_vehicle_age = train['Vehicle Age'].mean()\ntrain['Vehicle Age'] = train['Vehicle Age'].fillna(mean_vehicle_age)\ntest['Vehicle Age'] = test['Vehicle Age'].fillna(mean_vehicle_age)\n\n# Filling missing 'Credit Score' with mean\nmean_credit_score = train['Credit Score'].mean()\ntrain['Credit Score'] = train['Credit Score'].fillna(mean_credit_score)\ntest['Credit Score'] = test['Credit Score'].fillna(mean_credit_score)\n\n# Filling missing 'Insurance Duration' with mean\nmean_insurance_duration = train['Insurance Duration'].mean()\ntrain['Insurance Duration'] = train['Insurance Duration'].fillna(mean_insurance_duration)\ntest['Insurance Duration'] = test['Insurance Duration'].fillna(mean_insurance_duration)\n\n# Filling missing 'Customer Feedback' with random choice\ntrain['Customer Feedback'] = train['Customer Feedback'].fillna(np.random.choice(['Poor', 'Average', 'Good']))\ntest['Customer Feedback'] = test['Customer Feedback'].fillna(np.random.choice(['Poor', 'Average', 'Good']))\n\n# Dropping 'id' column\ntrain.drop(columns=['id'], inplace=True)\ntest.drop(columns=['id'], inplace=True)\n\nprint(\"✅ Data cleaned!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:29:16.566371Z","iopub.execute_input":"2024-12-04T06:29:16.566768Z","iopub.status.idle":"2024-12-04T06:29:17.597057Z","shell.execute_reply.started":"2024-12-04T06:29:16.566726Z","shell.execute_reply":"2024-12-04T06:29:17.595974Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📆 Feature Engineering: Date Components 🗓️\n\nLet's extract cyclical features from the policy start date! ⏰","metadata":{}},{"cell_type":"code","source":"train['Policy Start Date'] = pd.to_datetime(train['Policy Start Date'])\ntest['Policy Start Date'] = pd.to_datetime(test['Policy Start Date'])\n\n# Cyclical encoding of the year - no need for that\ntrain['Year'] = train['Policy Start Date'].dt.year\ntest['Year'] = test['Policy Start Date'].dt.year\n\n# train['Year_cos'] = CyclicalFeatures(variables=['Year']).fit_transform(train)['Year_cos']\n# train['Year_sin'] = CyclicalFeatures(variables=['Year']).fit_transform(train)['Year_sin']\n\n# test['Year_cos'] = CyclicalFeatures(variables=['Year']).fit_transform(test)['Year_cos']\n# test['Year_sin'] = CyclicalFeatures(variables=['Year']).fit_transform(test)['Year_sin']\n\n# Cyclical encoding of the month\ntrain['Month'] = train['Policy Start Date'].dt.month\ntest['Month'] = test['Policy Start Date'].dt.month\n\ntrain['Month_cos'] = CyclicalFeatures(variables=['Month']).fit_transform(train)['Month_cos']\ntrain['Month_sin'] = CyclicalFeatures(variables=['Month']).fit_transform(train)['Month_sin']\n\ntest['Month_cos'] = CyclicalFeatures(variables=['Month']).fit_transform(test)['Month_cos']\ntest['Month_sin'] = CyclicalFeatures(variables=['Month']).fit_transform(test)['Month_sin']\n\n# Cyclical encoding of the day\ntrain['Day'] = train['Policy Start Date'].dt.day\ntest['Day'] = test['Policy Start Date'].dt.day\n\ntrain['Day_cos'] = CyclicalFeatures(variables=['Day']).fit_transform(train)['Day_cos']\ntrain['Day_sin'] = CyclicalFeatures(variables=['Day']).fit_transform(train)['Day_sin']\n\ntest['Day_cos'] = CyclicalFeatures(variables=['Day']).fit_transform(test)['Day_cos']\ntest['Day_sin'] = CyclicalFeatures(variables=['Day']).fit_transform(test)['Day_sin']\n\ntrain.drop(columns=['Day','Month','Policy Start Date'], inplace=True)\ntest.drop(columns=['Day','Month','Policy Start Date' ], inplace=True)\n\nprint(\"✅ Cyclical features created!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:29:31.166486Z","iopub.execute_input":"2024-12-04T06:29:31.166936Z","iopub.status.idle":"2024-12-04T06:29:43.145418Z","shell.execute_reply.started":"2024-12-04T06:29:31.166898Z","shell.execute_reply":"2024-12-04T06:29:43.14433Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔠 Encoding Categorical Variables 🎭\n\nLabel encoding for our categorical features.","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\n\nfor col in train.columns:\n    if train[col].dtype == 'object':\n        train[col] = le.fit_transform(train[col])\n        test[col] = le.transform(test[col])\n\nprint(\"✅ Categorical variables encoded!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:29:48.715444Z","iopub.execute_input":"2024-12-04T06:29:48.71599Z","iopub.status.idle":"2024-12-04T06:29:52.256825Z","shell.execute_reply.started":"2024-12-04T06:29:48.715927Z","shell.execute_reply":"2024-12-04T06:29:52.25568Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🧪 Splitting Features and Target 🎯","metadata":{}},{"cell_type":"code","source":"y = train['Premium Amount']\nX = train.drop(columns=['Premium Amount'])\n\nprint(f\"Feature matrix shape: {X.shape}\")\nprint(f\"Target vector shape: {y.shape}\")","metadata":{"lines_to_next_cell":1,"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:30:01.774989Z","iopub.execute_input":"2024-12-04T06:30:01.7754Z","iopub.status.idle":"2024-12-04T06:30:01.873072Z","shell.execute_reply.started":"2024-12-04T06:30:01.775365Z","shell.execute_reply":"2024-12-04T06:30:01.872056Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🎛️ Hyperparameter Tuning with Optuna 🔧\n\nLet's find the best hyperparameters using Optuna! 🌟\n\nInspired by : https://www.kaggle.com/code/abdmental01/regression-lgbm-models","metadata":{}},{"cell_type":"code","source":"# def rmsLe(y_true, y_pred):\n#     y_pred = np.maximum(y_pred, 1e-6)\n#     return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\n# SEED = 42\n# n_splits = 5\n\n# def TrainML(params, e_stop=50):\n#     kfold = RepeatedKFold(n_splits=n_splits, n_repeats=1, random_state=SEED)\n#     train_rmse_scores = []\n#     val_rmse_scores = []\n\n#     for fold, (train_idx, val_idx) in enumerate(tqdm(kfold.split(X, y), desc=\"Training Folds\", total=n_splits)):\n#         X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n#         y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n        \n#         y_train_log = np.log1p(y_train)\n#         y_val_log = np.log1p(y_val)\n        \n#         callbacks = [lgb.early_stopping(stopping_rounds=e_stop, verbose=False)]\n#         model = lgb.LGBMRegressor(**params, random_state=SEED, verbose=-1, n_jobs=-1)\n#         model.fit(X_train, y_train_log, \n#                   eval_set=[(X_val, y_val_log)], \n#                   eval_metric='rmse', \n#                   callbacks=callbacks)\n        \n#         y_train_log_pred = model.predict(X_train)\n#         y_val_log_pred = model.predict(X_val)\n        \n#         y_train_pred = np.expm1(y_train_log_pred)\n#         y_val_pred = np.expm1(y_val_log_pred)\n        \n#         train_rmse = rmsLe(y_train, y_train_pred)\n#         val_rmse = rmsLe(y_val, y_val_pred)\n\n#         train_rmse_scores.append(train_rmse)\n#         val_rmse_scores.append(val_rmse)\n\n#     mean_train_rmse = np.mean(train_rmse_scores)\n#     mean_val_rmse = np.mean(val_rmse_scores)\n\n#     print(\"\\n🎯 Final Mean Scores:\")\n#     print(f\"🏋️‍♂️ Mean Train RMSLE: {mean_train_rmse:.4f}\")\n#     print(f\"🧪 Mean Validation RMSLE: {mean_val_rmse:.4f}\")\n\n#     return mean_train_rmse, mean_val_rmse\n\n# def objective(trial):\n#     params = {\n#         'n_estimators': trial.suggest_int('n_estimators', 100, 1000, step=50),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.2, log=True),\n#         'num_leaves': trial.suggest_int('num_leaves', 20, 150),\n#         'max_depth': trial.suggest_int('max_depth', 3, 15),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n#         'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.5, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 1e-4, 10.0, log=True),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 1e-4, 10.0, log=True),\n#         'device': 'gpu'\n#     }\n\n#     mean_train_rmse, mean_val_rmse = TrainML(params)\n#     trial.set_user_attr('train_rmse', mean_train_rmse)\n#     trial.set_user_attr('val_rmse', mean_val_rmse)\n#     return mean_val_rmse\n\n# print(\"🚀 Starting hyperparameter tuning...\")\n\n# study = optuna.create_study(direction='minimize')\n# study.optimize(objective, n_trials=5)\n\n# print(\"✅ Hyperparameter tuning completed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:30:07.235656Z","iopub.execute_input":"2024-12-04T06:30:07.236054Z","iopub.status.idle":"2024-12-04T06:30:07.242692Z","shell.execute_reply.started":"2024-12-04T06:30:07.23602Z","shell.execute_reply":"2024-12-04T06:30:07.241189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🏆 Best Hyperparameters","metadata":{}},{"cell_type":"code","source":"best_params = {'n_estimators': 550, 'learning_rate': 0.011815382331813444, 'num_leaves': 116, 'max_depth': 12, 'min_child_samples': 16, 'subsample': 0.6354134229005893, 'colsample_bytree': 0.8861944216276492, 'reg_alpha': 1.7789343457859361, 'reg_lambda': 0.4341060744803095}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:30:11.734951Z","iopub.execute_input":"2024-12-04T06:30:11.735409Z","iopub.status.idle":"2024-12-04T06:30:11.741286Z","shell.execute_reply.started":"2024-12-04T06:30:11.735372Z","shell.execute_reply":"2024-12-04T06:30:11.740097Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📈 Training the Final Model","metadata":{}},{"cell_type":"code","source":"best_model = LGBMRegressor(**best_params, verbose=-1, n_jobs=-1)\nbest_model.fit(X, np.log1p(y))\n\nprint(\"✅ Final model trained!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:30:16.87525Z","iopub.execute_input":"2024-12-04T06:30:16.87563Z","iopub.status.idle":"2024-12-04T06:31:11.784131Z","shell.execute_reply.started":"2024-12-04T06:30:16.875597Z","shell.execute_reply":"2024-12-04T06:31:11.783071Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🧮 Making Predictions","metadata":{}},{"cell_type":"code","source":"test_preds = best_model.predict(test)\ntest_preds = np.expm1(test_preds)\n\nsample_submission['Premium Amount'] = test_preds\nsample_submission.to_csv('submission1.csv', index=False)\n\nprint(\"✅ Predictions saved to 'submissionOptuna.csv'!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:31:23.006245Z","iopub.execute_input":"2024-12-04T06:31:23.006633Z","iopub.status.idle":"2024-12-04T06:32:07.476483Z","shell.execute_reply.started":"2024-12-04T06:31:23.0066Z","shell.execute_reply":"2024-12-04T06:32:07.475347Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 📊 Visualizing Feature Importance\n\nLet's see which features were the most important! 🌟","metadata":{}},{"cell_type":"code","source":"importances = best_model.feature_importances_\nfeatures = X.columns\nimportance_df = pd.DataFrame({'Feature': features, 'Importance': importances})\nimportance_df.sort_values(by='Importance', ascending=False, inplace=True)\n\nplt.figure(figsize=(12, 8))\nsns.barplot(x='Importance', y='Feature', data=importance_df)\nplt.title('Feature Importance')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-04T06:32:14.395062Z","iopub.execute_input":"2024-12-04T06:32:14.395463Z","iopub.status.idle":"2024-12-04T06:32:14.915998Z","shell.execute_reply.started":"2024-12-04T06:32:14.395429Z","shell.execute_reply":"2024-12-04T06:32:14.914738Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🎉 Congratulations!\n\nYou've successfully completed this data science adventure! 🥳 Feel free to explore and tweak the notebook to make it your own. Happy Kaggle-ing! 🚀","metadata":{}},{"cell_type":"markdown","source":"# 🚀 Next Steps\n\n- Experiment with different models like XGBoost or CatBoost.\n- Perform more feature engineering.\n- Try out stacking or blending models.\n- Dive deeper into data visualization.","metadata":{}},{"cell_type":"markdown","source":"# 🙌 Acknowledgments\n\nThanks for joining this journey! Keep exploring and pushing the boundaries. The data science world is vast and full of wonders! 🌌","metadata":{}}]}