{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import necessary libraries.\nimport pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder, OrdinalEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler\nimport xgboost as xgb\nimport numpy as np\nimport pandas as pd\nfrom statistics import mean\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import Pipeline\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nimport optuna\nimport numpy as np\nimport pandas as pd\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.metrics import mean_squared_log_error\nimport os\nos.environ[\"LOKY_MAX_CPU_COUNT\"] = \"4\" ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:20.791363Z","iopub.execute_input":"2024-12-28T15:47:20.791598Z","iopub.status.idle":"2024-12-28T15:47:22.095099Z","shell.execute_reply.started":"2024-12-28T15:47:20.791575Z","shell.execute_reply":"2024-12-28T15:47:22.094433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest_df = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:22.095812Z","iopub.execute_input":"2024-12-28T15:47:22.09625Z","iopub.status.idle":"2024-12-28T15:47:30.26432Z","shell.execute_reply.started":"2024-12-28T15:47:22.096219Z","shell.execute_reply":"2024-12-28T15:47:30.263415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:30.265185Z","iopub.execute_input":"2024-12-28T15:47:30.265435Z","iopub.status.idle":"2024-12-28T15:47:30.297788Z","shell.execute_reply.started":"2024-12-28T15:47:30.265414Z","shell.execute_reply":"2024-12-28T15:47:30.297023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:30.299914Z","iopub.execute_input":"2024-12-28T15:47:30.300199Z","iopub.status.idle":"2024-12-28T15:47:30.316757Z","shell.execute_reply.started":"2024-12-28T15:47:30.300178Z","shell.execute_reply":"2024-12-28T15:47:30.316054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:30.320558Z","iopub.execute_input":"2024-12-28T15:47:30.320753Z","iopub.status.idle":"2024-12-28T15:47:30.868539Z","shell.execute_reply.started":"2024-12-28T15:47:30.320727Z","shell.execute_reply":"2024-12-28T15:47:30.867759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:30.869315Z","iopub.execute_input":"2024-12-28T15:47:30.86957Z","iopub.status.idle":"2024-12-28T15:47:31.225628Z","shell.execute_reply.started":"2024-12-28T15:47:30.869549Z","shell.execute_reply":"2024-12-28T15:47:31.224763Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:31.226511Z","iopub.execute_input":"2024-12-28T15:47:31.226761Z","iopub.status.idle":"2024-12-28T15:47:31.788518Z","shell.execute_reply.started":"2024-12-28T15:47:31.226728Z","shell.execute_reply":"2024-12-28T15:47:31.787738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.describe()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:31.789316Z","iopub.execute_input":"2024-12-28T15:47:31.789605Z","iopub.status.idle":"2024-12-28T15:47:32.110579Z","shell.execute_reply.started":"2024-12-28T15:47:31.789582Z","shell.execute_reply":"2024-12-28T15:47:32.109797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:32.111313Z","iopub.execute_input":"2024-12-28T15:47:32.111584Z","iopub.status.idle":"2024-12-28T15:47:32.633889Z","shell.execute_reply.started":"2024-12-28T15:47:32.111562Z","shell.execute_reply":"2024-12-28T15:47:32.633136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:32.634662Z","iopub.execute_input":"2024-12-28T15:47:32.634913Z","iopub.status.idle":"2024-12-28T15:47:32.985942Z","shell.execute_reply.started":"2024-12-28T15:47:32.634892Z","shell.execute_reply":"2024-12-28T15:47:32.985298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.hist('Premium Amount')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:32.986635Z","iopub.execute_input":"2024-12-28T15:47:32.986838Z","iopub.status.idle":"2024-12-28T15:47:33.313168Z","shell.execute_reply.started":"2024-12-28T15:47:32.986822Z","shell.execute_reply":"2024-12-28T15:47:33.312423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df.hist('Annual Income')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:33.315362Z","iopub.execute_input":"2024-12-28T15:47:33.315583Z","iopub.status.idle":"2024-12-28T15:47:33.515817Z","shell.execute_reply.started":"2024-12-28T15:47:33.315558Z","shell.execute_reply":"2024-12-28T15:47:33.514986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def date_trans(df):\n    df['Policy Start Date']= pd.to_datetime(df['Policy Start Date'])\n    df['Year'] = df['Policy Start Date'].dt.year\n    df['Day'] = df['Policy Start Date'].dt.day\n    df['Month'] = df['Policy Start Date'].dt.month\n    df['DayOfWeek'] = df['Policy Start Date'].dt.dayofweek\n    df.drop('Policy Start Date' , axis =1, inplace = True)\n    return df\n\ntrain_df = date_trans(train_df)\ntest_df = date_trans(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:33.516805Z","iopub.execute_input":"2024-12-28T15:47:33.517049Z","iopub.status.idle":"2024-12-28T15:47:34.715589Z","shell.execute_reply.started":"2024-12-28T15:47:33.517011Z","shell.execute_reply":"2024-12-28T15:47:34.714682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:34.716435Z","iopub.execute_input":"2024-12-28T15:47:34.716751Z","iopub.status.idle":"2024-12-28T15:47:34.720475Z","shell.execute_reply.started":"2024-12-28T15:47:34.71672Z","shell.execute_reply":"2024-12-28T15:47:34.71984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in train_df.select_dtypes(include='object').columns:\n    train_df[col] = train_df[col].astype('category')\nfor col in test_df.select_dtypes(include='object').columns:\n    test_df[col] = test_df[col].astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:34.721219Z","iopub.execute_input":"2024-12-28T15:47:34.721481Z","iopub.status.idle":"2024-12-28T15:47:36.617523Z","shell.execute_reply.started":"2024-12-28T15:47:34.721461Z","shell.execute_reply":"2024-12-28T15:47:36.616798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import KBinsDiscretizer\nfrom lightgbm import LGBMRegressor\nfrom sklearn.metrics import mean_squared_error, mean_squared_log_error\n\ndef log_transform(y):\n    \"\"\"\n    Apply log transformation safely, handling zero and negative values\n    \"\"\"\n    return np.log1p(y)\n\ndef inverse_log_transform(y_log):\n    \"\"\"\n    Revert log transformation\n    \"\"\"\n    return np.expm1(y_log)\n\n# Example dataset (replace with your dataset)\nnp.random.seed(42)\ndf = train_df\n\n# Find the target column (assumes it contains 'Premium' or 'Amount')\ntarget_column = [col for col in df.columns if 'premium' in col.lower() or 'amount' in col.lower()]\nif not target_column:\n    raise ValueError(\"Could not find target column. Please specify the column name for premium/amount.\")\ntarget_column = target_column[0]\n\n# Identify column types\nnumeric_features = df.select_dtypes(include=['int64', 'float64']).columns.tolist()\ncategorical_features = df.select_dtypes(include=['object', 'category']).columns.tolist()\n\n# Remove target variable from features\nif target_column in numeric_features:\n    numeric_features.remove(target_column)\nif target_column in categorical_features:\n    categorical_features.remove(target_column)\n\n# Features and target\nX = df.drop(columns=[target_column])\ny = df[target_column]\n\n# Log transform the target variable\ny_log = log_transform(y)\n\n# Stratify target by binning into discrete intervals\nbinner = KBinsDiscretizer(n_bins=5, encode='ordinal', strategy='uniform')\ny_binned = binner.fit_transform(y_log.values.reshape(-1, 1)).astype(int).ravel()\n\n# Stratified K-Fold\nn_splits = 5\nskf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n\n# Arrays to store predictions\noof_predictions_log = np.zeros(len(X))  # Out-of-fold predictions in log space\noof_predictions = np.zeros(len(X))  # Out-of-fold predictions in original space\n\n# Test set (replace with your actual test set)\nX_test = test_df\n\n# Prepare for cross-validation\ntest_preds_per_fold_log = np.zeros((len(X_test), n_splits))  # Store test predictions per fold in log space\ntest_preds_per_fold = np.zeros((len(X_test), n_splits))  # Store test predictions per fold in original space\nfold_oof_results = []  # To store ID, target, and OOF predictions\n\n# LightGBM Regressor parameters\nmodel_params = {\n    'objective': 'regression',\n    'metric': 'rmsle',\n    'random_state': 42,\n    'is_unbalance': True,\n      'nan_as_missing': True,\n      'n_estimators': 2900, \n      'learning_rate': 0.009292502487271825,\n      'max_depth': 11,\n      'num_leaves': 88, \n      'min_child_samples': 46,\n      'min_child_weight': 0.0008888702484611456, \n      'subsample': 0.8552895795397466, \n      'subsample_freq': 5, \n      'colsample_bytree': 0.9135439280459978,\n      'reg_alpha': 0.0021100979917979567,\n      'reg_lambda': 5.0153235478869446e-08,\n      'verbose': -1,\n        'device': 'gpu',\n        'gpu_platform_id': 0,\n        'gpu_device_id': 0,\n        'gpu_use_dp': True  \n}\n\n# Cross-validation loop\nfor fold, (train_idx, valid_idx) in enumerate(skf.split(X, y_binned)):\n    print(f\"Fold {fold + 1}/{n_splits}\")\n    X_train, X_valid = X.iloc[train_idx], X.iloc[valid_idx]\n    y_train_log, y_valid_log = y_log.iloc[train_idx], y_log.iloc[valid_idx]\n    y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n    \n    # Model with early stopping on log-transformed target\n    model = LGBMRegressor(**{**model_params, 'verbose': -1})\n    model.fit(X_train, y_train_log, \n              eval_set=[(X_valid, y_valid_log)],\n              )\n    \n    # Predictions in log space\n    oof_predictions_log[valid_idx] = model.predict(X_valid)\n    \n    # Convert log predictions back to original space\n    oof_predictions[valid_idx] = inverse_log_transform(oof_predictions_log[valid_idx])\n\n    fold_rmsle = rmsle(y.iloc[valid_idx], oof_predictions[valid_idx])\n    print(f\"Fold {fold + 1} RMSLE: {fold_rmsle:.4f}\")\n    \n    # Store fold results with ID, target, and OOF predictions\n    fold_result = pd.DataFrame({\n        'ID': X.index[valid_idx],       # Assuming X has an index as IDs\n        'Actual': y.iloc[valid_idx],    # Actual target values\n        'OOF_Pred_LGB': oof_predictions[valid_idx],    # OOF predictions\n        'Fold': fold + 1               # Fold number\n    })\n    fold_oof_results.append(fold_result)\n    \n    # Test set predictions for this fold\n    test_preds_per_fold_log[:, fold] = model.predict(X_test)\n    test_preds_per_fold[:, fold] = inverse_log_transform(test_preds_per_fold_log[:, fold])\n\n# Combine OOF results\noof_results_df = pd.concat(fold_oof_results, axis=0, ignore_index=True)\n\n# Average predictions on test data\nfinal_test_predictions = test_preds_per_fold.mean(axis=1)\n\n# Evaluate OOF predictions\noof_mse = mean_squared_error(y, oof_predictions)\noof_rmsle = np.sqrt(mean_squared_log_error(y, oof_predictions))\n\nprint(f\"OOF Mean Squared Error: {oof_mse:.4f}\")\nprint(f\"OOF Root Mean Squared Log Error: {oof_rmsle:.4f}\")\n\nprint(\"Final Test Predictions:\", final_test_predictions)\nprint(oof_results_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-28T15:47:36.618284Z","iopub.execute_input":"2024-12-28T15:47:36.618495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')\nsub['Premium Amount'] = final_test_predictions\nsub.to_csv('submission.csv', index=False)\nsub.head()\noof_results_df.to_csv('oof_lgbm.csv',index = False)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}