{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn==1.5.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:19:53.953919Z","iopub.execute_input":"2024-12-11T10:19:53.954354Z","iopub.status.idle":"2024-12-11T10:20:02.385938Z","shell.execute_reply.started":"2024-12-11T10:19:53.954324Z","shell.execute_reply":"2024-12-11T10:20:02.384774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sklearn\nsklearn.__version__","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:02.388001Z","iopub.execute_input":"2024-12-11T10:20:02.388313Z","iopub.status.idle":"2024-12-11T10:20:02.394529Z","shell.execute_reply.started":"2024-12-11T10:20:02.388284Z","shell.execute_reply":"2024-12-11T10:20:02.393701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import root_mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:02.395833Z","iopub.execute_input":"2024-12-11T10:20:02.396583Z","iopub.status.idle":"2024-12-11T10:20:02.405161Z","shell.execute_reply.started":"2024-12-11T10:20:02.39654Z","shell.execute_reply":"2024-12-11T10:20:02.404503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:02.40741Z","iopub.execute_input":"2024-12-11T10:20:02.408039Z","iopub.status.idle":"2024-12-11T10:20:07.486018Z","shell.execute_reply.started":"2024-12-11T10:20:02.408Z","shell.execute_reply":"2024-12-11T10:20:07.485252Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pre-processing","metadata":{}},{"cell_type":"markdown","source":"## Remove Outlier","metadata":{}},{"cell_type":"code","source":"Q1 = train_df['Premium Amount'].quantile(0.25)  \nQ3 = train_df['Premium Amount'].quantile(0.75)  \nIQR = Q3 - Q1 \n\nlower_bound = Q1 - 1.5 * IQR  \nupper_bound = Q3 + 1.5 * IQR  \n\ntrain_df = train_df[(train_df['Premium Amount'] >= lower_bound) & (train_df['Premium Amount'] <= upper_bound)]  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:07.486995Z","iopub.execute_input":"2024-12-11T10:20:07.487264Z","iopub.status.idle":"2024-12-11T10:20:07.718131Z","shell.execute_reply.started":"2024-12-11T10:20:07.487239Z","shell.execute_reply":"2024-12-11T10:20:07.717396Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transforming Policy Start Date","metadata":{}},{"cell_type":"code","source":"train_df['Policy Start Date'] = pd.to_datetime(train_df['Policy Start Date'].str[:26]) \ntrain_df['Year'] = train_df['Policy Start Date'].dt.year\ntrain_df['Month'] = train_df['Policy Start Date'].dt.month\ntrain_df.drop(columns=['Policy Start Date'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:07.719053Z","iopub.execute_input":"2024-12-11T10:20:07.719312Z","iopub.status.idle":"2024-12-11T10:20:08.408219Z","shell.execute_reply.started":"2024-12-11T10:20:07.719287Z","shell.execute_reply":"2024-12-11T10:20:08.407285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'].str[:26]) \ntest_df['Year'] = test_df['Policy Start Date'].dt.year\ntest_df['Month'] = test_df['Policy Start Date'].dt.month\ntest_df.drop(columns=['Policy Start Date'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:08.409517Z","iopub.execute_input":"2024-12-11T10:20:08.40987Z","iopub.status.idle":"2024-12-11T10:20:08.887043Z","shell.execute_reply.started":"2024-12-11T10:20:08.409831Z","shell.execute_reply":"2024-12-11T10:20:08.886361Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split Target Variable","metadata":{}},{"cell_type":"code","source":"X = train_df.drop(columns=['Premium Amount'])\ny = train_df['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:08.887949Z","iopub.execute_input":"2024-12-11T10:20:08.888176Z","iopub.status.idle":"2024-12-11T10:20:09.038963Z","shell.execute_reply.started":"2024-12-11T10:20:08.888153Z","shell.execute_reply":"2024-12-11T10:20:09.038266Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## pre-processing pipeline","metadata":{}},{"cell_type":"code","source":"num_col = [col for col in train_df.columns if train_df[col].dtype == 'float64' or train_df[col].dtype == 'int64' or train_df[col].dtype == 'int32']\ncat_col = [col for col in train_df.columns if train_df[col].dtype == 'object']\n\nnum_col.remove('Premium Amount')\ntarget_col = ['Premium Amount']\n\nprint(\"numerical columns:\", num_col)\nprint(\"categorical columns:\", cat_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:09.039859Z","iopub.execute_input":"2024-12-11T10:20:09.040085Z","iopub.status.idle":"2024-12-11T10:20:09.141786Z","shell.execute_reply.started":"2024-12-11T10:20:09.040062Z","shell.execute_reply":"2024-12-11T10:20:09.140917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_col.remove('id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:09.145306Z","iopub.execute_input":"2024-12-11T10:20:09.145634Z","iopub.status.idle":"2024-12-11T10:20:09.15388Z","shell.execute_reply.started":"2024-12-11T10:20:09.145595Z","shell.execute_reply":"2024-12-11T10:20:09.153173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\n\nnum_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy=\"mean\")),\n    ('std_scaler', StandardScaler()),\n])\n\ncat_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy=\"constant\", fill_value=\"Unknown\")),\n    ('onehot', OneHotEncoder(handle_unknown='ignore')),\n])\n\npreprocessor = ColumnTransformer([\n    (\"num\", num_pipeline, num_col),\n    (\"cat\", cat_pipeline, cat_col),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:09.155052Z","iopub.execute_input":"2024-12-11T10:20:09.155377Z","iopub.status.idle":"2024-12-11T10:20:09.165415Z","shell.execute_reply.started":"2024-12-11T10:20:09.155351Z","shell.execute_reply":"2024-12-11T10:20:09.164599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_processed = preprocessor.fit_transform(X)\ntest_processed = preprocessor.transform(test_df.drop(columns=['id']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:15.545847Z","iopub.execute_input":"2024-12-11T10:20:15.54622Z","iopub.status.idle":"2024-12-11T10:20:22.926293Z","shell.execute_reply.started":"2024-12-11T10:20:15.546188Z","shell.execute_reply":"2024-12-11T10:20:22.925585Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split data into train and test set","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_processed, y, test_size=0.3, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:22.927691Z","iopub.execute_input":"2024-12-11T10:20:22.92796Z","iopub.status.idle":"2024-12-11T10:20:23.157514Z","shell.execute_reply.started":"2024-12-11T10:20:22.927934Z","shell.execute_reply":"2024-12-11T10:20:23.156791Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"markdown","source":"## Linear Regression","metadata":{}},{"cell_type":"code","source":"model = LinearRegression()\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:23.15855Z","iopub.execute_input":"2024-12-11T10:20:23.158884Z","iopub.status.idle":"2024-12-11T10:20:24.609031Z","shell.execute_reply.started":"2024-12-11T10:20:23.158848Z","shell.execute_reply":"2024-12-11T10:20:24.608357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nrmsle = root_mean_squared_log_error(y_test, y_pred)\nprint(f\"RMSLE: {rmsle:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:24.610413Z","iopub.execute_input":"2024-12-11T10:20:24.610727Z","iopub.status.idle":"2024-12-11T10:20:24.65055Z","shell.execute_reply.started":"2024-12-11T10:20:24.610692Z","shell.execute_reply":"2024-12-11T10:20:24.649521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBoost","metadata":{}},{"cell_type":"code","source":"import optuna\nfrom xgboost import XGBRegressor\nfrom sklearn.metrics import mean_squared_log_error\n\ndef objective(trial):\n    params = {\n        'device': 'gpu',\n        'seed': 42,\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n        'n_estimators': trial.suggest_int('n_estimators', 50, 200),\n        'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n        'eta': trial.suggest_float('eta', 0, 1),\n        'max_depth': trial.suggest_int('max_depth', 5, 20),\n        'alpha': trial.suggest_float(\"alpha\", 1e-4, 10.0, log=True),\n        'lambda': trial.suggest_float(\"lambda\", 1e-4, 10.0,log=True)\n    }\n    model = XGBRegressor(**params)\n    model.fit(X_train, y_train, eval_set=[(X_test, y_test)], verbose=0)\n\n    y_pred = model.predict(X_test)\n    y_pred = np.clip(y_pred, 0, None)  \n\n    rmsle = root_mean_squared_log_error(y_test, y_pred)\n    return rmsle  \n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:13.31888Z","iopub.status.idle":"2024-12-11T10:20:13.319292Z","shell.execute_reply.started":"2024-12-11T10:20:13.319078Z","shell.execute_reply":"2024-12-11T10:20:13.319101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(study.best_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:13.32093Z","iopub.status.idle":"2024-12-11T10:20:13.321348Z","shell.execute_reply.started":"2024-12-11T10:20:13.32113Z","shell.execute_reply":"2024-12-11T10:20:13.321153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\nmodel = XGBRegressor(**best_params)\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:30.827623Z","iopub.execute_input":"2024-12-11T10:20:30.827934Z","iopub.status.idle":"2024-12-11T10:20:39.825499Z","shell.execute_reply.started":"2024-12-11T10:20:30.827909Z","shell.execute_reply":"2024-12-11T10:20:39.824739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nrmsle = root_mean_squared_log_error(y_test, y_pred)\nprint(f\"RMSLE: {rmsle:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:39.826643Z","iopub.execute_input":"2024-12-11T10:20:39.826911Z","iopub.status.idle":"2024-12-11T10:20:40.534892Z","shell.execute_reply.started":"2024-12-11T10:20:39.826884Z","shell.execute_reply":"2024-12-11T10:20:40.533659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Prediction","metadata":{}},{"cell_type":"code","source":"predictions = model.predict(test_processed)\nsubmission = pd.DataFrame({'id': test_df['id'], 'Premium Amount': predictions})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-11T10:20:13.326671Z","iopub.status.idle":"2024-12-11T10:20:13.327078Z","shell.execute_reply.started":"2024-12-11T10:20:13.326869Z","shell.execute_reply":"2024-12-11T10:20:13.32689Z"}},"outputs":[],"execution_count":null}]}