{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30805,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn==1.5.2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:47:47.90185Z","iopub.execute_input":"2024-12-14T07:47:47.902504Z","iopub.status.idle":"2024-12-14T07:47:56.293329Z","shell.execute_reply.started":"2024-12-14T07:47:47.90247Z","shell.execute_reply":"2024-12-14T07:47:56.292395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sklearn\nsklearn.__version__","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:47:56.295842Z","iopub.execute_input":"2024-12-14T07:47:56.296239Z","iopub.status.idle":"2024-12-14T07:47:56.302926Z","shell.execute_reply.started":"2024-12-14T07:47:56.296196Z","shell.execute_reply":"2024-12-14T07:47:56.302108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import root_mean_squared_log_error","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:47:56.304284Z","iopub.execute_input":"2024-12-14T07:47:56.304679Z","iopub.status.idle":"2024-12-14T07:47:56.314277Z","shell.execute_reply.started":"2024-12-14T07:47:56.304637Z","shell.execute_reply":"2024-12-14T07:47:56.313539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:47:56.315798Z","iopub.execute_input":"2024-12-14T07:47:56.316141Z","iopub.status.idle":"2024-12-14T07:48:01.930594Z","shell.execute_reply.started":"2024-12-14T07:47:56.316116Z","shell.execute_reply":"2024-12-14T07:48:01.929851Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Pre-processing","metadata":{}},{"cell_type":"markdown","source":"## Remove Outlier","metadata":{}},{"cell_type":"code","source":"Q1 = train_df['Premium Amount'].quantile(0.25)  \nQ3 = train_df['Premium Amount'].quantile(0.75)  \nIQR = Q3 - Q1 \n\nlower_bound = Q1 - 1.5 * IQR  \nupper_bound = Q3 + 1.5 * IQR  \n\ntrain_df = train_df[(train_df['Premium Amount'] >= lower_bound) & (train_df['Premium Amount'] <= upper_bound)]  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:01.931728Z","iopub.execute_input":"2024-12-14T07:48:01.932091Z","iopub.status.idle":"2024-12-14T07:48:02.159556Z","shell.execute_reply.started":"2024-12-14T07:48:01.932053Z","shell.execute_reply":"2024-12-14T07:48:02.158836Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transforming Policy Start Date","metadata":{}},{"cell_type":"code","source":"train_df['Policy Start Date'] = pd.to_datetime(train_df['Policy Start Date'].str[:26])  # Keep only one datetime part\ntrain_df['Policy Start Year'] = train_df['Policy Start Date'].dt.year\ntrain_df.drop(columns=['Policy Start Date'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:02.160574Z","iopub.execute_input":"2024-12-14T07:48:02.160829Z","iopub.status.idle":"2024-12-14T07:48:02.83176Z","shell.execute_reply.started":"2024-12-14T07:48:02.160787Z","shell.execute_reply":"2024-12-14T07:48:02.83093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['Policy Start Date'] = pd.to_datetime(test_df['Policy Start Date'].str[:26])  # Keep only one datetime part\ntest_df['Policy Start Year'] = test_df['Policy Start Date'].dt.year\ntest_df.drop(columns=['Policy Start Date'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:02.833048Z","iopub.execute_input":"2024-12-14T07:48:02.833409Z","iopub.status.idle":"2024-12-14T07:48:03.274219Z","shell.execute_reply.started":"2024-12-14T07:48:02.833357Z","shell.execute_reply":"2024-12-14T07:48:03.273392Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split Target Variable","metadata":{}},{"cell_type":"code","source":"X = train_df.drop(columns=['Premium Amount'])\ny = train_df['Premium Amount']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.275483Z","iopub.execute_input":"2024-12-14T07:48:03.275888Z","iopub.status.idle":"2024-12-14T07:48:03.462728Z","shell.execute_reply.started":"2024-12-14T07:48:03.275848Z","shell.execute_reply":"2024-12-14T07:48:03.46187Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## pre-processing pipeline","metadata":{}},{"cell_type":"code","source":"num_col = [col for col in train_df.columns if train_df[col].dtype == 'float64' or train_df[col].dtype == 'int64' or train_df[col].dtype == 'int32']\ncat_col = [col for col in train_df.columns if train_df[col].dtype == 'object']\n\nnum_col.remove('Premium Amount')\ntarget_col = ['Premium Amount']\n\nprint(\"numerical columns:\", num_col)\nprint(\"categorical columns:\", cat_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.464164Z","iopub.execute_input":"2024-12-14T07:48:03.464546Z","iopub.status.idle":"2024-12-14T07:48:03.471759Z","shell.execute_reply.started":"2024-12-14T07:48:03.464488Z","shell.execute_reply":"2024-12-14T07:48:03.470872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_col.remove('id')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.474503Z","iopub.execute_input":"2024-12-14T07:48:03.474872Z","iopub.status.idle":"2024-12-14T07:48:03.485794Z","shell.execute_reply.started":"2024-12-14T07:48:03.474831Z","shell.execute_reply":"2024-12-14T07:48:03.48507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(type(num_col))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.57352Z","iopub.execute_input":"2024-12-14T07:48:03.574028Z","iopub.status.idle":"2024-12-14T07:48:03.578776Z","shell.execute_reply.started":"2024-12-14T07:48:03.573993Z","shell.execute_reply":"2024-12-14T07:48:03.577883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OneHotEncoder\n\nnum_strategies = {\n    'Age': 'median',\n    'Annual Income': 'median',\n    'Number of Dependents': 'most_frequent',\n    'Health Score': 'mean',\n    'Previous Claims': 'constant', \n    'Vehicle Age': 'median',\n    'Credit Score': 'mean',\n    'Insurance Duration': 'mean',\n    'Policy Start Year': 'most_frequent'\n}\n\nnum_imputers = [(col, SimpleImputer(strategy=num_strategies[col], fill_value=0), [col]) for col in num_col]\n\nnum_preprocessor = ColumnTransformer(transformers=num_imputers)\n\nnum_pipeline = Pipeline([\n    ('imputer', num_preprocessor), \n    ('scaler', StandardScaler()),  \n])\n\ncat_pipeline = Pipeline([\n    ('imputer', SimpleImputer(strategy=\"constant\", fill_value=\"Unknown\")),\n    ('onehot', OneHotEncoder(handle_unknown='ignore')),\n])\n\npreprocessor = ColumnTransformer([\n    (\"num\", num_pipeline, num_col),\n    (\"cat\", cat_pipeline, cat_col),\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.580083Z","iopub.execute_input":"2024-12-14T07:48:03.580712Z","iopub.status.idle":"2024-12-14T07:48:03.593787Z","shell.execute_reply.started":"2024-12-14T07:48:03.580672Z","shell.execute_reply":"2024-12-14T07:48:03.593048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_processed = preprocessor.fit_transform(X)\ntest_processed = preprocessor.transform(test_df.drop(columns=['id']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:03.594904Z","iopub.execute_input":"2024-12-14T07:48:03.595262Z","iopub.status.idle":"2024-12-14T07:48:11.525985Z","shell.execute_reply.started":"2024-12-14T07:48:03.595222Z","shell.execute_reply":"2024-12-14T07:48:11.525163Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Split data into train and test set","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X_processed, y, test_size=0.3, random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:11.526896Z","iopub.execute_input":"2024-12-14T07:48:11.527157Z","iopub.status.idle":"2024-12-14T07:48:11.744203Z","shell.execute_reply.started":"2024-12-14T07:48:11.527132Z","shell.execute_reply":"2024-12-14T07:48:11.743221Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Model","metadata":{}},{"cell_type":"markdown","source":"## Linear Regression","metadata":{}},{"cell_type":"code","source":"model = LinearRegression()\nmodel.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:11.745307Z","iopub.execute_input":"2024-12-14T07:48:11.745569Z","iopub.status.idle":"2024-12-14T07:48:13.128697Z","shell.execute_reply.started":"2024-12-14T07:48:11.745543Z","shell.execute_reply":"2024-12-14T07:48:13.127845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nrmsle = root_mean_squared_log_error(y_test, y_pred)\nprint(f\"RMSLE: {rmsle:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:13.129597Z","iopub.execute_input":"2024-12-14T07:48:13.129915Z","iopub.status.idle":"2024-12-14T07:48:13.178531Z","shell.execute_reply.started":"2024-12-14T07:48:13.12988Z","shell.execute_reply":"2024-12-14T07:48:13.177503Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LightGBM","metadata":{}},{"cell_type":"code","source":"import optuna\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold\n\ndef objective(trial):\n    params = {\n        \"device_type\": \"gpu\",\n        \"objective\": \"regression\",\n        \"metric\": \"rmse\",\n        \"boosting_type\": trial.suggest_categorical(\"boosting_type\", [\"gbdt\", \"dart\", \"rf\"]),\n        \"max_depth\": trial.suggest_int(\"max_depth\", -1, 20),\n        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-4, 10.0, log=True),\n        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-4, 10.0,log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 200, 512),\n        \"learning_rate\": trial.suggest_float(\"learning_rate\", 1e-4, 1e-1, log=True),\n        \"feature_fraction\": trial.suggest_uniform(\"feature_fraction\", 0.6, 1.0),\n        \"bagging_fraction\": trial.suggest_uniform(\"bagging_fraction\", 0.6, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 5, 12),\n        \"min_data_in_leaf\": trial.suggest_int(\"min_data_in_leaf\", 20, 100),\n        \"seed\" : 42,\n        \"verbose\": -100\n    }\n    \n    \n    \n    kf = KFold(n_splits=5, shuffle=True, random_state=42) \n    rmsle_scores = []\n\n    for train_idx, valid_idx in kf.split(X_processed):\n        X_train, X_valid = X_processed[train_idx], X_processed[valid_idx]\n        y_train, y_valid = y.iloc[train_idx], y.iloc[valid_idx]\n        \n        train_set = lgb.Dataset(X_train, label=y_train)\n        test_set = lgb.Dataset(X_valid, label=y_valid, reference=train_set)\n    \n        model = lgb.train(\n            params,\n            train_set,\n            valid_sets=[test_set],\n        )\n        \n        y_pred = model.predict(X_valid)\n        rmsle = root_mean_squared_log_error(y_valid, y_pred)\n        rmsle_scores.append(rmsle)\n    \n    return np.mean(rmsle_scores)   \n\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=50)\n\nprint(\"Best parameters:\", study.best_params)\nprint(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-12-14T07:48:13.179947Z","iopub.execute_input":"2024-12-14T07:48:13.180403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(study.best_params)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_params = study.best_params\n\nmodel = lgb.train(\n        best_params,\n        lgb.Dataset(X_train, label=y_train),\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Evaluation","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(X_test)\nrmsle = root_mean_squared_log_error(y_test, y_pred)\nprint(f\"RMSLE: {rmsle:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Test Prediction","metadata":{}},{"cell_type":"code","source":"predictions = model.predict(test_processed)\nsubmission = pd.DataFrame({'id': test_df['id'], 'Premium Amount': predictions})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}