{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"},{"sourceId":9178166,"sourceType":"datasetVersion","datasetId":5547076}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"# Imports\nimport numpy as np \nimport pandas as pd \nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\nfrom sklearn.metrics import *\nfrom sklearn.model_selection import *\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport optuna \nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import cross_val_score\n\n# CSV File Locations\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:12.427978Z","iopub.execute_input":"2024-12-31T15:18:12.428299Z","iopub.status.idle":"2024-12-31T15:18:17.117227Z","shell.execute_reply.started":"2024-12-31T15:18:12.42827Z","shell.execute_reply":"2024-12-31T15:18:17.116495Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"code","source":"tune_lgb = False\ntune_xgb = False\ntune_cat = False\ntune_features = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:17.118109Z","iopub.execute_input":"2024-12-31T15:18:17.118629Z","iopub.status.idle":"2024-12-31T15:18:17.122147Z","shell.execute_reply.started":"2024-12-31T15:18:17.118602Z","shell.execute_reply":"2024-12-31T15:18:17.121288Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/playground-series-s4e12/train.csv')\ntest = pd.read_csv('/kaggle/input/playground-series-s4e12/test.csv')\n\n# Drop id = 0.01 (decent) improvement to score\ntrain.drop('id', axis = 1, inplace = True)\ntest.drop('id', axis = 1, inplace = True) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:17.12424Z","iopub.execute_input":"2024-12-31T15:18:17.124644Z","iopub.status.idle":"2024-12-31T15:18:26.000074Z","shell.execute_reply.started":"2024-12-31T15:18:17.12461Z","shell.execute_reply":"2024-12-31T15:18:25.999413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dep = 'Premium Amount'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:26.001156Z","iopub.execute_input":"2024-12-31T15:18:26.001399Z","iopub.status.idle":"2024-12-31T15:18:26.004967Z","shell.execute_reply.started":"2024-12-31T15:18:26.001375Z","shell.execute_reply":"2024-12-31T15:18:26.004096Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering and Pre Processing","metadata":{}},{"cell_type":"code","source":"def feature_eng(df):\n    df['date'] = df['Policy Start Date'].apply(lambda x: x.split(' ')[0])\n    df['date'] = pd.to_datetime(df['date'])\n    df['year'] = df['date'].dt.year \n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day \n    df.drop('Policy Start Date', axis = 1, inplace = True)\n    df.drop('date', axis = 1, inplace = True)\n\n    df['Log Annual Income'] = np.log(df['Annual Income']) # big improvement to score\n\n    df['Credit_Score_Over_512'] = (df['Credit Score'] > 512).astype('object')\n\nfeature_eng(train)\nfeature_eng(test)\n\ncats = test.select_dtypes(include=[\"object_\"]).columns.tolist()\nconts = [col for col in test.columns if col not in cats]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:26.005953Z","iopub.execute_input":"2024-12-31T15:18:26.006279Z","iopub.status.idle":"2024-12-31T15:18:28.360723Z","shell.execute_reply.started":"2024-12-31T15:18:26.006236Z","shell.execute_reply":"2024-12-31T15:18:28.360049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# try different features. Ratios, additions etc.\n\ndef extra_features(df):\n    df['Policy_Start_Date'] = pd.to_datetime(\n        dict(year=df['year'], month=df['month'], day=df['day']), errors='coerce'\n    )\n\n    df[\"Week\"]        = df[\"Policy_Start_Date\"].dt.isocalendar().week\n    df[\"Weekday\"]     = df[\"Policy_Start_Date\"].dt.weekday\n\n    df['DaySin']      = np.sin(2 * np.pi * df['day'] / 30) # Credit to https://www.kaggle.com/code/ravi20076/playgrounds4e12-baseline-v1#PREPROCESSING  \n    df['DayCos']      = np.cos(2 * np.pi * df['day'] / 30)\n    df['WeekdaySin']  = np.sin(2 * np.pi * df['Weekday'] / 7)\n    df['WeekdayCos']  = np.cos(2 * np.pi * df['Weekday'] / 7)\n    \n    \n    \n    \n    # Determine the most recent policy start date\n    most_recent_date = df['Policy_Start_Date'].max()\n\n    # Temporal Features\n    df['Policy_Age'] = (most_recent_date - df['Policy_Start_Date']).dt.days\n    df.drop('Policy_Start_Date', axis = 1, inplace = True)\n    df['Season'] = df['month'].apply(lambda x: 'Winter' if x in [12, 1, 2] else\n                                                'Spring' if x in [3, 4, 5] else\n                                                'Summer' if x in [6, 7, 8] else 'Fall')\n    \n    # Interaction Features\n    df['Claims_Per_Duration'] = df['Previous Claims'] / (df['Insurance Duration'] + 1e-5)\n    \n    # Behavioral and Demographic Indicators\n    df['Risk_Score'] = (\n    df['Health Score'].astype('float32') - \n    df['Smoking Status'].map({'Yes': 1, 'No': 0}).astype('float32') + \n    df['Exercise Frequency'].map({'Daily': 3, 'Weekly': 2, 'Monthly': 1, 'Rarely': 0}).astype('float32')\n)\n\n    df[\"Ratio_IncomeAge\"]   = np.clip(df[\"Annual Income\"] / df[\"Age\"], a_min = 1e-6, a_max = 1e9)\n \n\n    \n    df['Dependents_Affordability'] = df['Annual Income'] / (df['Number of Dependents'] + 1)\n    \n    # Derived Financial Metrics\n    df['Credit_to_Income_Ratio'] = df['Credit Score'] / (df['Annual Income'] + 1e-5)\n    \n    # Credit, Health, and Income Interactions\n    df[\"Overall Score\"]             = df[\"Credit Score\"] + df[\"Health Score\"]\n    df['Credit_x_Health'] = df['Credit Score'] * df['Health Score']\n    df['Credit_x_Income'] = df['Credit Score'] * df['Annual Income']\n    df['Health_x_Income'] = df['Health Score'] * df['Annual Income']\n    df['Credit_x_Health_x_Income'] = df['Credit Score'] * df['Health Score'] * df['Annual Income']\n    df['Credit_to_Health_Ratio'] = df['Credit Score'] / (df['Health Score'] + 1e-5)\n    df['Health_to_Income_Ratio'] = df['Health Score'] / (df['Annual Income'] + 1e-5)\n    df['Credit_to_Income_Ratio_Squared'] = (df['Credit Score'] / (df['Annual Income'] + 1e-5)) ** 2\n    df['Credit_Health_Income_Avg'] = (df['Credit Score'] + df['Health Score'] + df['Annual Income']) / 3\n    \n    # Vehicle-Related Features\n    df['Vehicle_Age_Group'] = pd.cut(df['Vehicle Age'], bins=[-1, 3, 10, float('inf')], \n                                      labels=['New', 'Midlife', 'Old'])\n    df['Policy_Type_Vehicle_Age'] = df['Policy Type'].astype(str) + \"_\" + df['Vehicle_Age_Group'].astype(str)\n\n    # Location-Based Features\n    if 'Location' in df.columns:\n        location_stats = df.groupby('Location')['Premium Amount'].mean().rename('Location_Avg_Premium')\n        df = df.merge(location_stats, on='Location', how='left')\n\n    \n    # Log Transformations for Skewed Features\n\n    df['Log Credit Score'] = np.log1p(df['Credit Score'])\n    \n    # Polynomial Features\n    df['Income_Squared'] = df['Annual Income'] ** 2\n    df['Credit_Score_x_Duration'] = df['Credit Score'] * df['Insurance Duration']\n\n    \n    # Some ideas that definitely didn't work:\n\n    # binned health scores -  bad score\n    #bin_edges = [0, 7, 23, 30, 45, 48, 51, 54, 56, float('inf')]\n    #df['Health Score Binned Integer'] = pd.cut(df['Health Score'], bins=bin_edges, right=False, labels=False).astype('object')\n\n    #date_num - bad score\n    #df['day_num'] = df['day'] + df['month'] * 30 + df['year'] * 365\n    #earliest = df['day_num'].min()\n    #df['day_num'] = df['day_num'] - earliest\n    \n    \n    return df\n\n\ntrain_df = train.copy()\ntrain_feats = extra_features(train_df)\n\ntest_df = train.copy()\ntest_feats = extra_features(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:28.36153Z","iopub.execute_input":"2024-12-31T15:18:28.361816Z","iopub.status.idle":"2024-12-31T15:18:35.388466Z","shell.execute_reply.started":"2024-12-31T15:18:28.361785Z","shell.execute_reply":"2024-12-31T15:18:35.387588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"colstokeep = ['Credit_Health_Income_Avg', 'Ratio_IncomeAge', 'Policy_Age']  # 'Credit_x_Health', 'Health_x_Income', 'Credit_to_Health_Ratio', 'Health_to_Income_Ratio', 'Credit_to_Income_Ratio_Squared', 'Credit_Health_Income_Avg', 'Vehicle_Age_Group', 'Policy_Type_Vehicle_Age', 'Log Credit Score', 'Credit_Score_x_Duration']\n# had highest impact on score and made most intuitive sense given the feature importance of income, credit score and health","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:35.389293Z","iopub.execute_input":"2024-12-31T15:18:35.389539Z","iopub.status.idle":"2024-12-31T15:18:35.392917Z","shell.execute_reply.started":"2024-12-31T15:18:35.389517Z","shell.execute_reply":"2024-12-31T15:18:35.392159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"new_cats = [\n    \"Season\",\n    \"Vehicle_Age_Group\",\n    \"Policy_Type_Vehicle_Age\",\n]\n\nnew_conts = [\n    \"Week\",\n    \"Weekday\",\n    \"DaySin\",\n    \"DayCos\",\n    \"WeekdaySin\",\n    \"WeekdayCos\",\n    \"Policy_Age\",\n    \"Claims_Per_Duration\",\n    \"Risk_Score\",\n    \"Ratio_IncomeAge\",\n    \"Dependents_Affordability\",\n    \"Credit_to_Income_Ratio\",\n    \"Overall Score\",\n    \"Credit_x_Health\",\n    \"Credit_x_Income\",\n    \"Health_x_Income\",\n    \"Credit_x_Health_x_Income\",\n    \"Credit_to_Health_Ratio\",\n    \"Health_to_Income_Ratio\",\n    \"Credit_to_Income_Ratio_Squared\",\n    \"Credit_Health_Income_Avg\",\n    \"Log Credit Score\",\n    \"Income_Squared\",\n    \"Location_Avg_Premium\",\n]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:35.393665Z","iopub.execute_input":"2024-12-31T15:18:35.393846Z","iopub.status.idle":"2024-12-31T15:18:35.408357Z","shell.execute_reply.started":"2024-12-31T15:18:35.393829Z","shell.execute_reply":"2024-12-31T15:18:35.407745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for col in colstokeep:\n    train[col] = train_feats[col]\n    test[col] = test_feats[col]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:35.41054Z","iopub.execute_input":"2024-12-31T15:18:35.410735Z","iopub.status.idle":"2024-12-31T15:18:35.486617Z","shell.execute_reply.started":"2024-12-31T15:18:35.410718Z","shell.execute_reply":"2024-12-31T15:18:35.48587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pp(df):\n    for col in cats:\n        df[col] = df[col].fillna('Other')\n        df[col] = df[col].astype('category')\n    \n    for col in conts:\n        print(col)\n        mean = df[col].mean()\n        df[col] = df[col].fillna(mean)\n        df[col] = df[col].astype('float32')\n\n    for col in new_conts:\n        if col in colstokeep:\n            df[col] = df[col].astype('float32')\n    \n    for col in new_cats:\n        if col in colstokeep:\n            df[col] = df[col].astype('category')\n\ndef coerce(df):\n    for col in cats:\n        df[col] = df[col].astype('category')\n    \n    for col in conts:\n        mean = df[col].mean()\n        df[col] = df[col].astype('float32')\n        \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:35.488557Z","iopub.execute_input":"2024-12-31T15:18:35.488881Z","iopub.status.idle":"2024-12-31T15:18:35.49453Z","shell.execute_reply.started":"2024-12-31T15:18:35.48885Z","shell.execute_reply":"2024-12-31T15:18:35.493643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"columns_with_na = train.columns[train.isna().any()].tolist()\n\n# Credit to https://www.kaggle.com/competitions/playground-series-s4e12/discussion/552165\n\nnewcats = []\nfor c in columns_with_na:\n    newcats.append(f\"is_{c}_na\")\n    train[f\"is_{c}_na\"] = train[c].isna()\n    test[f\"is_{c}_na\"] = test[c].isna()\n\n    train[f\"is_{c}_na\"] = train[f\"is_{c}_na\"].astype('category')\n    test[f\"is_{c}_na\"] = test[f\"is_{c}_na\"].astype('category')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:35.495255Z","iopub.execute_input":"2024-12-31T15:18:35.495482Z","iopub.status.idle":"2024-12-31T15:18:36.529603Z","shell.execute_reply.started":"2024-12-31T15:18:35.495462Z","shell.execute_reply":"2024-12-31T15:18:36.528837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pp(train)\npp(test)\n\ncats.extend(newcats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:36.530482Z","iopub.execute_input":"2024-12-31T15:18:36.53071Z","iopub.status.idle":"2024-12-31T15:18:39.047855Z","shell.execute_reply.started":"2024-12-31T15:18:36.53069Z","shell.execute_reply":"2024-12-31T15:18:39.047028Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Finishing Preprocessing","metadata":{}},{"cell_type":"code","source":"X = train.copy()\nX.drop(dep, axis = 1, inplace = True)\n\n\n\ny = train.copy()\ny = y[dep]\ny = np.log(y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.048583Z","iopub.execute_input":"2024-12-31T15:18:39.048782Z","iopub.status.idle":"2024-12-31T15:18:39.242736Z","shell.execute_reply.started":"2024-12-31T15:18:39.048763Z","shell.execute_reply":"2024-12-31T15:18:39.241996Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# LGB: Cross Validation / Tuning","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    params = {\n        'boosting_type': 'gbdt',\n        'num_leaves': trial.suggest_int('num_leaves', 20, 150),\n        'learning_rate': trial.suggest_float('learning_rate', 0.005, 0.8),\n        'feature_fraction': trial.suggest_float('feature_fraction', 0.5, 1.0),\n        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.5, 1.0),\n        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),\n        'min_child_samples': trial.suggest_int('min_child_samples', 5, 150),\n        'lambda_l1': trial.suggest_float('lambda_l1', 0, 7),\n        'lambda_l2': trial.suggest_float('lambda_l2', 0, 7),\n        'verbose': -1,\n        'random_state': 42,\n        'device' : 'gpu'\n    }\n    model = lgb.LGBMRegressor(**params, n_estimators = 200)\n\n\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    for i, (train_idx, val_idx) in enumerate(kfold.split(X)):\n        X_train = X.iloc[train_idx, :]\n        y_train = y.iloc[train_idx]\n    \n        X_val = X.iloc[val_idx, :]\n        y_val = y.iloc[val_idx]\n    \n        model.fit(X_train, y_train)\n        fold_preds = model.predict(X_val)\n    \n        score =  mean_squared_log_error(np.exp(y_val), np.exp(fold_preds), squared = False)\n        \n        print(f\"fold {i} score: {score}\")\n        scores.append(score)\n\n    print(f\"\"\"\n    \n    MEAN SCORE: {np.mean(scores)}\n    \n    \"\"\")\n    \n    return np.mean(scores)\n\nif tune_lgb:\n    \n\n    study = optuna.create_study(direction=\"minimize\")\n    study.optimize(objective, n_trials=14)\n    best_lgbm_params = study.best_params\n    print(\"Best parameters:\", study.best_params)\n    print(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.243564Z","iopub.execute_input":"2024-12-31T15:18:39.243767Z","iopub.status.idle":"2024-12-31T15:18:39.251378Z","shell.execute_reply.started":"2024-12-31T15:18:39.243749Z","shell.execute_reply":"2024-12-31T15:18:39.250405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# XGB: Cross Validation / Tuning","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n    param = {\n        'n_estimators': trial.suggest_int('iterations', 200, 2000),\n        'max_depth': trial.suggest_int('max_depth', 2, 10),\n        'learning_rate': trial.suggest_float('learning_rate', 1e-4, 0.2, log=True),\n        'subsample': trial.suggest_float('subsample', 0.5, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.4, 0.9),\n        'gamma': trial.suggest_float('gamma', 0, 5),\n        'reg_alpha': trial.suggest_float('reg_alpha', 1e-4, 1.0, log=True),\n        'reg_lambda': trial.suggest_float('reg_lambda', 1e-4, 1.0, log=True),\n        'random_state': 42,\n        'use_label_encoder': False,\n        'enable_categorical':True,\n        \"tree_method\":\"hist\",\n        \"device\": \"cuda\"\n\n    }\n    model = xgb.XGBRegressor(**param)\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    for i, (train_idx, val_idx) in enumerate(kfold.split(X)):\n        X_train = X.iloc[train_idx, :]\n        y_train = y.iloc[train_idx]\n    \n        X_val = X.iloc[val_idx, :]\n        y_val = y.iloc[val_idx]\n    \n        model.fit(X_train, y_train)\n        fold_preds = model.predict(X_val)\n    \n        score =  mean_squared_log_error(np.exp(y_val), np.exp(fold_preds), squared = False)\n        \n        print(f\"fold {i} score: {score}\")\n        scores.append(score)\n\n    print(f\"\"\"\n    \n    MEAN SCORE: {np.mean(scores)}\n    \n    \"\"\")\n    \n    return np.mean(scores)\n\nif tune_xgb:\n    \n\n    study = optuna.create_study(direction=\"minimize\")\n    study.optimize(objective, n_trials=14)\n    best_lgbm_params = study.best_params\n    print(\"Best parameters:\", study.best_params)\n    print(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.252252Z","iopub.execute_input":"2024-12-31T15:18:39.25253Z","iopub.status.idle":"2024-12-31T15:18:39.267587Z","shell.execute_reply.started":"2024-12-31T15:18:39.252508Z","shell.execute_reply":"2024-12-31T15:18:39.266703Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Catboost: Cross Validation / Tuning","metadata":{}},{"cell_type":"code","source":"def objective(trial):\n\n    params = {\n        'iterations': trial.suggest_int('iterations', 500, 2000),\n        'learning_rate': 0.05,\n        'depth': trial.suggest_int('depth', 6, 8),\n        'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 0.4, 1.0),\n        'logging_level': 'Silent',\n        'task_type': 'GPU',\n        'loss_function' : 'RMSE',\n        'cat_features' : cats, \n        'random_seed': 42, # hyperparm optuna range inspired by https://www.kaggle.com/code/backpaker/rid-catboost-nonlog-as-feature\n    }\n    model = cb.CatBoostRegressor(**params)\n    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n    scores = []\n    for i, (train_idx, val_idx) in enumerate(kfold.split(X)):\n        X_train = X.iloc[train_idx, :]\n        y_train = y.iloc[train_idx]\n    \n        X_val = X.iloc[val_idx, :]\n        y_val = y.iloc[val_idx]\n    \n        model.fit(X_train, y_train)\n        fold_preds = model.predict(X_val)\n    \n        score =  mean_squared_log_error(np.exp(y_val), np.exp(fold_preds), squared = False)\n        \n        print(f\"fold {i} score: {score}\")\n        scores.append(score)\n\n    print(f\"\"\"\n    \n    MEAN SCORE: {np.mean(scores)}\n    \n    \"\"\")\n    \n    return np.mean(scores)\n\nif tune_cat:\n    \n\n    study = optuna.create_study(direction=\"minimize\")\n    study.optimize(objective, n_trials=10)\n    best_lgbm_params = study.best_params\n    print(\"Best parameters:\", study.best_params)\n    print(\"Best RMSLE:\", study.best_value)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.268407Z","iopub.execute_input":"2024-12-31T15:18:39.268701Z","iopub.status.idle":"2024-12-31T15:18:39.285593Z","shell.execute_reply.started":"2024-12-31T15:18:39.268671Z","shell.execute_reply":"2024-12-31T15:18:39.284534Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"markdown","source":"## LGBM","metadata":{}},{"cell_type":"code","source":"lgb1_params = {'num_leaves': 110, 'learning_rate': 0.05505967648696841, 'feature_fraction': 0.8312501494642098, 'bagging_fraction': 0.9763215599946862, 'bagging_freq': 5, 'min_child_samples': 22, 'lambda_l1': 0.412011914549929, 'lambda_l2': 0.31104102451998483}\n\n\nlgbm1 =  lgb.LGBMRegressor( boosting_type= 'gbdt',n_estimators = 200, random_seed=42, verbose = -1, device = 'gpu', **lgb1_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.28631Z","iopub.execute_input":"2024-12-31T15:18:39.286571Z","iopub.status.idle":"2024-12-31T15:18:39.303695Z","shell.execute_reply.started":"2024-12-31T15:18:39.286552Z","shell.execute_reply":"2024-12-31T15:18:39.302794Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm1.fit(X, y)\nlgbm1_preds = lgbm1.predict(test)\nlgbm1_preds = np.exp(lgbm1_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:18:39.30448Z","iopub.execute_input":"2024-12-31T15:18:39.304726Z","iopub.status.idle":"2024-12-31T15:19:10.514545Z","shell.execute_reply.started":"2024-12-31T15:18:39.304707Z","shell.execute_reply":"2024-12-31T15:19:10.513758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb.plot_importance(lgbm1, max_num_features=20, importance_type='split') \nplt.title(\"Feature Importances\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:10.515516Z","iopub.execute_input":"2024-12-31T15:19:10.515838Z","iopub.status.idle":"2024-12-31T15:19:10.915365Z","shell.execute_reply.started":"2024-12-31T15:19:10.515805Z","shell.execute_reply":"2024-12-31T15:19:10.914457Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## LGBM 2","metadata":{}},{"cell_type":"code","source":"lgb2_params = {'num_leaves': 116, 'learning_rate': 0.02528610350834941, 'feature_fraction': 0.7580194170276913, 'bagging_fraction': 0.8425453238597505, 'bagging_freq': 7, 'min_child_samples': 59, 'lambda_l1': 4.430136968344415, 'lambda_l2': 4.7475336323919395}\n\nlgbm2 =  lgb.LGBMRegressor( boosting_type= 'gbdt',n_estimators = 200, random_seed=42, verbose = -1, device = 'gpu', **lgb2_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:10.916265Z","iopub.execute_input":"2024-12-31T15:19:10.916604Z","iopub.status.idle":"2024-12-31T15:19:10.920774Z","shell.execute_reply.started":"2024-12-31T15:19:10.91657Z","shell.execute_reply":"2024-12-31T15:19:10.9199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgbm2.fit(X, y)\nlgbm2_preds = lgbm2.predict(test)\nlgbm2_preds = np.exp(lgbm2_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:10.921502Z","iopub.execute_input":"2024-12-31T15:19:10.921736Z","iopub.status.idle":"2024-12-31T15:19:43.188277Z","shell.execute_reply.started":"2024-12-31T15:19:10.921716Z","shell.execute_reply":"2024-12-31T15:19:43.187555Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## XGBM","metadata":{}},{"cell_type":"code","source":"xgb_params = {\n        'iterations': 808, \n        'max_depth': 7, \n        'learning_rate': 0.01801023148218898, \n        'subsample': 0.6741856229679314, \n        'colsample_bytree': 0.7011128102800052, \n        'gamma': 0.7175981877013271, \n        'reg_alpha': 0.000551347715455769, \n        'reg_lambda': 0.029819254786464938,\n        'random_state': 42,\n        'use_label_encoder': False,\n        'enable_categorical':True,\n        \"tree_method\":\"hist\",\n        \"device\": \"cuda\"\n\n    }\nxgbm = xgb.XGBRegressor(**xgb_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:43.189183Z","iopub.execute_input":"2024-12-31T15:19:43.18956Z","iopub.status.idle":"2024-12-31T15:19:43.1941Z","shell.execute_reply.started":"2024-12-31T15:19:43.189523Z","shell.execute_reply":"2024-12-31T15:19:43.193081Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgbm.fit(X, y)\nxgbm_preds = xgbm.predict(test)\nxgbm_preds = np.exp(xgbm_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:43.195108Z","iopub.execute_input":"2024-12-31T15:19:43.195321Z","iopub.status.idle":"2024-12-31T15:19:49.731196Z","shell.execute_reply.started":"2024-12-31T15:19:43.195301Z","shell.execute_reply":"2024-12-31T15:19:49.730241Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CatBoost","metadata":{}},{"cell_type":"code","source":"cb_params = {\n        'iterations': 603,\n        'learning_rate': 0.05,\n        'depth': 8, \n        'l2_leaf_reg': 0.43024600177681127,\n        'logging_level': 'Silent',\n        'task_type': 'GPU',\n        'loss_function' : 'RMSE',\n        'cat_features' : cats, \n        'random_seed': 42, # hyperparm optuna range inspired by https://www.kaggle.com/code/backpaker/rid-catboost-nonlog-as-feature\n}\ncbm = cb.CatBoostRegressor(**cb_params)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:49.732208Z","iopub.execute_input":"2024-12-31T15:19:49.732481Z","iopub.status.idle":"2024-12-31T15:19:49.738517Z","shell.execute_reply.started":"2024-12-31T15:19:49.732458Z","shell.execute_reply":"2024-12-31T15:19:49.737699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cbm.fit(X, y)\ncbm_preds = cbm.predict(test)\ncbm_preds = np.exp(cbm_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:19:49.739322Z","iopub.execute_input":"2024-12-31T15:19:49.739606Z","iopub.status.idle":"2024-12-31T15:21:25.436981Z","shell.execute_reply.started":"2024-12-31T15:19:49.739585Z","shell.execute_reply":"2024-12-31T15:21:25.436148Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Blending","metadata":{}},{"cell_type":"code","source":"preds = 0.85 * lgbm1_preds + 0.05 * lgbm2_preds + 0.05 * xgbm_preds + 0.05 * cbm_preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:21:25.437849Z","iopub.execute_input":"2024-12-31T15:21:25.438167Z","iopub.status.idle":"2024-12-31T15:21:25.449628Z","shell.execute_reply.started":"2024-12-31T15:21:25.438132Z","shell.execute_reply":"2024-12-31T15:21:25.448711Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/playground-series-s4e12/sample_submission.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:21:25.453257Z","iopub.execute_input":"2024-12-31T15:21:25.453554Z","iopub.status.idle":"2024-12-31T15:21:25.721291Z","shell.execute_reply.started":"2024-12-31T15:21:25.453531Z","shell.execute_reply":"2024-12-31T15:21:25.720529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub[dep] = preds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:21:25.722493Z","iopub.execute_input":"2024-12-31T15:21:25.722819Z","iopub.status.idle":"2024-12-31T15:21:25.727508Z","shell.execute_reply.started":"2024-12-31T15:21:25.722788Z","shell.execute_reply":"2024-12-31T15:21:25.726693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.to_csv('submission.csv', index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:21:25.728448Z","iopub.execute_input":"2024-12-31T15:21:25.728657Z","iopub.status.idle":"2024-12-31T15:21:27.090974Z","shell.execute_reply.started":"2024-12-31T15:21:25.728638Z","shell.execute_reply":"2024-12-31T15:21:27.090246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T15:21:27.091881Z","iopub.execute_input":"2024-12-31T15:21:27.09219Z","iopub.status.idle":"2024-12-31T15:21:27.105113Z","shell.execute_reply.started":"2024-12-31T15:21:27.092159Z","shell.execute_reply":"2024-12-31T15:21:27.104402Z"}},"outputs":[],"execution_count":null}]}