{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ✨*Data Exploration*💰  \n\n> *Looking each feature of dataset*  \n\n---","metadata":{"_uuid":"bd39a901-0939-4903-9b07-4b70c6c8d0c7","_cell_guid":"a1f3cecf-f596-49ca-81bc-2fd856cfa290","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## ✨*Preparation*💰  \n\n> *Import modules and read data*  \n\n---","metadata":{"_uuid":"74cfc03a-9e8e-4d38-9ae1-10e083aba6c9","_cell_guid":"5607eddf-86ef-4abd-b5bc-b2f7f50db8c5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# import modules\nimport polars as pl\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_log_error\n\n# import optuna.integration.lightgbm as lgb\nimport lightgbm as lgb\n\nimport shap\n\nsns.set_theme(context=\"notebook\", style=\"whitegrid\", palette=\"Set2\")\nwarnings.simplefilter(action=\"ignore\", category=FutureWarning)\n\nimport time","metadata":{"_uuid":"ab90ba6d-14b1-42ec-910f-5f53df317c20","_cell_guid":"b059531e-08a5-439a-be5e-84d42dfa7672","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Stop watch\nstart_time = time.time()\n\n# data reading\ndf = (\n    pl.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\")\n    # pl.read_csv(\"train.csv\")\n    .drop(\"id\")\n)\n\ndf_obj = pl.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n# df_obj = pl.read_csv(\"test.csv\")\ndf_obj_id = df_obj.select(\"id\")\ndf_obj = df_obj.drop(\"id\")","metadata":{"_uuid":"eab5d676-e8ee-44c1-bd57-4590845b8be3","_cell_guid":"8fe2a07d-cc72-468e-a382-4f389a411371","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Data Handling class definition*💰  \n\n> *Class definition for handling features*  \n\n---","metadata":{"_uuid":"cdaf93bb-5966-4647-a049-ad246dea78ac","_cell_guid":"43751ef0-0fce-4c6e-888f-ba7a2e665a4b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Data preprocessing class\nclass CustomTramsformer:\n    def __init__(self, target):\n        self.target = target\n        self.ordered_elements = {}\n        self.not_need_other = {}\n        self.feature_threshold = {}\n        self.scaler = StandardScaler()\n        self.encoder = OneHotEncoder(drop=\"first\")\n        self.ct = None\n        self.categories = None\n        self.ct_feature_names = None\n        self.num_features = None\n        self.cat_features = None\n\n    def reset_features(self, df: pl.DataFrame):\n        self.num_features = [\n            col\n            for col in df.columns\n            if df[col].dtype in [pl.Int64, pl.Float64, pl.Int32]\n        ]\n        self.cat_features = [col for col in df.columns if df[col].dtype in [pl.String]]\n\n    def set_threshold(\n        self, df1: pl.DataFrame, df2: pl.DataFrame, feature: str, val: int\n    ):\n        self.feature_threshold[feature] = val\n        df1 = self.set_categories_of_feature(df1, feature, val)\n        df2 = self.set_categories_of_feature(df2, feature)\n\n        if (\"other\" not in df1[feature].unique()) & (\n            \"other\" not in df2[feature].unique()\n        ):\n            self.not_need_other[feature] = True\n        else:\n            self.not_need_other[feature] = False\n\n        if self.not_need_other[feature]:\n            self.ordered_elements[feature] = [\n                x for x in self.ordered_elements[feature] if x != \"other\"\n            ]\n        return df1, df2\n\n    def set_categories_of_feature(\n        self, df: pl.DataFrame, feature: str, n_items: int = 0\n    ):\n        if n_items != 0:\n            counts = df[feature].value_counts().sort(by=\"count\", descending=True)\n            self.ordered_elements[feature] = [\n                col for col, ct in counts.to_numpy() if ct >= n_items\n            ]\n\n        df = df.with_columns(\n            pl.when(pl.col(feature).is_in(self.ordered_elements[feature]))\n            .then(pl.col(feature))\n            .when(pl.col(feature).is_null())\n            .then(pl.col(feature))\n            .otherwise(pl.lit(\"other\"))\n            .alias(feature)\n        )\n\n        if feature not in self.not_need_other.keys():\n            self.not_need_other[feature] = False\n\n        if (\n            not self.not_need_other[feature]\n            and \"other\" not in self.ordered_elements[feature]\n        ):\n            self.ordered_elements[feature].append(\"other\")\n        return df\n\n    def set_ct(self):\n        num_wo_target = [val for val in self.num_features if val != self.target]\n        self.categories = [self.ordered_elements[col] for col in self.cat_features]\n        self.encoder = OneHotEncoder(drop=\"first\", categories=self.categories)\n\n        self.ct = ColumnTransformer(\n            [\n                (\"scaler\", self.scaler, num_wo_target),\n                (\"encoder\", self.encoder, self.cat_features),\n            ],\n            remainder=\"passthrough\",\n        )\n\n    def ct_transform(self, df: pl.DataFrame):\n        if self.target in df.columns:\n            df = df.drop(self.target)\n\n        data_scaled = self.ct.fit_transform(\n            df.to_pandas(use_pyarrow_extension_array=True)\n        )\n        if self.categories is None:\n            self.categories = self.ct.named_transformers_[\"encoder\"].categories_\n        if self.ct_feature_names is None:\n            self.ct_feature_names = self.ct.get_feature_names_out().tolist()\n        return data_scaled\n\n    def pipeline(self, df: pl.DataFrame, set_categories: bool = False):\n        cols = self.cat_features\n        if set_categories:\n            n_threshes = []\n            for col in cols:\n                if col in self.feature_threshold.keys():\n                    val = self.feature_threshold[col]\n                else:\n                    val = 1\n                n_threshes.append(val)\n        else:\n            n_threshes = [0 for _ in range(len(cols))]\n\n        for col, thresh in zip(cols, n_threshes):\n            df = self.set_categories_of_feature(df, col, thresh)\n\n        if set_categories:\n            self.set_ct()\n        return df","metadata":{"_uuid":"6b97989c-8835-4d8b-8c4c-88a526246ba7","_cell_guid":"38503a1c-cc81-4dd6-b185-9640554c113c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# function to plot cat features\ndef plot_cat(df1: pl.DataFrame, df2: pl.DataFrame, feature: str, order=None):\n\n    if order is None:\n        order = ut.ordered_elements[feature]\n\n    palette = dict(zip(order, sns.color_palette(\"Set2\")[: len(order)]))\n    _, ax = plt.subplots(1, 3, figsize=(10, 3), tight_layout=True)\n    plt.subplots_adjust(right=0.85)\n    sns.violinplot(\n        df1.to_pandas(),\n        x=feature,\n        y=ut.target,\n        ax=ax[0],\n        order=order,\n        palette=palette,\n        # showfliers=False,\n    )\n    sns.countplot(df1.to_pandas(), x=feature, ax=ax[1], order=order, palette=palette)\n    sns.countplot(df2.to_pandas(), x=feature, ax=ax[2], order=order, palette=palette)\n\n    for i, ylabel in enumerate([\"Premium\", \"train cnt.\", \"test cnt.\"]):\n        ax[i].set_ylabel(ylabel)\n        ax[i].set_xlabel(\"\")\n        # ax[i].set_xticks(order)\n        ax[i].set_xticklabels(order, fontsize=11, rotation=90)\n    plt.suptitle(feature)\n    plt.show()\n\n    if order is None:\n        order = ut.ordered_elements[feature]","metadata":{"_uuid":"2deaa008-6ee7-46b0-acdd-268c19eadfca","_cell_guid":"30b0844c-5453-4e17-8386-0cf8198418d7","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# function to plot num features\ndef plot_num(df1: pl.DataFrame, df2: pl.DataFrame, feature: str):\n    if feature == ut.target:\n        return\n\n    min_val = min(df1[feature].min(), df2[feature].min())\n    max_val = max(df1[feature].max(), df2[feature].max())\n    bins = np.linspace(min_val, max_val, 31)\n\n    _, ax = plt.subplots(1, 3, figsize=(10, 3), tight_layout=True)\n    # sns.scatterplot(df1.to_pandas(), x=feature, y=ut.target, alpha=0.1, ax=ax[0])\n\n    df1_bin = (\n        df1.with_columns(\n            pl.col(feature)\n            .cut(bins, labels=[str(val) for val in bins] + [f\"{bins[-1]} ~\"])\n            .alias(\"bin\")\n        )\n        .with_columns(pl.col(ut.target).mean().over(\"bin\").alias(\"mean\"))\n        .with_columns(pl.col(ut.target).quantile(0.05).over(\"bin\").alias(\"qt-05\"))\n        .with_columns(pl.col(ut.target).quantile(0.25).over(\"bin\").alias(\"qt-25\"))\n        .with_columns(pl.col(ut.target).quantile(0.75).over(\"bin\").alias(\"qt-75\"))\n        .with_columns(pl.col(ut.target).quantile(0.95).over(\"bin\").alias(\"qt-95\"))\n        .select([feature, \"mean\", \"qt-05\", \"qt-25\",\"qt-75\", \"qt-95\",])\n        .unique()\n        .sort(feature)\n    )\n\n    ax[0].fill_between(\n        df1_bin.to_pandas()[feature],\n        df1_bin.to_pandas()[\"qt-05\"],\n        df1_bin.to_pandas()[\"qt-95\"],\n        color=\"#ddffdd\",\n        alpha=0.5,\n    )\n    ax[0].fill_between(\n        df1_bin.to_pandas()[feature],\n        df1_bin.to_pandas()[\"qt-25\"],\n        df1_bin.to_pandas()[\"qt-75\"],\n        color=\"#aaeeaa\",\n        alpha=0.5,\n    )\n    sns.lineplot(df1_bin.to_pandas(), x=feature, y=\"mean\", ax=ax[0])\n    sns.histplot(df1.to_pandas(), x=feature, bins=bins, ax=ax[1])\n    sns.histplot(df2.to_pandas(), x=feature, bins=bins, ax=ax[2])\n    for i, ylabel in enumerate([\"Premium\", \"train cnt.\", \"test cnt.\"]):\n        ax[i].set_ylabel(ylabel)\n    plt.suptitle(feature)\n    plt.show()","metadata":{"_uuid":"66f49aa2-66dc-4406-8051-e375a9cbaf2a","_cell_guid":"3b4fa78f-4b8e-4823-8cb4-eee62826cf00","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Null Data Check*💰  \n\n> *There are many nulls. It's appropriate to replace null to other value. but null values at \"Financial Stress\" is only in train data, so they could be droped.*  \n\n---","metadata":{"_uuid":"2cf16f5d-57b5-4786-80c7-398f6f0f269b","_cell_guid":"328375fd-ac8b-4815-944e-e6ba45f2a5f8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def nullcheck(df: pl.DataFrame, title: str):\n    has_null = False\n    df_nullcheck = pl.DataFrame()\n    for col in df.columns:\n        nc = df[col].null_count()\n        if nc >= 1:\n            df_nullcheck = pl.concat(\n                [\n                    df_nullcheck,\n                    pl.DataFrame(\n                        {\"feature\": col, \"nulls\": nc, \"nulls_ratio\": nc / len(df)}\n                    ),\n                ]\n            )\n            # print(f\"{col} has {nc} nulls. ({nc/len(df)*100:.1f} %)\")\n            has_null = True\n\n    if has_null:\n        colors = sns.color_palette(\"pastel\")\n        plt.figure(figsize=(8, 3))\n        ax = sns.barplot(\n            df_nullcheck.to_pandas(), y=\"feature\", x=\"nulls\", palette=colors\n        )\n        for container in ax.containers:\n            ax.bar_label(container, fmt=\"{:,.0f}\", padding=3, fontsize=10)\n        plt.title(f\"Nulls check ({title})\")\n        plt.show()\n    else:\n        print(f\"dateframe {title} has no null.\")\n\n\nnullcheck(df, \"train\")\nnullcheck(df_obj, \"test\")","metadata":{"_uuid":"7fcde96e-fbdb-4920-acbb-efc9a83170d5","_cell_guid":"94295e1c-8682-4919-a22a-d45e29518079","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## ✨*Feature handling*💰  \n\n> *Looking each feature of dataset*  \n\n---","metadata":{"_uuid":"51457dda-6d41-4219-9dc8-5345add78491","_cell_guid":"c9d24844-a07c-4ca0-991f-c482027e4494","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"ut = CustomTramsformer(\"Premium Amount\")\nut.reset_features(df)\ndf = ut.pipeline(df, True)","metadata":{"_uuid":"09a8f5e7-3605-4ce5-9273-33c43853c358","_cell_guid":"3bb60120-46bf-4cd2-9e05-4d937863199e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Add Some features*💰  \n\n> *Added multiplication and division of features that seem to have a large impact*  \n\n---","metadata":{}},{"cell_type":"code","source":"# Extract year, month, day from Policy Start Date\ndef handle_policy_start(df: pl.DataFrame):\n    feature = \"Policy Start Date\"\n    df = df.with_columns(\n        pl.col(feature).str.extract(r\"^(\\d+)-\").cast(pl.Int32).alias(\"Start Year\"),\n        pl.col(feature).str.extract(r\"^\\d+-(\\d+)\").cast(pl.Int32).alias(\"Start Month\"),\n        pl.col(feature)\n        .str.extract(r\"^\\d+-\\d+-(\\d+) \")\n        .cast(pl.Int32)\n        .alias(\"Start Day\"),\n    ).drop(feature)\n    return df\n\n\ndf = handle_policy_start(df)\ndf_obj = handle_policy_start(df_obj)\nut.reset_features(df)","metadata":{"_uuid":"3283164e-e0d0-43e0-b3a6-c69e8fafcf40","_cell_guid":"c53cb2c3-27ed-466e-be5b-9d6b23feb3b4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# create some features\ndef create_feature(df: pl.DataFrame):\n    df = df.with_columns(\n        (pl.col(\"Annual Income\") * pl.col(\"Previous Claims\")).alias(\"Income x Claims\"),\n        (pl.col(\"Annual Income\") / pl.col(\"Health Score\")).alias(\"Income Health ratio\"),\n        (pl.col(\"Annual Income\") / pl.col(\"Credit Score\")).alias(\"Income Credit ratio\"),\n        (pl.col(\"Annual Income\") * pl.col(\"Credit Score\")).alias(\"Income x Credit\"),\n        (pl.col(\"Credit Score\") * pl.col(\"Health Score\")).alias(\"Credit x Health\"),\n        # (pl.col(\"Credit Score\") * pl.col(\"Previous Claims\")).alias(\"Credit x Claims\"),\n    )\n    return df\n\n\ndf = create_feature(df)\ndf_obj = create_feature(df_obj)\nut.reset_features(df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Remove Some features*💰  \n\n> *Removed features with very small impact of change*  \n\n---","metadata":{}},{"cell_type":"code","source":"# drop some features\n\ndroplist = [\n    \"Gender\",\n    \"Education Level\",\n    \"Location\",\n    \"Policy Type\",\n    \"Smoking Status\",\n    \"Exercise Frequency\",\n    \"Property Type\",\n    \"Number of Dependents\",\n    \"Start Month\",\n]\ndf = df.drop(droplist)\ndf_obj = df_obj.drop(droplist)\nut.reset_features(df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Target Value*💰  \n\n> *It seems to have three peaks.*  \n\n---","metadata":{"_uuid":"d2987892-c0fb-44d0-ae55-36c7d23d23ee","_cell_guid":"d6471b9a-000e-4087-975e-2122cc776505","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# target value\nprint(\"<< target value >>\")\nplt.figure(figsize=(10, 4))\nsns.histplot(df.to_pandas(), x=ut.target, bins=50)\nplt.show()","metadata":{"_uuid":"1ccaef72-1ac1-4b03-884d-026c10a40ce2","_cell_guid":"3ac557e2-49bd-482f-8fdf-8d62527fdef6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Categorical features*💰  \n\n---","metadata":{}},{"cell_type":"code","source":"# Handle nulls and plot Categorical features\n\n# Replace null value to mode\ndef handle_null_of_cat_features(df: pl.DataFrame, feature: str):\n    df = df.with_columns(pl.col(feature).fill_null(pl.col(feature).mode().first()))\n    df = df.with_columns(pl.col(feature).fill_null(pl.lit(\"Unknown\")))\n    return df\n\n\n# violinplot and countplot of train & test data\nfor feature in ut.cat_features:\n    df = handle_null_of_cat_features(df, feature)\n    df_obj = handle_null_of_cat_features(df_obj, feature)\n    ut.reset_features(df)\n    ut.set_threshold(df, df_obj, feature, 1)\n    plot_cat(df, df_obj, feature)","metadata":{"_uuid":"d15f3480-8c58-4cb7-a3f8-17d7c2c61666","_cell_guid":"4972a049-cf29-48b6-93ae-97f70e835907","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Numeric features*💰　　\n\n---","metadata":{"_uuid":"f016949a-08d2-4861-9dd9-8ab30c628fe0","_cell_guid":"35271940-33f5-4a55-b791-908f076a6de5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Handle nulls and plot Numerical features\n\n# Replace null to mean value.\ndef handle_null_of_num_features(df: pl.DataFrame, feature: str):\n    df = df.with_columns(\n        pl.col(feature).fill_null(pl.col(feature).mean()).alias(feature)\n    )\n    return df\n\n\n# lineplot and histgram of train & test data\nfor feature in [val for val in ut.num_features if val != ut.target]:\n    df = handle_null_of_num_features(df, feature)\n    df_obj = handle_null_of_num_features(df_obj, feature)\n    plot_num(df, df_obj, feature)","metadata":{"_uuid":"4cf012e0-9df2-4bbd-95d9-fc3eb1e70ec2","_cell_guid":"d8d20efc-2eea-4b88-8c6a-fc183a94869e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Re: Null check\nnullcheck(df, \"train\")\nnullcheck(df_obj, \"test\")","metadata":{"_uuid":"b2a8fbb2-2d80-47de-ae70-367e6360f60d","_cell_guid":"ec5fcaf2-9a24-40c8-8af2-d02c12687bf5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Co-reration of numerical features*💰　　\n\n> *Plotting heatmap*　　\n\n---","metadata":{"_uuid":"c68d69c3-26a1-4803-b074-fb9740c8f311","_cell_guid":"ba272266-6a61-4a80-b724-b9e869958619","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 1, figsize=(10, 10), tight_layout=True)\nsns.heatmap(\n    df.select(ut.num_features).to_pandas().corr(),\n    cmap=\"coolwarm\",\n    annot=True,\n    fmt=\".2f\",\n    vmin=-1,\n    vmax=1,\n    center=0,\n    square=True,\n    linewidths=2,\n    linecolor=\"white\",\n    cbar_kws={\"shrink\": 0.7},\n)\nplt.show()","metadata":{"_uuid":"f26f9531-3e85-43b2-a3ca-af348e67d157","_cell_guid":"804bcd0d-e96f-401e-9d4d-91370618a2f5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ✨*Model creation and Prediction*💰　　\n\n> *LightGBM regression with Oof cross validation.*　　\n\n---","metadata":{"_uuid":"c0f6017c-17c3-4f21-9a18-3bf7e062f059","_cell_guid":"0c2c8fdc-f3ca-442b-9d8c-57abda5b8490","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# One hot encoding \nut.pipeline(df, True)\nX = ut.ct_transform(df)\ny = df.select(ut.target).to_numpy()\ny_log = np.log1p(y)","metadata":{"_uuid":"b5b57034-4d7d-4b67-9fb3-8e50d6d04501","_cell_guid":"d70e320e-b410-4d07-b3b5-7bff3732a802","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# definision of training function\n\ndef rmsle(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\n\ndef train_model(X, y):\n    kf = KFold(n_splits=5, shuffle=True, random_state=42)\n    oof = np.zeros(len(X))\n    models = []\n\n    for fold, (train_idx, valid_idx) in enumerate(kf.split(X)):\n        print(f\"Fold {fold + 1}\")\n        X_train, X_valid = X[train_idx], X[valid_idx]\n        y_train, y_valid = y[train_idx], y[valid_idx]\n\n        params_lgb = {\n            \"objective\": \"regression\",\n            \"metric\": \"rmse\",\n            \"random_state\": 42,\n            \"verbose\": -1,\n            \"learning_rate\": 0.005,\n            \"max_depth\": -1,\n            \"lambda_l1\": 8.061284137884293,\n            \"lambda_l2\": 0.1808845183357011,\n            \"num_leaves\": 53,\n            \"feature_fraction\": 0.8,\n            \"min_child_samples\": 50,\n        }\n\n        train_data = lgb.Dataset(X_train, label=y_train)\n        valid_data = lgb.Dataset(X_valid, label=y_valid)\n\n        model = lgb.train(\n            params_lgb,\n            train_data,\n            num_boost_round=10000,\n            valid_sets=[train_data, valid_data],\n            callbacks=[\n                lgb.early_stopping(stopping_rounds=500, verbose=True),\n                lgb.log_evaluation(250),\n            ],\n        )\n\n        models.append(model)\n\n        oof[valid_idx] = np.maximum(0, model.predict(X_valid))\n        fold_rmsle = rmsle(np.expm1(y_valid), np.expm1(oof[valid_idx]))\n        print(f\"Fold {fold + 1} RMSLE: {fold_rmsle}\")\n\n    return models, oof","metadata":{"_uuid":"84e258bb-554d-47d9-9d24-0647cc5b6956","_cell_guid":"121886a5-819e-459b-afc8-b34419493595","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train models\nmodels, oof = train_model(X, y_log)","metadata":{"_uuid":"2f0b9ecb-d9ea-4ab3-a077-1fb5fde51103","_cell_guid":"be52c49e-dd33-4279-b695-2360be3c6c22","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# valid data result\npred_valid = np.expm1(oof)\nprint(rmsle(y, pred_valid))\n\nplt.figure(figsize=(6, 6))\nsns.scatterplot(x=y.flatten(), y=pred_valid, alpha=0.01)\nsns.lineplot(x=[0, y.max()], y=[0, y.max()], color=\"gray\", linestyle=\"--\", linewidth=1)\nplt.xlabel(\"Premium Amount (true)\")\nplt.ylabel(\"predicted values\")\nplt.title(\"True vs Pred of train data\")\nplt.show()","metadata":{"_uuid":"b0951397-9efb-4839-a39a-f4772d486279","_cell_guid":"d26615f4-201b-4218-ad80-3270885f64b3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Predicttion*💰  \n\n> *Predict Premium Amount of test data*  \n\n---","metadata":{"_uuid":"bb4dc2e9-4278-4205-a326-73908110c05f","_cell_guid":"0e3fe627-8548-47eb-ba1c-673112d8943a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"X_obj = ut.ct_transform(df_obj)\n\nobj_predictions = np.zeros(len(X_obj))\nfor model in models:\n    obj_predictions += np.maximum(0, np.expm1(model.predict(X_obj))) / len(models)","metadata":{"_uuid":"17ca098a-af16-4c71-b710-b9f0530435f8","_cell_guid":"a0a3bf89-e5be-4561-b592-a754bbe44391","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ✨*Making Submission data*💰  \n\n---","metadata":{"_uuid":"19e5ee42-8003-4ae6-b63d-69921aec8815","_cell_guid":"65b83e56-326e-4508-8a8b-38fe9f1a3fef","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"result = pl.concat(\n    [df_obj_id, pl.DataFrame(obj_predictions, schema=[\"Premium Amount\"])],\n    how=\"horizontal\",\n)\nresult.write_csv(\"submission.csv\")","metadata":{"_uuid":"213978ad-09c8-4ab5-ad0e-6424f8580902","_cell_guid":"098031ef-0f94-46a3-ad92-003586731a0e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# analyze model (model[0] as sample) with SHAP\ndef analyze_shap_values(model, X, model_type=\"lightgbm\"):\n    feature_names = ut.ct.get_feature_names_out()\n\n    explainer = shap.TreeExplainer(model)\n    shap_values = explainer.shap_values(X)\n\n    # SHAP値の基本的な可視化（サマリープロット）\n    shap.summary_plot(shap_values, X, feature_names=feature_names, show=False)\n    plt.show()\n\n\nanalyze_shap_values(models[0], np.random.permutation(X)[:3000])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"end_time = time.time()\nelapsed = end_time - start_time\nprint(f\"elapsed time : {elapsed//3600}h {elapsed//60}min {elapsed%60:.1f}sec\")","metadata":{"_uuid":"c3ee2945-df0b-43e7-8792-8cd4d7809292","_cell_guid":"fc9289ac-bf0a-4424-b661-59c84849ef1b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"449f6bb9-37e9-40ae-853a-922269617fed","_cell_guid":"2b9a865c-7c4f-4ac7-83f7-310c87eb8583","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}