{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.16","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n%pip install --upgrade pip\n%pip install xgboost polars pytorch_lightning\nimport time\nimport sys \nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom xgboost import XGBRegressor\nimport xgboost as xgb, time\n\nimport pickle\nimport polars as pl\nimport gc\n\nimport os\nimport warnings\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nfrom pytorch_lightning import (LightningDataModule, LightningModule, Trainer)\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, Timer\nfrom pytorch_lightning.loggers import WandbLogger\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import Dataset, DataLoader\nfrom pandas import read_parquet","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-16T05:28:34.464884Z","iopub.execute_input":"2025-04-16T05:28:34.465164Z","iopub.status.idle":"2025-04-16T05:28:46.741222Z","shell.execute_reply.started":"2025-04-16T05:28:34.465139Z","shell.execute_reply":"2025-04-16T05:28:46.735271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/playground-series-s4e12/train.csv\",)\ntest = pd.read_csv(\"/kaggle/input/playground-series-s4e12/test.csv\")\n\ncol_map = {}\nfor col in train.columns:\n  col_map[col] = '_'.join(col.split(' '))\ntrain = train.rename(columns=col_map)\ntest = test.rename(columns=col_map)\n\n## 处理日期变量\ndef pre_process(data):\n  data[\"Policy_Start_Date\"] = pd.to_datetime( data[\"Policy_Start_Date\"] )\n  data[\"year\"] = data[\"Policy_Start_Date\"].dt.year.astype(\"float32\")\n  data[\"month\"] = data[\"Policy_Start_Date\"].dt.month.astype(\"float32\")\n  data[\"day\"] = data[\"Policy_Start_Date\"].dt.day.astype(\"float32\")\n  data[\"dow\"] = data[\"Policy_Start_Date\"].dt.dayofweek.astype(\"float32\")\n  data[\"seconds\"] = (data[\"Policy_Start_Date\"].astype(\"int64\") // 10**9).astype(\"float32\")\n\n  return data\n\ntrain = pre_process(train)\ntest = pre_process(test)\ntrain = train.drop('Policy_Start_Date',axis=1)\ntest = test.drop('Policy_Start_Date',axis=1)\n\ntrain[\"target\"] = np.log1p( train[\"Premium_Amount\"])  ##np.log(train[\"Premium_Amount\"]+1)\ntrain.drop('Premium_Amount',axis=1,inplace=True)\n\nimport pandas as pd\n\n# 定义移除的列\ncolumns_to_remove = [\"id\", \"Policy Start Date\", \"Premium Amount\", \"target\"]\n\n# 从训练集和测试集中筛选出需要的特征列\nfeatures = [col for col in train.columns if col not in columns_to_remove]\n\n# 合并训练集和测试集\ncombined = pd.concat([train, test], axis=0, ignore_index=True)\n\n# 初始化分类特征和高基数特征列表\ncategorical_features = []\nhigh_cardinality_features = []\n\nprint(f\"THE {len(features)} BASIC features ARE:\")\n\n# 遍历特征列，处理数据类型和缺失值\nfor feature in features:\n    feature_type = \"numerical\"\n    unique_values_count = combined[feature].nunique()\n\n    # 处理分类特征\n    if combined[feature].dtype == \"object\":\n        categorical_features.append(feature)\n        combined[feature] = combined[feature].fillna(\"NAN\")\n        combined[feature], _ = combined[feature].factorize()\n        combined[feature] -= combined[feature].min()\n        feature_type = \"categorical\"\n\n    # 优化数据类型\n    if combined[feature].dtype == \"int64\":\n        combined[feature] = combined[feature].astype(\"int32\")\n    elif combined[feature].dtype == \"float64\":\n        combined[feature] = combined[feature].astype(\"float32\")\n\n    print(f\"{feature} ({feature_type}) with {unique_values_count} unique values\")\n\n    # 检测高基数特征\n    if unique_values_count >= 9:\n        high_cardinality_features.append(feature)\n\n# 分离训练集和测试集\ntrain = combined.iloc[:len(train)].copy()\ntest = combined.iloc[len(train):].reset_index(drop=True).copy()\n\nprint(\"\\nTHE FOLLOWING HAVE 9 OR MORE UNIQUE VALUES:\", high_cardinality_features)\n\nencoding_features = [\n    ['Annual_Income', 'Health_Score'], \n    ['Credit_Score', 'Health_Score'], \n    ['Customer_Feedback', 'Gender', 'Marital_Status', 'Occupation', 'Smoking_Status', 'year'],\n    ['Exercise_Frequency', 'Health_Score'],\n    ['Health_Score', 'Marital_Status'],\n    ['Education_Level', 'Gender', 'Health_Score'],\n    ['Health_Score', 'Occupation'],\n    ['Age', 'Health_Score'],\n    ['Health_Score', 'dow'],\n    ['Age', 'Exercise_Frequency', 'Location'],\n    ['Health_Score', 'Smoking_Status', 'month'],\n    ['Health_Score', 'Location', 'Policy_Type'],\n    ['Health_Score', 'Insurance_Duration'],\n    ['Health_Score', 'Number_of_Dependents'],\n    ['Customer_Feedback', 'Exercise_Frequency', 'Previous_Claims', 'Property_Type', 'dow'],\n    ['Customer_Feedback', 'Health_Score'],\n    ['Health_Score', 'Property_Type'],\n    ['Health_Score', 'day', 'seconds'],\n    ['Health_Score', 'year'],\n    ['Age', 'Gender', 'Insurance_Duration', 'year']\n]\n\ndef target_encode(train, valid, test, col, target=\"target\", kfold=5, smooth=20, agg=\"mean\"):\n    \"\"\"\n    对指定列进行目标编码（Target Encoding），并使用 k 折交叉验证来避免过拟合。\n\n    参数：\n        train (pd.DataFrame): 训练集。\n        valid (pd.DataFrame): 验证集。\n        test (pd.DataFrame): 测试集。\n        col (list): 需要进行目标编码的列名列表。\n        target (str): 目标变量列名，默认为 \"target\"。\n        kfold (int): 交叉验证的折数，默认为 5。\n        smooth (int): 平滑参数，用于防止过拟合，默认为 20。\n        agg (str): 聚合方法，可选值为 \"mean\"、\"median\"、\"min\"、\"max\"、\"nunique\"，默认为 \"mean\"。\n\n    返回：\n        tuple: 包含目标编码列的 (train, valid, test)。\n    \"\"\"\n    # 初始化 k 折列和目标编码列名\n    train['kfold'] = train.index % kfold\n    col_name = '_'.join(col)\n    target_encoded_col = f'TE_{agg.upper()}_{col_name}'\n    train[target_encoded_col] = 0.0\n\n    # 计算全局聚合值\n    if agg == \"mean\":\n        global_agg = train[target].mean()\n    elif agg == \"median\":\n        global_agg = train[target].median()\n    elif agg == \"min\":\n        global_agg = train[target].min()\n    elif agg == \"max\":\n        global_agg = train[target].max()\n    elif agg == \"nunique\":\n        global_agg = 0\n    else:\n        raise ValueError(\"无效的聚合方法。请选择 'mean'、'median'、'min'、'max' 或 'nunique'。\")\n\n    # 对训练集进行 k 折目标编码\n    for fold in range(kfold):\n        # 分割数据为训练折和验证折\n        df_train = train[train['kfold'] != fold]\n        df_valid = train[train['kfold'] == fold]\n\n        # 计算每个组合的聚合值\n    \n        agg_values = df_train.groupby(col)[target].agg([agg, 'count']).reset_index()\n\n        agg_values.columns = col + [agg, 'count']\n\n        if agg == \"nunique\":\n            # 如果是 nunique，计算唯一值的比例\n            agg_values['TE_tmp'] = agg_values[agg] / agg_values['count']\n        else:\n            # 使用平滑公式计算目标编码值\n            agg_values['TE_tmp'] = ((agg_values[agg] * agg_values['count']) + (global_agg * smooth)) / (agg_values['count'] + smooth)\n\n        # 将目标编码值合并回验证折\n        df_valid = df_valid.merge(agg_values[col + ['TE_tmp']], on=col, how='left')\n        df_valid[target_encoded_col] = df_valid['TE_tmp'].fillna(global_agg)\n\n        # 更新训练集的目标编码列\n        train.loc[train['kfold'] == fold, target_encoded_col] = df_valid[target_encoded_col].values\n\n    # 删除 k 折列\n    train = train.drop('kfold', axis=1)\n    train[target_encoded_col] = train[target_encoded_col].astype(\"float32\")\n\n    # 计算全局目标编码值，用于验证集和测试集\n    agg_values = train.groupby(col)[target].agg([agg, 'count']).reset_index()\n    agg_values.columns = col + [agg, 'count']\n\n    if agg == \"nunique\":\n        agg_values['TE_tmp'] = agg_values[agg] / agg_values['count']\n    else:\n        agg_values['TE_tmp'] = ((agg_values[agg] * agg_values['count']) + (global_agg * smooth)) / (agg_values['count'] + smooth)\n\n    # 将目标编码值合并到验证集和测试集，未删TE_tmp列\n    valid = valid.merge(agg_values[col + ['TE_tmp']], on=col, how='left',suffixes=['','_'])\n    valid[target_encoded_col] = valid['TE_tmp'].fillna(global_agg).astype(\"float32\")\n\n    test = test.merge(agg_values[col + ['TE_tmp']], on=col, how='left',suffixes=['','_'])\n    test[target_encoded_col] = test['TE_tmp'].fillna(global_agg).astype(\"float32\")\n\n    return train, valid, test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T05:21:56.571514Z","iopub.execute_input":"2025-04-16T05:21:56.571808Z","iopub.status.idle":"2025-04-16T05:22:14.893039Z","shell.execute_reply.started":"2025-04-16T05:21:56.571788Z","shell.execute_reply":"2025-04-16T05:22:14.892025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass config:\n    def __init__(self):\n        self.usegpu = True\n        self.gpuid = 1\n        self.seed = 2025\n        self.model = 'nn'\n        self.loader_workers = 0\n        self.bs = 1000 #batch_size\n        self.lr = 1e-3            ###1e-3 --->2e-4\n        self.weight_decay = 5e-4  ##5e-4 --->3e-4\n        self.dropouts = [0.1, 0.1,0.1]\n        self.n_hidden = [512,256,256] #hidden_dims\n        self.max_epochs = 200\n        self.N_fold = 5\n        self.patience =  10\n        self.accelerator = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nmy_args = config()\n\nclass NNDataset(Dataset):\n    def __init__(self, df, accelerator,feature_names,label):\n        self.features = torch.FloatTensor(df[feature_names].values).to(accelerator)\n        self.labels = torch.FloatTensor(df[label].values).to(accelerator)\n    \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, idx):\n        x = self.features[idx]\n        y = self.labels[idx]\n        return x, y\n\n\nclass DataModule(LightningDataModule):\n    def __init__(self, train_df, batch_size, valid_df=None, accelerator='cuda' if torch.cuda.is_available() else 'cpu'):\n        super().__init__()\n        self.df = train_df\n        self.batch_size = batch_size\n        self.accelerator = accelerator\n        self.train_dataset = None\n        self.valid_df = None\n        if valid_df is not None:\n            self.valid_df = valid_df\n        self.val_dataset = None\n\n    #下面是overwrite LightningDataModule的\n    def setup(self):\n        self.train_dataset = NNDataset(self.df, self.accelerator,feature_names=self.df.drop(\"target\",axis=1).columns,label='target')\n        if self.valid_df is not None:\n            self.val_dataset = NNDataset(self.valid_df, self.accelerator,feature_names=self.df.drop(\"target\",axis=1).columns,label='target')\n\n    def train_dataloader(self, n_workers=0):\n        return DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=False, num_workers=n_workers)\n\n    def val_dataloader(self, n_workers=0):\n        return DataLoader(self.val_dataset, batch_size=self.batch_size, shuffle=False, num_workers=n_workers)\n\nclass NN(LightningModule):\n    def __init__(self, input_dim, hidden_dims, dropouts, lr, weight_decay):\n        super().__init__()\n        self.save_hyperparameters()\n        layers = []\n        in_dim = input_dim\n        for i, hidden_dim in enumerate(hidden_dims):\n            layers.append(nn.BatchNorm1d(in_dim))\n            if i > 0:\n                layers.append(nn.SiLU())\n            if i < len(dropouts):\n                layers.append(nn.Dropout(dropouts[i]))\n            layers.append(nn.Linear(in_dim, hidden_dim))\n            in_dim = hidden_dim\n        layers.append(nn.Linear(in_dim, 1))\n        # layers.append(nn.Tanh())\n        self.model = nn.Sequential(*layers)\n        self.lr = lr\n        self.weight_decay = weight_decay\n        self.validation_step_outputs = []\n\n    def forward(self, x):\n        return  self.model(x).squeeze(-1)\n    \n    def root_mean_squared_error(self, y_true, y_pred):\n        return np.sqrt(np.mean((y_true - y_pred) ** 2))\n\n    #下面是overwrite LightningModule的\n    def training_step(self, batch):\n        x, y = batch\n        y_hat = self(x)\n        # print(y_hat)\n        loss = F.mse_loss(y_hat, y, reduction='none')\n        loss = loss.mean()\n        self.log('train_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        return loss\n\n    def validation_step(self, batch):\n        x, y = batch\n        y_hat = self(x)\n        loss = F.mse_loss(y_hat, y, reduction='none')\n        loss = loss.mean()\n        self.log('val_loss', loss, on_step=False, on_epoch=True, batch_size=x.size(0))\n        self.validation_step_outputs.append((y_hat, y,))\n        return loss\n\n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=5,\n                                                               verbose=True)\n        return {\n            'optimizer': optimizer,\n            'lr_scheduler': {\n                'scheduler': scheduler,\n                'monitor': 'val_loss',\n            }\n        }\n    \n    #下面的在LightningDataModule中没找到，怎么打印？\n    def on_validation_epoch_end(self):\n        \"\"\"Calculate validation WRMSE at the end of the epoch.\"\"\"\n        y = torch.cat([x[1] for x in self.validation_step_outputs]).cpu().numpy()\n        if self.trainer.sanity_checking:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n        else:\n            prob = torch.cat([x[0] for x in self.validation_step_outputs]).cpu().numpy()\n            # print('validation_step_outputs',self.validation_step_outputs)\n            # print('y',y)\n            # print('prob',prob)\n            val_score = self.root_mean_squared_error(y, prob)\n            self.log(\"val_score\", val_score, prog_bar=True, on_step=False, on_epoch=True)\n        self.validation_step_outputs.clear()\n\n    def on_train_epoch_end(self):\n        if self.trainer.sanity_checking:\n            return\n        epoch = self.trainer.current_epoch\n        metrics = {k: v.item() if isinstance(v, torch.Tensor) else v for k, v in self.trainer.logged_metrics.items()}\n        formatted_metrics = {k: f\"{v:.5f}\" for k, v in metrics.items()}\n        print(f\"Epoch {epoch}: {formatted_metrics}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-16T05:22:24.983015Z","iopub.execute_input":"2025-04-16T05:22:24.983393Z","iopub.status.idle":"2025-04-16T05:22:25.008216Z","shell.execute_reply.started":"2025-04-16T05:22:24.983367Z","shell.execute_reply":"2025-04-16T05:22:25.007292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"FOLDS = 5\nfrom sklearn.model_selection import KFold\nkf = KFold(n_splits=FOLDS, shuffle=True, random_state=42)\n\noof = np.zeros(len(train))\npred = np.zeros(len(test))\n\nfor i, (train_index, test_index) in enumerate(kf.split(train)):\n\n    print(\"#\"*25)\n    print(f\"### Fold {i+1}\")\n    print(\"#\"*25)\n    \n    x_train = train.loc[train_index,features+[\"target\"] ].copy()\n    y_train = train.loc[train_index,\"target\"]\n    x_valid = train.loc[test_index,features+[\"target\"]].copy()\n    y_valid = train.loc[test_index,\"target\"]\n    x_test = test[features].copy()\n\n    start = time.time()\n    print(f\"FEATURE ENGINEER {len(features)} COLUMNS and {len(encoding_features)} GROUPS: \",end=\"\")\n    for j,f in enumerate(features+encoding_features):\n\n        if j<len(features): c = [f]\n        else: c = f \n        print(f\"({j+1}){c}\",\", \",end=\"\")\n\n        # LOW CARDINALITY features - TARGET ENCODE MEAN AND MEDIAN\n        x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=20, agg=\"mean\")\n        x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"median\")\n\n        # HIGH CARDINALITY features - TE MIN, MAX, NUNIQUE and CE\n        if (j>=len(features)) | (c[0] in high_cardinality_features):\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"min\")\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"max\")\n            x_train, x_valid, x_test = target_encode(x_train, x_valid, x_test, c, smooth=0, agg=\"nunique\")\n    \n            # COUNT ENCODING (USING COMBINED TRAIN TEST)\n            tmp = combined.groupby(c)['target'].count()\n            nm = f\"CE_{'_'.join(c)}\"; tmp.name = nm\n            x_train = x_train.merge(tmp, on=c, how=\"left\")\n            x_valid = x_valid.merge(tmp, on=c, how=\"left\")\n            x_test = x_test.merge(tmp, on=c, how=\"left\")\n\n            x_train[nm] = x_train[nm].fillna(x_train[nm].mean())\n            x_valid[nm] = x_valid[nm].fillna(x_valid[nm].mean())\n            x_test[nm] = x_test[nm].fillna(x_test[nm].mean())\n\n            x_train[nm] = x_train[nm].astype(\"int32\").fillna(0)\n            x_valid[nm] = x_valid[nm].astype(\"int32\").fillna(0)\n            x_test[nm] = x_test[nm].astype(\"int32\").fillna(0)\n\n    FEATURES = x_train.columns.tolist()\n\n    x_train[FEATURES] =  x_train[FEATURES].fillna(0).astype('float32')\n    x_valid[FEATURES] =  x_valid[FEATURES].fillna(0).astype('float32')\n    x_test =  x_test.fillna(0).astype('float32')\n\n    data_module = DataModule(train_df = x_train, valid_df=x_valid,batch_size=my_args.bs, accelerator=my_args.accelerator)\n    data_module.setup()\n    input_dim = data_module.train_dataset.features.shape[1]\n    model = NN(\n        input_dim=input_dim,\n        hidden_dims=my_args.n_hidden,\n        dropouts=my_args.dropouts,\n        lr=my_args.lr,\n        weight_decay=my_args.weight_decay\n    )\n    early_stopping = EarlyStopping('val_loss', patience=my_args.patience, mode='min', verbose=False)\n    checkpoint_callback = ModelCheckpoint(monitor='val_loss', mode='min', save_top_k=1, verbose=False, filename=f\"./models/nn.model\") \n    timer = Timer()\n\n    trainer = Trainer(\n        max_epochs=my_args.max_epochs,\n        accelerator=my_args.accelerator,\n        devices= my_args.gpuid if my_args.usegpu else None,\n        logger=None,\n        callbacks=[early_stopping, checkpoint_callback, timer],\n        enable_progress_bar=True\n    )\n    trainer.fit(model, data_module.train_dataloader(my_args.loader_workers), data_module.val_dataloader(my_args.loader_workers))\n    print(f' Training completed in {timer.time_elapsed(\"train\"):.2f}s')\n    end = time.time()\n    elapsed = end-start\n    print(f\"Feature engineering took {elapsed:.1f} seconds\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def root_mean_squared_error( y_true, y_pred):\n    return np.sqrt(np.mean((y_true - y_pred)**2))\n\npreds = np.zeros((len(test),))\nfor i in range(0,5):\n   model_path = f\"lightning_logs/version_{i}/checkpoints/models/nn.model.ckpt\"\n   model = NN.load_from_checkpoint(model_path)\n   model.eval()\n   model = model.to(my_args.accelerator)\n   test_input = torch.FloatTensor(x_test[x_train.drop('target',axis=1).columns.tolist()].fillna(0).to_numpy()).to(my_args.accelerator)\n   with torch.no_grad():\n    nn_preds = model(test_input)\n    preds += nn_preds.cpu().numpy()\npreds /= 5\n\nsubmission=pd.read_csv(r\"/kaggle/input/playground-series-s4e12/sample_submission.csv\")\nsubmission['Premium Amount']=np.exp(preds)-1\nsubmission.to_csv(r\"submission.csv\",index=False)\nsubmission.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}