{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":119082,"databundleVersionId":14993753,"sourceType":"competition"},{"sourceId":13904981,"sourceType":"datasetVersion","datasetId":8762382}],"dockerImageVersionId":31260,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn\n!pip install category_encoders\n!pip install catboost","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Imports & Configuration","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\nimport warnings\nimport os\nimport gc\nimport torch\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.cluster import KMeans\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import root_mean_squared_error \n\nfrom sklearn.preprocessing import StandardScaler, OrdinalEncoder\nfrom sklearn.preprocessing import TargetEncoder\n\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.linear_model import RidgeCV\nfrom sklearn.impute import SimpleImputer\n\n# from category_encoders import TargetEncoder\n\nfrom joblib import Parallel, delayed\n\nimport xgboost as xgb\nimport lightgbm as lgb\nimport catboost as cb\n\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', None)\nRANDOM_STATE = 42\nSEEDS = [42, 2024, 123]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Data Loading & Integration","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/playground-series-s6e1/train.csv')\ntest_df = pd.read_csv('/kaggle/input/playground-series-s6e1/test.csv')\noriginal_df = pd.read_csv('/kaggle/input/exam-score-prediction-dataset/Exam_Score_Prediction.csv')\n\nif 'id' in original_df.columns:\n    original_df = original_df.drop(columns=['id'])\n\nTARGET = 'exam_score'\nCATS = train_df.select_dtypes('object').columns.to_list() ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Feature Engineering","metadata":{}},{"cell_type":"code","source":"def preprocess(df):\n    df = df.copy()\n    eps = 1e-5\n    \n    # 1. Base Numeric Transforms\n    df['study_hours_sq'] = df['study_hours'] ** 2\n    df['attendance_sq'] = df['class_attendance'] ** 2\n    df['sleep_hours_sq'] = df['sleep_hours'] ** 2\n    df['age_sq'] = df['age'] ** 2\n    \n    df['log_study'] = np.log1p(df['study_hours'])\n    df['log_attendance'] = np.log1p(df['class_attendance'])\n    df['log_sleep'] = np.log1p(df['sleep_hours'])\n    \n    df['sqrt_study'] = np.sqrt(df['study_hours'])\n    df['sqrt_attendance'] = np.sqrt(df['class_attendance'])\n    \n    # 2. Numeric Interactions\n    df['study_x_attend'] = df['study_hours'] * df['class_attendance']\n    df['study_x_sleep'] = df['study_hours'] * df['sleep_hours']\n    df['attend_x_sleep'] = df['class_attendance'] * df['sleep_hours']\n    df['age_x_study'] = df['age'] * df['study_hours']\n    \n    # 3. Ratios\n    df['study_over_sleep'] = df['study_hours'] / (df['sleep_hours'] + eps)\n    df['attend_over_sleep'] = df['class_attendance'] / (df['sleep_hours'] + eps)\n    df['attend_over_study'] = df['class_attendance'] / (df['study_hours'] + eps)\n    df['efficiency'] = (df['study_hours'] * df['class_attendance']) / (df['sleep_hours'] + 1)\n    \n    # 4. Ordinal Mappings\n    sleep_map = {'Poor': 0, 'Average': 1, 'Good': 2, 'poor': 0, 'average': 1, 'good': 2}\n    facility_map = {'Low': 0, 'Moderate': 1, 'High': 2, 'low': 0, 'moderate': 1, 'high': 2}\n    difficulty_map = {'Easy': 0, 'Moderate': 1, 'Hard': 2, 'easy': 0, 'moderate': 1, 'hard': 2}\n    \n    df['sleep_qual_num'] = df['sleep_quality'].map(sleep_map).fillna(1)\n    df['facility_num'] = df['facility_rating'].map(facility_map).fillna(1)\n    df['difficulty_num'] = df['exam_difficulty'].map(difficulty_map).fillna(1)\n    \n    # 5. Ordinal Interactions\n    df['study_x_sleep_qual'] = df['study_hours'] * df['sleep_qual_num']\n    df['attend_x_facility'] = df['class_attendance'] * df['facility_num']\n    df['sleep_x_difficulty'] = df['sleep_hours'] * df['difficulty_num']\n    df['facility_x_sleep_qual'] = df['facility_num'] * df['sleep_qual_num']\n    df['difficulty_x_facility'] = df['difficulty_num'] * df['facility_num']\n    \n    # 6. Bins\n    df[\"study_bin\"] = pd.cut(df[\"study_hours\"], bins=[-1, 2, 4, 6, 8, 100], labels=[0, 1, 2, 3, 4]).astype(float)\n    df[\"attendance_bin\"] = pd.cut(df[\"class_attendance\"], bins=[-1, 60, 75, 85, 95, 101], labels=[0, 1, 2, 3, 4]).astype(float)\n    df[\"sleep_bin\"] = pd.cut(df[\"sleep_hours\"], bins=[-1, 5, 6, 7, 8, 100], labels=[0, 1, 2, 3, 4]).astype(float)\n    df[\"age_bin\"] = pd.cut(df[\"age\"], bins=[0, 17, 19, 21, 23, 100], labels=[0, 1, 2, 3, 4]).astype(float)\n    \n    # 7. Flags & Gaps\n    df[\"high_att_high_study\"] = ((df[\"class_attendance\"] >= 90) & (df[\"study_hours\"] >= 6)).astype(int)\n    df[\"ideal_sleep\"] = ((df[\"sleep_hours\"] >= 7) & (df[\"sleep_hours\"] <= 9)).astype(int)\n    df[\"high_study_flag\"] = (df[\"study_hours\"] >= 7).astype(int)\n    \n    df['sleep_gap_8'] = (df['sleep_hours'] - 8.0).abs()\n    df['attendance_gap_100'] = (df['class_attendance'] - 100.0).abs()\n\n    return df\n\nX_train = preprocess(train_df.drop(columns=[TARGET, 'id'], errors='ignore'))\ny_train = train_df[TARGET]\nX_test = preprocess(test_df.drop(columns=['id'], errors='ignore'))\nX_orig = preprocess(original_df.drop(columns=[TARGET, 'id'], errors='ignore'))\ny_orig = original_df[TARGET]\n\nprint(f\"Feature Set Created. Train Shape: {X_train.shape}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Target Encoding & Ridge Stacking Layer","metadata":{}},{"cell_type":"code","source":"print(\"Starting Stacking Layer\")\n\ncommon_cols = X_train.columns.intersection(X_orig.columns).intersection(X_test.columns)\nX_train_aligned = X_train[common_cols].copy()\nX_orig_aligned = X_orig[common_cols].copy()\nX_test_aligned = X_test[common_cols].copy()\n\n# We select columns that are object or category type\nCATS_ALIGNED = X_train_aligned.select_dtypes(include=['object', 'category']).columns.tolist()\nprint(f\"Encoding {len(CATS_ALIGNED)} categorical features: {CATS_ALIGNED}\")\n\noof_preds_lr = np.zeros(len(X_train))\ntest_preds_lr = np.zeros((len(X_test), 10)) \norig_preds_lr = np.zeros(len(X_orig)) \n\nkf = KFold(n_splits=10, shuffle=True, random_state=42)\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train_aligned, y_train)):\n    X_tr_fold, y_tr_fold = X_train_aligned.iloc[train_idx], y_train.iloc[train_idx]\n    X_val_fold, y_val_fold = X_train_aligned.iloc[val_idx], y_train.iloc[val_idx]\n    \n    X_tr_aug = pd.concat([X_tr_fold, X_orig_aligned], axis=0)\n    y_tr_aug = pd.concat([y_tr_fold, y_orig], axis=0)\n    \n    # 3. ENCODING \n    enc = TargetEncoder(smooth='auto', target_type='continuous', random_state=42)\n    \n    # Fit on Augmented Train\n    X_tr_cat_encoded = enc.fit_transform(X_tr_aug[CATS_ALIGNED], y_tr_aug)\n    X_val_cat_encoded = enc.transform(X_val_fold[CATS_ALIGNED])\n    X_test_cat_encoded = enc.transform(X_test_aligned[CATS_ALIGNED])\n    X_orig_cat_encoded = enc.transform(X_orig_aligned[CATS_ALIGNED])\n    \n    # Replace columns in the dataset\n    X_tr_aug_final = X_tr_aug.copy()\n    X_tr_aug_final[CATS_ALIGNED] = X_tr_cat_encoded\n    \n    X_val_final = X_val_fold.copy()\n    X_val_final[CATS_ALIGNED] = X_val_cat_encoded\n    \n    X_test_final = X_test_aligned.copy()\n    X_test_final[CATS_ALIGNED] = X_test_cat_encoded\n    \n    X_orig_final = X_orig_aligned.copy()\n    X_orig_final[CATS_ALIGNED] = X_orig_cat_encoded\n\n    # 4. Impute & Clean\n    X_tr_aug_final = X_tr_aug_final.astype(float)\n    X_val_final = X_val_final.astype(float)\n    X_test_final = X_test_final.astype(float)\n    X_orig_final = X_orig_final.astype(float)\n    \n    imputer = SimpleImputer(strategy='mean')\n    X_tr_aug_final = imputer.fit_transform(X_tr_aug_final)\n    X_val_final = imputer.transform(X_val_final)\n    X_test_final = imputer.transform(X_test_final)\n    X_orig_final = imputer.transform(X_orig_final)\n    \n    # 5. Train Ridge\n    alphas = np.logspace(-3, 3, 20)\n    model = RidgeCV(alphas=alphas, cv=3) \n    model.fit(X_tr_aug_final, y_tr_aug)\n    \n    oof_preds_lr[val_idx] = np.clip(model.predict(X_val_final), 0, 100)\n    test_preds_lr[:, fold] = np.clip(model.predict(X_test_final), 0, 100)\n    orig_preds_lr += np.clip(model.predict(X_orig_final), 0, 100) / 10\n    \n    score = root_mean_squared_error(y_val_fold, oof_preds_lr[val_idx])\n    print(f\"Fold {fold+1} Ridge RMSE: {score:.4f}\")\n\nX_train['feature_lr_pred'] = oof_preds_lr\nX_test['feature_lr_pred'] = test_preds_lr.mean(axis=1)\nX_orig['feature_lr_pred'] = orig_preds_lr\n\nprint(\"Stacking Complete.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Training: XGBoost\nDefining the XGBoost training logic. \n* **Seed Averaging:** Inside every fold, we train multiple models with different random seeds to reduce variance.\n* **Hyperparameters:** Optimized for depth, learning rate, and regularization (Reg Alpha/Lambda).","metadata":{}},{"cell_type":"code","source":"print(\"Starting XGBoost Seed Averaging\")\n\ncommon_cols = X_train.columns.intersection(X_orig.columns).intersection(X_test.columns)\nprint(f\"Features: {len(common_cols)} columns\")\n\nX_train_final = X_train[common_cols].copy()\nX_orig_final = X_orig[common_cols].copy()\nX_test_final = X_test[common_cols].copy()\ny_train_final = y_train.copy()\ny_orig_final = y_orig.copy()\n\nCATS_FINAL = [c for c in CATS if c in common_cols]\nfor col in CATS_FINAL:\n    X_train_final[col] = X_train_final[col].astype('category')\n    X_orig_final[col] = X_orig_final[col].astype('category')\n    X_test_final[col] = X_test_final[col].astype('category')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_fold_final_seeded(fold, train_idx, val_idx):\n    device_id = f\"cuda:{fold % 2}\"\n    \n    X_tr_fold, y_tr_fold = X_train_final.iloc[train_idx], y_train_final.iloc[train_idx]\n    X_val_fold, y_val_fold = X_train_final.iloc[val_idx], y_train_final.iloc[val_idx]\n    \n    X_tr_aug = pd.concat([X_tr_fold, X_orig_final], axis=0)\n    y_tr_aug = pd.concat([y_tr_fold, y_orig_final], axis=0)\n    \n    val_preds_avg = np.zeros(len(X_val_fold))\n    test_preds_avg = np.zeros(len(X_test_final))\n    \n    for seed in SEEDS:\n        params = {\n            'n_estimators': 20000,\n            'learning_rate': 0.004,\n            'max_depth': 9,\n            'subsample': 0.78,\n            'colsample_bytree': 0.55,\n            'colsample_bynode': 0.65,\n            'reg_lambda': 6,\n            'reg_alpha': 0.15,\n            'min_child_weight': 6,\n            'tree_method': 'hist',\n            'enable_categorical': True,\n            'early_stopping_rounds': 100,\n            'device': device_id,\n            'random_state': seed,\n            'n_jobs': 4,\n            'verbosity': 0,\n            'eval_metric': 'rmse'\n        }\n        \n        model = xgb.XGBRegressor(**params)\n        model.fit(X_tr_aug, y_tr_aug, eval_set=[(X_val_fold, y_val_fold)], verbose=False)\n        \n        val_preds_avg += model.predict(X_val_fold) / len(SEEDS)\n        test_preds_avg += model.predict(X_test_final) / len(SEEDS)\n        \n        del model\n        gc.collect()\n    \n    rmse = root_mean_squared_error(y_val_fold, val_preds_avg)\n    print(f\"✅ Fold {fold+1} Finished (Avg 3 Seeds) | RMSE: {rmse:.5f}\")\n    \n    return val_idx, val_preds_avg, test_preds_avg","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EXECUTE\nresults = Parallel(n_jobs=2, backend=\"loky\")(\n    delayed(train_fold_final_seeded)(fold, train_idx, val_idx)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_train_final, y_train_final))\n)\n\n# AGGREGATE\noof_preds_final = np.zeros(len(X_train_final))\ntest_preds_final = np.zeros(len(X_test_final))\n\nfor val_idx, val_pred, test_pred in results:\n    oof_preds_final[val_idx] = val_pred\n    test_preds_final += test_pred / 10\n\nxgb_oof = oof_preds_final.copy()\nxgb_test = test_preds_final.copy()\n\nglobal_rmse = root_mean_squared_error(y_train_final, oof_preds_final)\n\n# Saving XGBoost Outputs\nxgb_oof = oof_preds_final.copy()\nxgb_test = test_preds_final.copy()\n\nprint(f\"\\n XGBoost Seed Averaging Complete.\")\nprint(f\"Global OOF RMSE: {global_rmse:.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Training: LightGBM ","metadata":{}},{"cell_type":"code","source":"print(\"Starting LightGBM Seed Averaging\")\n\nlgbm_oof = np.zeros(len(X_train_final))\nlgbm_test = np.zeros(len(X_test_final))\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(X_train_final, y_train_final)):\n    gc.collect()\n    print(f\"\\n Fold {fold+1} Starting...\", flush=True)\n    \n    X_tr, y_tr = X_train_final.iloc[train_idx], y_train_final.iloc[train_idx]\n    X_val, y_val = X_train_final.iloc[val_idx], y_train_final.iloc[val_idx]\n    \n    X_tr_aug = pd.concat([X_tr, X_orig_final], axis=0)\n    y_tr_aug = pd.concat([y_tr, y_orig_final], axis=0)\n    \n    # Temp arrays for averaging seeds within this fold\n    val_preds_fold_avg = np.zeros(len(X_val))\n    test_preds_fold_avg = np.zeros(len(X_test_final))\n    \n    for seed in SEEDS:\n        params = {\n            'objective': 'regression',\n            'metric': 'rmse',\n            'learning_rate': 0.008,\n            'n_estimators': 12000,\n            'num_leaves': 100,\n            'min_child_samples': 40,\n            'colsample_bytree': 0.6,\n            'subsample': 0.8,\n            'subsample_freq': 1,\n            'reg_alpha': 2,\n            'reg_lambda': 5,\n            'random_state': seed,\n            'verbosity': -1,\n            'n_jobs': 4,\n            'device': 'gpu',\n            'gpu_device_id': 0,\n            'gpu_platform_id': 0,\n            'force_col_wise': True\n        }\n        \n        model = lgb.LGBMRegressor(**params)\n        \n        model.fit(\n            X_tr_aug, y_tr_aug,\n            eval_set=[(X_val, y_val)],\n            callbacks=[lgb.early_stopping(100, verbose=False)]\n        )\n        \n        val_preds_fold_avg += model.predict(X_val) / len(SEEDS)\n        test_preds_fold_avg += model.predict(X_test_final) / len(SEEDS)\n        \n        del model\n    \n    lgbm_oof[val_idx] = val_preds_fold_avg\n    lgbm_test += test_preds_fold_avg / 10\n    \n    rmse = root_mean_squared_error(y_val, val_preds_fold_avg)\n    print(f\"✅ Fold {fold+1} Finished (Avg 3 Seeds) | RMSE: {rmse:.5f}\")\n    \n    del X_tr_aug, y_tr_aug, X_tr, X_val\n    gc.collect()\n\nprint(f\"\\n LightGBM Seed Averaging Complete.\")\nprint(f\"Global LightGBM RMSE: {root_mean_squared_error(y_train_final, lgbm_oof):.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Training: CatBoost","metadata":{}},{"cell_type":"code","source":"print(\"Starting CatBoost Seed Averaging\")\n\ndef train_fold_catboost(fold, train_idx, val_idx):\n    gpu_id = str(fold % 2)\n    \n    print(f\"🔄 Fold {fold+1} Starting on GPU {gpu_id}...\", flush=True)\n    \n    X_tr, y_tr = X_train_final.iloc[train_idx], y_train_final.iloc[train_idx]\n    X_val, y_val = X_train_final.iloc[val_idx], y_train_final.iloc[val_idx]\n    \n    X_tr_aug = pd.concat([X_tr, X_orig_final], axis=0)\n    y_tr_aug = pd.concat([y_tr, y_orig_final], axis=0)\n    \n    # Creating Pools ONCE per fold (Re-used for all seeds)\n    train_pool = cb.Pool(X_tr_aug, y_tr_aug, cat_features=CATS_FINAL)\n    val_pool = cb.Pool(X_val, y_val, cat_features=CATS_FINAL)\n    test_pool = cb.Pool(X_test_final, cat_features=CATS_FINAL)\n    \n    # Temp arrays for averaging seeds within this fold\n    val_preds_fold_avg = np.zeros(len(X_val))\n    test_preds_fold_avg = np.zeros(len(X_test_final))\n    \n    for seed in SEEDS:\n        params = {\n            'loss_function': 'RMSE',\n            'eval_metric': 'RMSE',\n            'learning_rate': 0.02,\n            'iterations': 8000,\n            'depth': 6,\n            'random_strength': 0.5,\n            'l2_leaf_reg': 5,\n            'task_type': 'GPU',\n            'devices': gpu_id,\n            'random_seed': seed,\n            'verbose': 0,\n            'early_stopping_rounds': 100\n        }\n        \n        model = cb.CatBoostRegressor(**params)\n        model.fit(train_pool, eval_set=val_pool, verbose=False)\n        \n        val_preds_fold_avg += model.predict(val_pool) / len(SEEDS)\n        test_preds_fold_avg += model.predict(test_pool) / len(SEEDS)\n        \n        del model\n        gc.collect()\n    \n    rmse = root_mean_squared_error(y_val, val_preds_fold_avg)\n    print(f\"✅ Fold {fold+1} Finished (Avg 3 Seeds) | RMSE: {rmse:.5f}\")\n    \n    # Cleanup\n    del train_pool, val_pool, test_pool, X_tr_aug, y_tr_aug\n    gc.collect()\n    \n    return val_idx, val_preds_fold_avg, test_preds_fold_avg","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# EXECUTE\nresults = Parallel(n_jobs=2, backend=\"loky\")(\n    delayed(train_fold_catboost)(fold, train_idx, val_idx)\n    for fold, (train_idx, val_idx) in enumerate(kf.split(X_train_final, y_train_final))\n)\n\n# AGGREGATE \ncat_oof = np.zeros(len(X_train_final))\ncat_test = np.zeros(len(X_test_final))\n\nfor val_idx, val_pred, test_pred in results:\n    cat_oof[val_idx] = val_pred\n    cat_test += test_pred / 10\n\nprint(f\"\\n CatBoost Seed Averaging Complete.\")\nprint(f\"Global CatBoost RMSE: {root_mean_squared_error(y_train_final, cat_oof):.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Automated Ensemble Weighting","metadata":{}},{"cell_type":"code","source":"X_blend = pd.DataFrame({\n    'XGB': xgb_oof,\n    'LGBM': lgbm_oof,\n    'CAT': cat_oof\n})\n\nblender = LinearRegression(positive=True, fit_intercept=False)\nblender.fit(X_blend, y_train_final)\n\nweights = blender.coef_\nxgb_weight = weights[0]\nlgbm_weight = weights[1]\ncat_weight = weights[2]\n\ntotal = xgb_weight + lgbm_weight + cat_weight\nxgb_weight /= total\nlgbm_weight /= total\ncat_weight /= total\n\nprint(f\"Optimal Weights Found:\")\nprint(f\"XGBoost Weight:  {xgb_weight:.4f}\")\nprint(f\"LightGBM Weight: {lgbm_weight:.4f}\")\nprint(f\"CatBoost Weight: {cat_weight:.4f}\")\n\nfinal_oof_blend = (xgb_weight * xgb_oof) + (lgbm_weight * lgbm_oof) + (cat_weight * cat_oof)\nblend_rmse = root_mean_squared_error(y_train_final, final_oof_blend)\nprint(f\"   Combined OOF RMSE: {blend_rmse:.5f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 9. Final Submission","metadata":{}},{"cell_type":"code","source":"final_test_blend = (xgb_weight * xgb_test) + (lgbm_weight * lgbm_test) + (cat_weight * cat_test)\n\nsubmission = pd.DataFrame({\n    'id': test_df['id'],\n    'exam_score': np.clip(final_test_blend, 0, 100)\n})\n\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission Saved: 'submission.csv'\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}