{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":84896,"databundleVersionId":10305135,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q autogluon.tabular ray==2.10.0 scikit-learn==1.5.2\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:44:41.463768Z","iopub.execute_input":"2024-12-31T14:44:41.464082Z","iopub.status.idle":"2024-12-31T14:45:00.765821Z","shell.execute_reply.started":"2024-12-31T14:44:41.464057Z","shell.execute_reply":"2024-12-31T14:45:00.764836Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import root_mean_squared_error\nfrom sklearn.model_selection import KFold\nfrom autogluon.tabular import TabularPredictor\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport pickle\nimport shutil\nimport os\n\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:02.140162Z","iopub.execute_input":"2024-12-31T14:45:02.14057Z","iopub.status.idle":"2024-12-31T14:45:02.144764Z","shell.execute_reply.started":"2024-12-31T14:45:02.140545Z","shell.execute_reply":"2024-12-31T14:45:02.143993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = '/kaggle/input/playground-series-s4e12/train.csv'\n    test_path = '/kaggle/input/playground-series-s4e12/test.csv'\n    sample_sub_path = '/kaggle/input/playground-series-s4e12/sample_submission.csv'\n    \n    target = 'Premium Amount'\n    n_folds = 5\n    seed = 42\n    time_limit = 3600 * 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:02.145904Z","iopub.execute_input":"2024-12-31T14:45:02.14615Z","iopub.status.idle":"2024-12-31T14:45:02.174967Z","shell.execute_reply.started":"2024-12-31T14:45:02.146129Z","shell.execute_reply":"2024-12-31T14:45:02.174198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(CFG.train_path, index_col='id')\ntest = pd.read_csv(CFG.test_path, index_col='id')\n\ntrain[CFG.target] = np.log1p(train[CFG.target])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:02.175877Z","iopub.execute_input":"2024-12-31T14:45:02.176206Z","iopub.status.idle":"2024-12-31T14:45:10.810438Z","shell.execute_reply.started":"2024-12-31T14:45:02.176175Z","shell.execute_reply":"2024-12-31T14:45:10.809756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:10.829135Z","iopub.execute_input":"2024-12-31T14:45:10.829358Z","iopub.status.idle":"2024-12-31T14:45:10.843555Z","shell.execute_reply.started":"2024-12-31T14:45:10.829337Z","shell.execute_reply":"2024-12-31T14:45:10.842743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kf = KFold(n_splits=CFG.n_folds, random_state=CFG.seed, shuffle=True)\nsplit = kf.split(train, train[CFG.target])\nfor i, (_, val_index) in enumerate(split):\n    train.loc[val_index, 'fold'] = i","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:33.703967Z","iopub.execute_input":"2024-12-31T14:45:33.704302Z","iopub.status.idle":"2024-12-31T14:45:33.959089Z","shell.execute_reply.started":"2024-12-31T14:45:33.704276Z","shell.execute_reply":"2024-12-31T14:45:33.958144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor = TabularPredictor(\n    problem_type='regression',\n    eval_metric='rmse',\n    label=CFG.target,\n    groups='fold',\n    verbosity=2\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:44.452561Z","iopub.execute_input":"2024-12-31T14:45:44.452932Z","iopub.status.idle":"2024-12-31T14:45:44.45885Z","shell.execute_reply.started":"2024-12-31T14:45:44.45287Z","shell.execute_reply":"2024-12-31T14:45:44.458202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ag_args_fit={'num_gpus': 0, 'num_cpus': 4}\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:46.826067Z","iopub.execute_input":"2024-12-31T14:45:46.826354Z","iopub.status.idle":"2024-12-31T14:45:46.830166Z","shell.execute_reply.started":"2024-12-31T14:45:46.826332Z","shell.execute_reply":"2024-12-31T14:45:46.829416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor = TabularPredictor(\n    problem_type='regression',\n    eval_metric='rmse',\n    label=CFG.target,\n    groups='fold',\n    verbosity=2\n)\n\npredictor.fit(\n    train_data=train,\n    time_limit=CFG.time_limit,\n    presets='good_quality',  # Use a lighter preset\n    excluded_model_types=['KNN'],  # Keep as is\n    ag_args_fit={'num_gpus': 0, 'num_cpus': 4},  # Force CPU-only\n    dynamic_stacking=False,  # Turn off dynamic stacking\n    save_bag_folds=True  # Ensure bagged folds are saved\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T14:45:50.222228Z","iopub.execute_input":"2024-12-31T14:45:50.222528Z","iopub.status.idle":"2024-12-31T21:05:20.82773Z","shell.execute_reply.started":"2024-12-31T14:45:50.222507Z","shell.execute_reply":"2024-12-31T21:05:20.826745Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictor.leaderboard(silent=True).style.background_gradient(subset=['score_val'], cmap='RdYlGn')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:05:20.829104Z","iopub.execute_input":"2024-12-31T21:05:20.829422Z","iopub.status.idle":"2024-12-31T21:05:20.886883Z","shell.execute_reply.started":"2024-12-31T21:05:20.829387Z","shell.execute_reply":"2024-12-31T21:05:20.885738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_ensemble_weights(predictor):\n    info = predictor.info()\n    ensemble_weights = {}\n    for model_name, values in info[\"model_info\"].items():\n        if \"Ensemble\" in model_name:\n            children_info = values[\"children_info\"]\n            ensemble_weights[model_name] = values[\"children_info\"][list(children_info.keys())[0]][\"model_weights\"]\n    return ensemble_weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:05:20.888788Z","iopub.execute_input":"2024-12-31T21:05:20.889047Z","iopub.status.idle":"2024-12-31T21:05:20.893235Z","shell.execute_reply.started":"2024-12-31T21:05:20.889024Z","shell.execute_reply":"2024-12-31T21:05:20.892344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ensemble_weights = get_ensemble_weights(predictor)\n\nfor key, value in ensemble_weights.items():\n    plt.figure(figsize=(6, 6))\n    plt.pie(value.values(), labels=value.keys(), autopct='%1.1f%%', colors=sns.color_palette('Set2', len(value)))\n    plt.title(key)\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:05:20.894698Z","iopub.execute_input":"2024-12-31T21:05:20.894962Z","iopub.status.idle":"2024-12-31T21:06:01.398867Z","shell.execute_reply.started":"2024-12-31T21:05:20.894935Z","shell.execute_reply":"2024-12-31T21:06:01.397653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def save_preds(preds, cv_score, name, type, is_ensemble):\n    base_path = 'oof_preds' if type == 'oof' else 'test_preds'\n    base_path = '.' if is_ensemble else base_path\n    with open(f'{base_path}/{name}_{type}_preds_{cv_score:.6f}.pkl', 'wb') as f:\n        pickle.dump(np.expm1(preds), f)\n\ndef save_submission(test_preds, score):\n    output_dir = '/kaggle/working'  # Specify the base directory\n    sub = pd.read_csv(CFG.sample_sub_path)\n    sub[CFG.target] = np.expm1(test_preds)\n    # Save the submission file in the specified directory\n    sub.to_csv(os.path.join(output_dir, f'sub_autogluon_{score:.6f}.csv'), index=False)\n\n# Create directories for oof_preds and test_preds inside the working directory\noof_dir = '/kaggle/working/oof_preds'\ntest_dir = '/kaggle/working/test_preds'\n\nos.makedirs(oof_dir, exist_ok=True)\nos.makedirs(test_dir, exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:06:01.399785Z","iopub.execute_input":"2024-12-31T21:06:01.400186Z","iopub.status.idle":"2024-12-31T21:06:01.410201Z","shell.execute_reply.started":"2024-12-31T21:06:01.400148Z","shell.execute_reply":"2024-12-31T21:06:01.40916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oof_preds = {}\ntest_preds = {}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:06:01.410986Z","iopub.execute_input":"2024-12-31T21:06:01.411329Z","iopub.status.idle":"2024-12-31T21:06:01.437385Z","shell.execute_reply.started":"2024-12-31T21:06:01.411295Z","shell.execute_reply":"2024-12-31T21:06:01.436181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"best_model = predictor.model_best\n_test_preds = predictor.predict_multi(test)\n\n# Initialize dictionaries\noof_preds = {}\ntest_preds = {}\n\nfor model in predictor.model_names():\n    print(f\"Processing model: {model}\")\n    \n    try:\n        # Check if test predictions exist for this model\n        if model in _test_preds:\n            model_test_preds = _test_preds[model].values\n        else:\n            print(f\"Warning: No test predictions for {model}. Skipping...\")\n            continue\n        \n        # Get out-of-fold predictions\n        model_oof_preds = predictor.predict_oof(model).values\n        \n        # Calculate CV score\n        cv_score = root_mean_squared_error(train[CFG.target], np.maximum(model_oof_preds, 0))\n        \n        # Save predictions\n        if model != best_model:\n            save_preds(model_oof_preds, cv_score, model, 'oof', False)\n            save_preds(model_test_preds, cv_score, model, 'test', False)\n        else:\n            save_preds(model_oof_preds, cv_score, model, 'oof', True)\n            save_preds(model_test_preds, cv_score, model, 'test', True)\n            save_submission(model_test_preds, cv_score)\n        \n        # Store predictions\n        oof_preds[model] = model_oof_preds\n        test_preds[model] = model_test_preds\n        \n    except Exception as e:\n        print(f\"Error processing model {model}: {e}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:06:01.438611Z","iopub.execute_input":"2024-12-31T21:06:01.439264Z","iopub.status.idle":"2024-12-31T21:15:10.730149Z","shell.execute_reply.started":"2024-12-31T21:06:01.439223Z","shell.execute_reply":"2024-12-31T21:15:10.729434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = {}\nsplit = KFold(n_splits=CFG.n_folds, shuffle=False).split(train, train[CFG.target])\n\nfor fold_idx, (train_index, val_index) in enumerate(split):\n    for model in predictor.model_names():\n        if model not in oof_preds:  # Check if OOF predictions exist\n            print(f\"Warning: No OOF predictions for {model}. Skipping...\")\n            continue\n        \n        # Calculate RMSE for the fold\n        fold_score = root_mean_squared_error(\n            train.loc[val_index, CFG.target],\n            np.maximum(oof_preds[model][val_index], 0)\n        )\n        \n        # Store scores\n        if model not in scores:\n            scores[model] = []\n        scores[model].append(fold_score)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:15:10.731684Z","iopub.execute_input":"2024-12-31T21:15:10.732003Z","iopub.status.idle":"2024-12-31T21:15:12.316496Z","shell.execute_reply.started":"2024-12-31T21:15:10.731978Z","shell.execute_reply":"2024-12-31T21:15:12.315593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scores = pd.DataFrame(scores)\nmean_scores = scores.mean().sort_values(ascending=True)\norder = scores.mean().sort_values(ascending=True).index.tolist()\n\nmin_score = mean_scores.min()\nmax_score = mean_scores.max()\npadding = (max_score - min_score) * 0.5\nlower_limit = min_score - padding\nupper_limit = max_score + padding\n\nfig, axs = plt.subplots(1, 2, figsize=(15, scores.shape[1] * 0.3))\n\nsns.boxplot(data=scores, order=order, ax=axs[0], orient='h', palette='RdYlGn_r')\naxs[0].set_title('Fold RMSLE')\naxs[0].set_xlabel('')\naxs[0].set_ylabel('')\n\nbarplot = sns.barplot(x=mean_scores.values, y=mean_scores.index, ax=axs[1], palette='RdYlGn_r')\naxs[1].set_title('Average RMSLE')\naxs[1].set_xlabel('')\naxs[1].set_xlim(left=lower_limit, right=upper_limit)\naxs[1].set_ylabel('')\n\nfor i, score in enumerate(mean_scores.values):\n    barplot.text(score, i, round(score, 6), va='center')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-31T21:15:12.31767Z","iopub.execute_input":"2024-12-31T21:15:12.318025Z","iopub.status.idle":"2024-12-31T21:15:13.551231Z","shell.execute_reply.started":"2024-12-31T21:15:12.317991Z","shell.execute_reply":"2024-12-31T21:15:13.550225Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}