{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, gc\nimport numpy as np\nimport pandas as pd\nimport pickle\nimport sys\n\nimport lightgbm as lgb\nimport optuna\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction import DictVectorizer\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-01T03:40:04.188004Z","iopub.execute_input":"2022-12-01T03:40:04.18866Z","iopub.status.idle":"2022-12-01T03:40:06.56171Z","shell.execute_reply.started":"2022-12-01T03:40:04.188552Z","shell.execute_reply":"2022-12-01T03:40:06.560998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_tune_params = False\nCATEGORICAL_COL = [\"view\", \"implant\",\"machine_id\", \"laterality\"]\nNUMERICAL_COL = [\"age\"]\nTARGET_COLS = [\"cancer\"]\nRANDOM_SEED = 42\n\ndef set_seed(seed=2022):\n    np.random.seed(seed)\n    #tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    #os.environ['TF_DETERMINISTIC_OPS'] = '1'\n    \nset_seed(RANDOM_SEED)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:11.525521Z","iopub.execute_input":"2022-12-01T03:40:11.525879Z","iopub.status.idle":"2022-12-01T03:40:11.532335Z","shell.execute_reply.started":"2022-12-01T03:40:11.525849Z","shell.execute_reply":"2022-12-01T03:40:11.531316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/rsna-breast-cancer-detection/train.csv\")\ndf_test = pd.read_csv(\"../input/rsna-breast-cancer-detection/test.csv\").drop_duplicates(subset='prediction_id')\ndf_sub = pd.read_csv(\"../input/rsna-breast-cancer-detection/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:15.49398Z","iopub.execute_input":"2022-12-01T03:40:15.494399Z","iopub.status.idle":"2022-12-01T03:40:15.614441Z","shell.execute_reply.started":"2022-12-01T03:40:15.494367Z","shell.execute_reply":"2022-12-01T03:40:15.613526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FOLD = 5\nskf = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=RANDOM_SEED)\nfor n, (train_index, val_index) in enumerate(skf.split(df_train, df_train[TARGET_COLS])):\n    df_train.loc[val_index, 'fold'] = int(n)\ndf_train['fold'] = df_train['fold'].astype(int)\ndf_train[\"age\"] = df_train[\"age\"].fillna(df_train[\"age\"].median())","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:17.989201Z","iopub.execute_input":"2022-12-01T03:40:17.990429Z","iopub.status.idle":"2022-12-01T03:40:18.020648Z","shell.execute_reply.started":"2022-12-01T03:40:17.990385Z","shell.execute_reply":"2022-12-01T03:40:18.019596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:20.957927Z","iopub.execute_input":"2022-12-01T03:40:20.958286Z","iopub.status.idle":"2022-12-01T03:40:20.978855Z","shell.execute_reply.started":"2022-12-01T03:40:20.958258Z","shell.execute_reply":"2022-12-01T03:40:20.977518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[TARGET_COLS].value_counts() * 100 / len(df_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:29.791683Z","iopub.execute_input":"2022-12-01T03:40:29.792046Z","iopub.status.idle":"2022-12-01T03:40:29.808327Z","shell.execute_reply.started":"2022-12-01T03:40:29.792008Z","shell.execute_reply":"2022-12-01T03:40:29.80687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train) / (2 * np.bincount(df_train['cancer'].values))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:33.989602Z","iopub.execute_input":"2022-12-01T03:40:33.989941Z","iopub.status.idle":"2022-12-01T03:40:33.99694Z","shell.execute_reply.started":"2022-12-01T03:40:33.989916Z","shell.execute_reply":"2022-12-01T03:40:33.995866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial):\n    params = {\n        'metric': 'f1',\n        'random_state': 42,\n        'n_estimators': 300,\n        'learning_rate': 0.1,\n        'reg_alpha': trial.suggest_loguniform('reg_alpha', 1e-3, 10.0),\n        'reg_lambda': trial.suggest_loguniform('reg_lambda', 1e-3, 10.0),\n        'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.3,0.4,0.5,0.6,0.7,0.8,0.9, 1.0]),\n        'subsample': trial.suggest_categorical('subsample', [0.4,0.5,0.6,0.7,0.8,1.0]),\n        'max_depth': trial.suggest_categorical('max_depth', [10,20,100]),\n        'num_leaves' : trial.suggest_int('num_leaves', 1, 1000),\n        'min_child_samples': trial.suggest_int('min_child_samples', 1, 300),\n        'cat_smooth' : trial.suggest_int('min_data_per_groups', 1, 100)\n    }\n    model = lgb.LGBMClassifier(**params, zero_as_missing=True)\n\n    model.fit(X_train, y_train)\n\n    y_va_pred = model.predict(X_val)\n    f1 = f1_score(y_val, y_va_pred, pos_label=1, average='macro')\n    \n    return f1","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:43.04249Z","iopub.execute_input":"2022-12-01T03:40:43.042862Z","iopub.status.idle":"2022-12-01T03:40:43.051224Z","shell.execute_reply.started":"2022-12-01T03:40:43.042826Z","shell.execute_reply":"2022-12-01T03:40:43.050183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_tune_params:\n    study = optuna.create_study(\n        direction='maximize', \n        pruner=optuna.pruners.MedianPruner(n_warmup_steps=20),\n        study_name='RSNA')\n    study.optimize(objective, n_trials=20)\n    print(study.best_params)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:46.060916Z","iopub.execute_input":"2022-12-01T03:40:46.06127Z","iopub.status.idle":"2022-12-01T03:40:46.067913Z","shell.execute_reply.started":"2022-12-01T03:40:46.061242Z","shell.execute_reply":"2022-12-01T03:40:46.066593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params_tuned = {\n    'reg_alpha': 0.0028731193020013765,\n    'reg_lambda': 0.04370710510459441,\n    'colsample_bytree': 0.6,\n    'subsample': 0.7,\n    'max_depth': 20,\n    'num_leaves': 594,\n    'min_child_samples': 12,\n    'min_data_per_groups': 65\n}\n\nvalid_f1_scores = []\npredictions = []\n\nfor fold in range(N_FOLD):\n    print(f'\\n-----------FOLD {fold} ------------')\n    print('Data prepared.')\n    train_df = df_train[df_train['fold'] != fold].reset_index(drop=True)\n    valid_df = df_train[df_train['fold'] == fold].reset_index(drop=True)\n\n    dv = DictVectorizer(sparse=False)\n\n    train_dict = train_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_train = dv.fit_transform(train_dict)\n    y_train = train_df[TARGET_COLS].values\n\n    val_dict = valid_df[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_val = dv.transform(val_dict)\n    y_val = valid_df[TARGET_COLS].values\n\n    test_dict = df_test[CATEGORICAL_COL + NUMERICAL_COL].to_dict(orient=\"record\")\n    X_test = dv.transform(test_dict)\n\n#     save_dict = f\"./dict_fold{fold}.pkl\"\n#     pickle.dump(dv, open(save_dict, 'wb'))\n\n    class_weight_arr = len(train_df) / (2 * np.bincount(train_df['cancer'].values))\n\n    print('Model Training.')\n    tuned_model = lgb.LGBMClassifier(**params_tuned, \n                                    zero_as_missing=True, \n                                    class_weight={0: class_weight_arr[0], \n                                                  1: class_weight_arr[1]})\n    tuned_model.fit(X_train, y_train)\n\n    print('Model Inferencing.')\n    y_val_pred = tuned_model.predict(X_val)\n    y_test_pred = tuned_model.predict_proba(X_test)[:, 1]\n    \n    valid_f1_score = f1_score(y_val, y_val_pred, pos_label=1, average='macro')\n    print('Finished.')\n\n    valid_f1_scores.append(valid_f1_score)\n    predictions.append(y_test_pred)\n\n    del tuned_model\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:40:58.390929Z","iopub.execute_input":"2022-12-01T03:40:58.392372Z","iopub.status.idle":"2022-12-01T03:41:06.923333Z","shell.execute_reply.started":"2022-12-01T03:40:58.392322Z","shell.execute_reply":"2022-12-01T03:41:06.922644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'{len(valid_f1_scores)} Folds validation F1:\\n{valid_f1_scores}')\nprint(f'Local CV Average F1 score: {np.mean(valid_f1_scores)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:41:18.84892Z","iopub.execute_input":"2022-12-01T03:41:18.849287Z","iopub.status.idle":"2022-12-01T03:41:18.855482Z","shell.execute_reply.started":"2022-12-01T03:41:18.849256Z","shell.execute_reply":"2022-12-01T03:41:18.854374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = np.mean(predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:41:26.694405Z","iopub.execute_input":"2022-12-01T03:41:26.694773Z","iopub.status.idle":"2022-12-01T03:41:26.700579Z","shell.execute_reply.started":"2022-12-01T03:41:26.694742Z","shell.execute_reply":"2022-12-01T03:41:26.69947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"final_sub = pd.DataFrame()\nfinal_sub[\"prediction_id\"] = df_test['prediction_id']\nfinal_sub[\"cancer\"] = preds\nfinal_sub.to_csv('submission.csv', index=False)\nfinal_sub.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T03:41:32.213382Z","iopub.execute_input":"2022-12-01T03:41:32.213759Z","iopub.status.idle":"2022-12-01T03:41:32.231984Z","shell.execute_reply.started":"2022-12-01T03:41:32.213729Z","shell.execute_reply":"2022-12-01T03:41:32.230796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}