{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"try:\n    import pylibjpeg\nexcept:\n    !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-20T09:59:03.956535Z","iopub.execute_input":"2022-12-20T09:59:03.956929Z","iopub.status.idle":"2022-12-20T09:59:03.964909Z","shell.execute_reply.started":"2022-12-20T09:59:03.956897Z","shell.execute_reply":"2022-12-20T09:59:03.963474Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nsns.set_theme()\nimport matplotlib.pyplot as plt\nfrom scipy.stats import spearmanr","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-20T09:59:03.967068Z","iopub.execute_input":"2022-12-20T09:59:03.967557Z","iopub.status.idle":"2022-12-20T09:59:03.978623Z","shell.execute_reply.started":"2022-12-20T09:59:03.967488Z","shell.execute_reply":"2022-12-20T09:59:03.977403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/rsna-breast-cancer-detection\"\nTRAIN_DIR = os.path.join(DATA_DIR, \"train_images\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:03.980414Z","iopub.execute_input":"2022-12-20T09:59:03.980773Z","iopub.status.idle":"2022-12-20T09:59:03.989997Z","shell.execute_reply.started":"2022-12-20T09:59:03.980741Z","shell.execute_reply":"2022-12-20T09:59:03.988797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\nprint(train_df.shape)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:03.991278Z","iopub.execute_input":"2022-12-20T09:59:03.991674Z","iopub.status.idle":"2022-12-20T09:59:04.098201Z","shell.execute_reply.started":"2022-12-20T09:59:03.991641Z","shell.execute_reply":"2022-12-20T09:59:04.096842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Number of patients: {train_df['patient_id'].nunique()}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:04.101504Z","iopub.execute_input":"2022-12-20T09:59:04.102117Z","iopub.status.idle":"2022-12-20T09:59:04.111068Z","shell.execute_reply.started":"2022-12-20T09:59:04.10207Z","shell.execute_reply":"2022-12-20T09:59:04.10981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing values","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6, 8))\ntrain_df.isna().mean().plot(kind=\"barh\")\nplt.title(\"Missing value rates\");","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:04.112773Z","iopub.execute_input":"2022-12-20T09:59:04.113133Z","iopub.status.idle":"2022-12-20T09:59:04.475947Z","shell.execute_reply.started":"2022-12-20T09:59:04.1131Z","shell.execute_reply":"2022-12-20T09:59:04.474689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metadata","metadata":{}},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/allunia/rsna-breast-cancer-eda\nsns.set_theme()\nfig, axs = plt.subplots(nrows=2, ncols=2, figsize=(24, 12))\naxs = axs.flatten()\nsns.countplot(x=\"laterality\", data=train_df, ax=axs[0], palette=\"Greens_r\")\nsns.countplot(x=\"view\", data=train_df, ax=axs[1], palette=\"Reds_r\", order=train_df['view'].value_counts().index)\nsns.countplot(x=\"implant\", data=train_df, ax=axs[2], palette=\"Blues_r\")\nsns.countplot(x=\"density\", data=train_df, ax=axs[3], palette=\"Purples_r\", order=train_df['density'].value_counts().index)\nplt.suptitle(\"All images\");","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:04.477562Z","iopub.execute_input":"2022-12-20T09:59:04.478007Z","iopub.status.idle":"2022-12-20T09:59:05.677163Z","shell.execute_reply.started":"2022-12-20T09:59:04.477964Z","shell.execute_reply":"2022-12-20T09:59:05.675776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"view\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:05.678989Z","iopub.execute_input":"2022-12-20T09:59:05.679458Z","iopub.status.idle":"2022-12-20T09:59:05.695924Z","shell.execute_reply.started":"2022-12-20T09:59:05.679413Z","shell.execute_reply":"2022-12-20T09:59:05.694533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**\n* there are quite a lot of breasts with high density, this makes cancer detection harder\n\n**Mammography views**\n* standard views are bilateral craniocaudal (CC) and mediolateral oblique (MLO) views\n* generally, mammograms consist of 4 views: left and right CC and MLO \n* other views, i.e. ML, LM, AT and LMO are supplementary views\n* source: https://radiopaedia.org/articles/mammography-views","metadata":{}},{"cell_type":"code","source":"train_cancer_df = train_df[train_df[\"cancer\"] == 1]\nfig, axs = plt.subplots(nrows=2, ncols=2, figsize=(24, 12))\naxs = axs.flatten()\nsns.countplot(x=\"laterality\", data=train_cancer_df, ax=axs[0], palette=\"Greens_r\")\nsns.countplot(x=\"view\", data=train_cancer_df, ax=axs[1], palette=\"Reds_r\", order=train_cancer_df['view'].value_counts().index)\nsns.countplot(x=\"implant\", data=train_cancer_df, ax=axs[2], palette=\"Blues_r\")\nsns.countplot(x=\"density\", data=train_cancer_df, ax=axs[3], palette=\"Purples_r\", order=train_cancer_df['density'].value_counts().index)\nplt.suptitle(\"Images with breast cancer\");","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:05.697835Z","iopub.execute_input":"2022-12-20T09:59:05.698595Z","iopub.status.idle":"2022-12-20T09:59:06.354728Z","shell.execute_reply.started":"2022-12-20T09:59:05.698546Z","shell.execute_reply":"2022-12-20T09:59:06.353529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**\n* distribution of metadata features for breast cancer patients show similar patterns as features for all patients in the training set","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(28, 12))\nsns.countplot(x=\"cancer\", data=train_df, ax=ax1, palette=\"Greens_r\")\nax1.set_title(\"Image-level cancer\")\nsns.countplot(x=train_df.groupby(\"patient_id\")[\"cancer\"].max().to_numpy(), ax=ax2, palette=\"Reds_r\")\nax2.set_title(\"Patient-level cancer\")\nsns.countplot(x=train_df.groupby(\"patient_id\")[\"image_id\"].size().to_numpy(), ax=ax3, palette=\"Blues_r\")\nax3.set_title(\"Number of images per patient\");","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:06.358232Z","iopub.execute_input":"2022-12-20T09:59:06.358656Z","iopub.status.idle":"2022-12-20T09:59:06.973698Z","shell.execute_reply.started":"2022-12-20T09:59:06.35861Z","shell.execute_reply":"2022-12-20T09:59:06.972468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Correlation between number of images and cancer:\")\nspearmanr(train_df.groupby(\"patient_id\")[\"cancer\"].max().to_numpy(), train_df.groupby(\"patient_id\")[\"image_id\"].size().to_numpy(), nan_policy=\"omit\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:06.975735Z","iopub.execute_input":"2022-12-20T09:59:06.97649Z","iopub.status.idle":"2022-12-20T09:59:06.995591Z","shell.execute_reply.started":"2022-12-20T09:59:06.976449Z","shell.execute_reply":"2022-12-20T09:59:06.99438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**\n* huge class imbalance, both on image- and patient-level\n* most patients have the standard 4 mammography views, i.e. left and right CC and MLO\n* Spearman correlation between number of images and cancer shows very small positive association","metadata":{}},{"cell_type":"markdown","source":"## Age","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\ntrain_df.loc[train_df[\"cancer\"] == 0, \"age\"].plot.hist(bins=30, alpha=0.8, label=\"non-cancer\")\ntrain_df.loc[train_df[\"cancer\"] == 1, \"age\"].plot.hist(bins=30, alpha=0.8, label=\"cancer\")\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:06.997475Z","iopub.execute_input":"2022-12-20T09:59:06.998252Z","iopub.status.idle":"2022-12-20T09:59:07.497094Z","shell.execute_reply.started":"2022-12-20T09:59:06.998206Z","shell.execute_reply":"2022-12-20T09:59:07.495954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spearmanr(train_df[\"cancer\"], train_df[\"age\"], nan_policy=\"omit\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:07.498654Z","iopub.execute_input":"2022-12-20T09:59:07.499029Z","iopub.status.idle":"2022-12-20T09:59:07.552617Z","shell.execute_reply.started":"2022-12-20T09:59:07.498995Z","shell.execute_reply":"2022-12-20T09:59:07.551492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**:\n* patients with breast cancer are older on average than non-cancer patients\n* Spearman correlation shows a slight positive association between the two variables","metadata":{}},{"cell_type":"markdown","source":"## Show images","metadata":{}},{"cell_type":"code","source":"def show_imgs_for_patient(patient_id: int):\n    patient_dir = os.path.join(TRAIN_DIR, str(patient_id))\n    img_paths = glob.glob(f\"{patient_dir}/*\")   \n    fig, axs = plt.subplots(nrows=2, ncols=np.ceil(len(img_paths)/2).astype(int), figsize=(15,15))\n    plt.suptitle(f\"Number of images for patient {patient_id}: {len(img_paths)}\")\n    axs = axs.flatten()\n    for idx, img_path in enumerate(img_paths):\n        ds = pydicom.dcmread(img_path)\n        axs[idx].imshow(ds.pixel_array, cmap=\"bone\")\n        axs[idx].axis(\"off\")\n        \n        img_id = int(os.path.basename(img_path)[:-4])\n        cancer_label = train_df.loc[train_df[\"image_id\"] == img_id, \"cancer\"].values[0]\n        axs[idx].set_title(f\"Cancer label: {cancer_label}\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:07.554186Z","iopub.execute_input":"2022-12-20T09:59:07.555195Z","iopub.status.idle":"2022-12-20T09:59:07.564966Z","shell.execute_reply.started":"2022-12-20T09:59:07.555158Z","shell.execute_reply":"2022-12-20T09:59:07.563598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_imgs_for_patient(9840)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:07.566631Z","iopub.execute_input":"2022-12-20T09:59:07.567091Z","iopub.status.idle":"2022-12-20T09:59:12.85869Z","shell.execute_reply.started":"2022-12-20T09:59:07.567046Z","shell.execute_reply":"2022-12-20T09:59:12.857637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_imgs_for_patient(10226)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:12.860108Z","iopub.execute_input":"2022-12-20T09:59:12.861115Z","iopub.status.idle":"2022-12-20T09:59:28.93162Z","shell.execute_reply.started":"2022-12-20T09:59:12.861077Z","shell.execute_reply":"2022-12-20T09:59:28.93041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**\n* it is not straightforward to spot breast cancer with untrained human vision","metadata":{}},{"cell_type":"markdown","source":"## Difficult negative cases","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(24, 12))\naxs = axs.flatten()\ndiff_neg_df = train_df[train_df[\"difficult_negative_case\"]]\nsns.countplot(x=\"density\", data=diff_neg_df, ax=axs[0], palette=\"Greens_r\", order=diff_neg_df['density'].value_counts().index)\naxs[0].set_title(\"Difficult negative cases\")\n\nnot_diff_neg_df = train_df[(~train_df[\"difficult_negative_case\"]) & (train_df[\"cancer\"] == 0)]\nsns.countplot(x=\"density\", data=not_diff_neg_df, ax=axs[1], palette=\"Blues_r\", order=not_diff_neg_df['density'].value_counts().index)\naxs[1].set_title(\"Not difficult negative cases\");","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:28.93376Z","iopub.execute_input":"2022-12-20T09:59:28.93425Z","iopub.status.idle":"2022-12-20T09:59:29.392689Z","shell.execute_reply.started":"2022-12-20T09:59:28.934207Z","shell.execute_reply":"2022-12-20T09:59:29.391448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_diff_neg_imgs_for_patient(patient_id: int):\n    patient_dir = os.path.join(TRAIN_DIR, str(patient_id))\n    img_ids = train_df.loc[(train_df[\"patient_id\"] == patient_id) & (train_df[\"difficult_negative_case\"]), \"image_id\"].tolist()\n    img_paths = [os.path.join(patient_dir, f\"{img_id}.dcm\") for img_id in img_ids]\n    fig, axs = plt.subplots(nrows=1, ncols=len(img_paths), figsize=(10,10))\n    plt.suptitle(f\"Number of difficult negative images for patient {patient_id}: {len(img_paths)}\")\n    fig.tight_layout()\n    axs = axs.flatten()\n    for idx, img_path in enumerate(img_paths):\n        ds = pydicom.dcmread(img_path)\n        axs[idx].imshow(ds.pixel_array, cmap=\"bone\")\n        axs[idx].axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:29.394053Z","iopub.execute_input":"2022-12-20T09:59:29.394404Z","iopub.status.idle":"2022-12-20T09:59:29.403144Z","shell.execute_reply.started":"2022-12-20T09:59:29.394369Z","shell.execute_reply":"2022-12-20T09:59:29.401974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" show_diff_neg_imgs_for_patient(10011)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:29.404399Z","iopub.execute_input":"2022-12-20T09:59:29.404782Z","iopub.status.idle":"2022-12-20T09:59:32.09742Z","shell.execute_reply.started":"2022-12-20T09:59:29.404749Z","shell.execute_reply":"2022-12-20T09:59:32.096542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Takeaways**\n* as expected, difficult negative cases have overall higher density than not difficult negatives","metadata":{}},{"cell_type":"markdown","source":"## Baseline submission (age normalized)\n\nWe saw that there is a weak positive relationship between age and cancer label, so let's normalize age in the test set and submit that as our predictions as a baseline.","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv(os.path.join(DATA_DIR, \"test.csv\"))\nprint(test_df.shape)\ntest_df","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:32.098528Z","iopub.execute_input":"2022-12-20T09:59:32.099391Z","iopub.status.idle":"2022-12-20T09:59:32.118436Z","shell.execute_reply.started":"2022-12-20T09:59:32.099352Z","shell.execute_reply":"2022-12-20T09:59:32.117589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = test_df.copy()\nsubmission_df[\"age\"] = submission_df[\"age\"].fillna(submission_df[\"age\"].median())\nsubmission_df[\"cancer\"] = submission_df[\"age\"] / submission_df[\"age\"].max()\nsubmission_df = submission_df[[\"prediction_id\", \"cancer\"]].drop_duplicates(subset=\"prediction_id\")\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:32.11974Z","iopub.execute_input":"2022-12-20T09:59:32.120491Z","iopub.status.idle":"2022-12-20T09:59:32.135816Z","shell.execute_reply.started":"2022-12-20T09:59:32.120457Z","shell.execute_reply":"2022-12-20T09:59:32.134447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-20T09:59:32.137258Z","iopub.execute_input":"2022-12-20T09:59:32.137688Z","iopub.status.idle":"2022-12-20T09:59:32.146806Z","shell.execute_reply.started":"2022-12-20T09:59:32.13763Z","shell.execute_reply":"2022-12-20T09:59:32.145735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Please upvote if you liked this notebook! Thanks for your support!**","metadata":{}}]}