{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
RSNA Screening Mammography Breast Cancer Detection EDA
\n","metadata":{}},{"cell_type":"code","source":"#pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:06.495686Z","iopub.execute_input":"2022-12-04T02:04:06.496161Z","iopub.status.idle":"2022-12-04T02:04:06.502176Z","shell.execute_reply.started":"2022-12-04T02:04:06.496116Z","shell.execute_reply":"2022-12-04T02:04:06.500625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport pydicom\nimport glob\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\n\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:06.503994Z","iopub.execute_input":"2022-12-04T02:04:06.504556Z","iopub.status.idle":"2022-12-04T02:04:06.515435Z","shell.execute_reply.started":"2022-12-04T02:04:06.504507Z","shell.execute_reply":"2022-12-04T02:04:06.513831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data structure","metadata":{}},{"cell_type":"code","source":"import os\n\ndir_dict = {}\n\nfor dirname, _, filenames in os.walk('/kaggle/input/rsna-breast-cancer-detection/train_images'):\n if dirname == \"/kaggle/input/rsna-breast-cancer-detection/train_images\":\n continue\n dir_dict[dirname[-5:]] = len(filenames)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:06.518625Z","iopub.execute_input":"2022-12-04T02:04:06.519354Z","iopub.status.idle":"2022-12-04T02:04:15.553051Z","shell.execute_reply.started":"2022-12-04T02:04:06.519301Z","shell.execute_reply":"2022-12-04T02:04:15.551989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_files = pd.DataFrame({\"id\":dir_dict.keys(),\"amount\":dir_dict.values()})","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:15.55449Z","iopub.execute_input":"2022-12-04T02:04:15.555054Z","iopub.status.idle":"2022-12-04T02:04:15.56522Z","shell.execute_reply.started":"2022-12-04T02:04:15.555019Z","shell.execute_reply":"2022-12-04T02:04:15.564114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(id_files)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:15.566418Z","iopub.execute_input":"2022-12-04T02:04:15.566784Z","iopub.status.idle":"2022-12-04T02:04:15.583058Z","shell.execute_reply.started":"2022-12-04T02:04:15.566753Z","shell.execute_reply":"2022-12-04T02:04:15.581765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_files.amount.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:15.584649Z","iopub.execute_input":"2022-12-04T02:04:15.585065Z","iopub.status.idle":"2022-12-04T02:04:15.597365Z","shell.execute_reply.started":"2022-12-04T02:04:15.585032Z","shell.execute_reply":"2022-12-04T02:04:15.596023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(id_files, stat=\"probability\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:15.598993Z","iopub.execute_input":"2022-12-04T02:04:15.599944Z","iopub.status.idle":"2022-12-04T02:04:16.159473Z","shell.execute_reply.started":"2022-12-04T02:04:15.599907Z","shell.execute_reply":"2022-12-04T02:04:16.158049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations \n**We have a set of folders with an id in each of which there are 4 or more DICOM files. Basically, each folder contains 4 files, there are folders in which there are more files. There are 11913 unique IDs in total**","metadata":{}},{"cell_type":"markdown","source":"# Probabilistic F1 score","metadata":{}},{"cell_type":"markdown","source":"Submissions are evaluated using the **probabilistic F1** score (pF1):\n\n**$pF_1 = 2\\frac{pPrecision \\cdot pRecall}{pPrecision+pRecall}$**\n\nwhere:\n\n**$pPrecision = \\frac{pTP}{pTP+pFP}$**\n\n**$pRecall = \\frac{pTP}{TP+FN}$**\n\n","metadata":{}},{"cell_type":"code","source":"# implementation code\n# source - https://www.kaggle.com/code/sohier/probabilistic-f-score/notebook\n\ndef pfbeta(labels, predictions, beta):\n y_true_count = 0\n ctp = 0\n cfp = 0\n\n for idx in range(len(labels)):\n prediction = min(max(predictions[idx], 0), 1)\n if (labels[idx]):\n y_true_count += 1\n ctp += prediction\n cfp += 1 - prediction\n else:\n cfp += prediction\n\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return result\n else:\n return 0","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:16.161235Z","iopub.execute_input":"2022-12-04T02:04:16.162157Z","iopub.status.idle":"2022-12-04T02:04:16.17163Z","shell.execute_reply.started":"2022-12-04T02:04:16.162109Z","shell.execute_reply":"2022-12-04T02:04:16.169696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What about DICOM image format?","metadata":{}},{"cell_type":"markdown","source":"Here I used the functions and information from this notebook - https://www.kaggle.com/code/radek1/initial-eda-a-first-look-at-the-data","metadata":{}},{"cell_type":"markdown","source":"**Each file in this format stores an image with metadata about the patient**","metadata":{}},{"cell_type":"code","source":"pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10011/220375232.dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:16.174162Z","iopub.execute_input":"2022-12-04T02:04:16.174784Z","iopub.status.idle":"2022-12-04T02:04:16.194712Z","shell.execute_reply.started":"2022-12-04T02:04:16.174729Z","shell.execute_reply":"2022-12-04T02:04:16.192728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10025/288394860.dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:16.196769Z","iopub.execute_input":"2022-12-04T02:04:16.19759Z","iopub.status.idle":"2022-12-04T02:04:16.212045Z","shell.execute_reply.started":"2022-12-04T02:04:16.197545Z","shell.execute_reply":"2022-12-04T02:04:16.210482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/allunia/rsna-csf-cervical-spine-fracture-eda/notebook\ndef rescale_img_to_hu(dcm_ds):\n \"\"\"Rescales the image to Hounsfield unit.\"\"\"\n return dcm_ds.pixel_array * dcm_ds.RescaleSlope + dcm_ds.RescaleIntercept\n\ndef show_images_for_patient(patient_id):\n patient_dir = os.path.join('../input/rsna-breast-cancer-detection/train_images', str(patient_id))\n num_images = len(glob.glob(f\"{patient_dir}/*\"))\n print(f\"Number of images for patient: {num_images}\")\n fig, axs = plt.subplots(2, 2, figsize=(24,15))\n axs = axs.flatten()\n for i, img_path in enumerate(list(Path(patient_dir).iterdir())):\n ds = pydicom.dcmread(img_path)\n axs[i].imshow(rescale_img_to_hu(ds), cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:16.213889Z","iopub.execute_input":"2022-12-04T02:04:16.214332Z","iopub.status.idle":"2022-12-04T02:04:16.225359Z","shell.execute_reply.started":"2022-12-04T02:04:16.214293Z","shell.execute_reply":"2022-12-04T02:04:16.223313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_for_patient(10011)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:16.227877Z","iopub.execute_input":"2022-12-04T02:04:16.228444Z","iopub.status.idle":"2022-12-04T02:04:22.612617Z","shell.execute_reply.started":"2022-12-04T02:04:16.228389Z","shell.execute_reply":"2022-12-04T02:04:22.611367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_for_patient(10025)","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:22.614336Z","iopub.execute_input":"2022-12-04T02:04:22.615757Z","iopub.status.idle":"2022-12-04T02:04:46.819404Z","shell.execute_reply.started":"2022-12-04T02:04:22.615696Z","shell.execute_reply":"2022-12-04T02:04:46.817477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Tabular data","metadata":{}},{"cell_type":"markdown","source":"## Columns descriptions \n","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:46.821663Z","iopub.execute_input":"2022-12-04T02:04:46.822152Z","iopub.status.idle":"2022-12-04T02:04:46.92318Z","shell.execute_reply.started":"2022-12-04T02:04:46.82211Z","shell.execute_reply":"2022-12-04T02:04:46.921482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:46.924736Z","iopub.execute_input":"2022-12-04T02:04:46.925223Z","iopub.status.idle":"2022-12-04T02:04:46.936855Z","shell.execute_reply.started":"2022-12-04T02:04:46.925187Z","shell.execute_reply":"2022-12-04T02:04:46.935384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:46.946134Z","iopub.execute_input":"2022-12-04T02:04:46.946595Z","iopub.status.idle":"2022-12-04T02:04:46.971429Z","shell.execute_reply.started":"2022-12-04T02:04:46.946559Z","shell.execute_reply":"2022-12-04T02:04:46.970029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
    Missing columns in test data\n
  1. cancer
    2. \n
  2. invasive
    3. \n
  3. density
    4. \n
  4. difficult_negative_case
    5. \n
","metadata":{}},{"cell_type":"markdown","source":"**Age distribution**","metadata":{}},{"cell_type":"code","source":"sns.displot(data=train_csv, x=\"age\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:46.973588Z","iopub.execute_input":"2022-12-04T02:04:46.974089Z","iopub.status.idle":"2022-12-04T02:04:47.466426Z","shell.execute_reply.started":"2022-12-04T02:04:46.974048Z","shell.execute_reply":"2022-12-04T02:04:47.465264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Labels distribution**","metadata":{}},{"cell_type":"code","source":"train_csv.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:47.468423Z","iopub.execute_input":"2022-12-04T02:04:47.469301Z","iopub.status.idle":"2022-12-04T02:04:47.479549Z","shell.execute_reply.started":"2022-12-04T02:04:47.469257Z","shell.execute_reply":"2022-12-04T02:04:47.478165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_csv, x='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:47.481382Z","iopub.execute_input":"2022-12-04T02:04:47.482173Z","iopub.status.idle":"2022-12-04T02:04:47.678084Z","shell.execute_reply.started":"2022-12-04T02:04:47.48212Z","shell.execute_reply":"2022-12-04T02:04:47.676566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**missing data**","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nmsno.matrix(train_csv, figsize = (10,5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:47.679932Z","iopub.execute_input":"2022-12-04T02:04:47.680488Z","iopub.status.idle":"2022-12-04T02:04:48.330014Z","shell.execute_reply.started":"2022-12-04T02:04:47.680437Z","shell.execute_reply":"2022-12-04T02:04:48.32868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"train_csv[train_csv.cancer == 1]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:48.331509Z","iopub.execute_input":"2022-12-04T02:04:48.331883Z","iopub.status.idle":"2022-12-04T02:04:48.362653Z","shell.execute_reply.started":"2022-12-04T02:04:48.331841Z","shell.execute_reply":"2022-12-04T02:04:48.36151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv[train_csv.cancer == 0]","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:48.364169Z","iopub.execute_input":"2022-12-04T02:04:48.364648Z","iopub.status.idle":"2022-12-04T02:04:48.398449Z","shell.execute_reply.started":"2022-12-04T02:04:48.364616Z","shell.execute_reply":"2022-12-04T02:04:48.397029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show example with cancer","metadata":{}},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10130/1013166704.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"With cancer 10130 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:48.400631Z","iopub.execute_input":"2022-12-04T02:04:48.401021Z","iopub.status.idle":"2022-12-04T02:04:51.222344Z","shell.execute_reply.started":"2022-12-04T02:04:48.400981Z","shell.execute_reply":"2022-12-04T02:04:51.221277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10226/1943220805.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"With cancer 10226 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:51.223633Z","iopub.execute_input":"2022-12-04T02:04:51.224833Z","iopub.status.idle":"2022-12-04T02:04:53.903652Z","shell.execute_reply.started":"2022-12-04T02:04:51.224793Z","shell.execute_reply":"2022-12-04T02:04:53.902206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10011/1031443799.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"Without cancer 10006 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-04T02:04:53.905497Z","iopub.execute_input":"2022-12-04T02:04:53.905895Z","iopub.status.idle":"2022-12-04T02:04:55.414059Z","shell.execute_reply.started":"2022-12-04T02:04:53.905859Z","shell.execute_reply":"2022-12-04T02:04:55.413072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thank you very much for reading!**\n**If you found this notebook useful, please upvote!**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}