{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
RSNA Screening Mammography Breast Cancer Detection EDA
\n","metadata":{}},{"cell_type":"code","source":"#pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:46:57.343935Z","iopub.execute_input":"2022-12-01T17:46:57.344426Z","iopub.status.idle":"2022-12-01T17:46:57.35196Z","shell.execute_reply.started":"2022-12-01T17:46:57.344323Z","shell.execute_reply":"2022-12-01T17:46:57.350125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport pydicom\nimport glob\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\n\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:46:57.35512Z","iopub.execute_input":"2022-12-01T17:46:57.355709Z","iopub.status.idle":"2022-12-01T17:46:57.873762Z","shell.execute_reply.started":"2022-12-01T17:46:57.355672Z","shell.execute_reply":"2022-12-01T17:46:57.871952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data structure","metadata":{}},{"cell_type":"code","source":"import os\n\ndir_dict = {}\n\nfor dirname, _, filenames in os.walk('/kaggle/input/rsna-breast-cancer-detection/train_images'):\n if dirname == \"/kaggle/input/rsna-breast-cancer-detection/train_images\":\n continue\n dir_dict[dirname[-5:]] = len(filenames)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:46:57.875082Z","iopub.execute_input":"2022-12-01T17:46:57.875391Z","iopub.status.idle":"2022-12-01T17:47:10.171071Z","shell.execute_reply.started":"2022-12-01T17:46:57.875366Z","shell.execute_reply":"2022-12-01T17:47:10.170337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_files = pd.DataFrame({\"id\":dir_dict.keys(),\"amount\":dir_dict.values()})","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.172223Z","iopub.execute_input":"2022-12-01T17:47:10.172623Z","iopub.status.idle":"2022-12-01T17:47:10.182403Z","shell.execute_reply.started":"2022-12-01T17:47:10.172599Z","shell.execute_reply":"2022-12-01T17:47:10.180894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(id_files)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.186449Z","iopub.execute_input":"2022-12-01T17:47:10.186826Z","iopub.status.idle":"2022-12-01T17:47:10.201636Z","shell.execute_reply.started":"2022-12-01T17:47:10.186798Z","shell.execute_reply":"2022-12-01T17:47:10.200531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_files.amount.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.203411Z","iopub.execute_input":"2022-12-01T17:47:10.20406Z","iopub.status.idle":"2022-12-01T17:47:10.225303Z","shell.execute_reply.started":"2022-12-01T17:47:10.204025Z","shell.execute_reply":"2022-12-01T17:47:10.22426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(id_files, stat=\"probability\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.226531Z","iopub.execute_input":"2022-12-01T17:47:10.22693Z","iopub.status.idle":"2022-12-01T17:47:10.648725Z","shell.execute_reply.started":"2022-12-01T17:47:10.226902Z","shell.execute_reply":"2022-12-01T17:47:10.647755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observations \n**We have a set of folders with an id in each of which there are 4 or more DICOM files. Basically, each folder contains 4 files, there are folders in which there are more files. There are 11913 unique IDs in total**","metadata":{}},{"cell_type":"markdown","source":"# Probabilistic F1 score","metadata":{}},{"cell_type":"markdown","source":"Submissions are evaluated using the **probabilistic F1** score (pF1):\n\n**$pF_1 = 2\\frac{pPrecision \\cdot pRecall}{pPrecision+pRecall}$**\n\nwhere:\n\n**$pPrecision = \\frac{pTP}{pTP+pFP}$**\n\n**$pRecall = \\frac{pTP}{TP+FN}$**\n\n","metadata":{}},{"cell_type":"code","source":"# implementation code\n# source - https://www.kaggle.com/code/sohier/probabilistic-f-score/notebook\n\ndef pfbeta(labels, predictions, beta):\n y_true_count = 0\n ctp = 0\n cfp = 0\n\n for idx in range(len(labels)):\n prediction = min(max(predictions[idx], 0), 1)\n if (labels[idx]):\n y_true_count += 1\n ctp += prediction\n cfp += 1 - prediction\n else:\n cfp += prediction\n\n beta_squared = beta * beta\n c_precision = ctp / (ctp + cfp)\n c_recall = ctp / y_true_count\n if (c_precision > 0 and c_recall > 0):\n result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n return result\n else:\n return 0","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.649917Z","iopub.execute_input":"2022-12-01T17:47:10.650162Z","iopub.status.idle":"2022-12-01T17:47:10.658816Z","shell.execute_reply.started":"2022-12-01T17:47:10.650138Z","shell.execute_reply":"2022-12-01T17:47:10.656531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What about DICOM image format?","metadata":{}},{"cell_type":"markdown","source":"Here I used the functions and information from this notebook - https://www.kaggle.com/code/radek1/initial-eda-a-first-look-at-the-data","metadata":{}},{"cell_type":"markdown","source":"**Each file in this format stores an image with metadata about the patient**","metadata":{}},{"cell_type":"code","source":"pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10011/220375232.dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.660994Z","iopub.execute_input":"2022-12-01T17:47:10.661936Z","iopub.status.idle":"2022-12-01T17:47:10.679561Z","shell.execute_reply.started":"2022-12-01T17:47:10.661899Z","shell.execute_reply":"2022-12-01T17:47:10.678535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10025/288394860.dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.682363Z","iopub.execute_input":"2022-12-01T17:47:10.682702Z","iopub.status.idle":"2022-12-01T17:47:10.696514Z","shell.execute_reply.started":"2022-12-01T17:47:10.682668Z","shell.execute_reply":"2022-12-01T17:47:10.69493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/allunia/rsna-csf-cervical-spine-fracture-eda/notebook\ndef rescale_img_to_hu(dcm_ds):\n \"\"\"Rescales the image to Hounsfield unit.\"\"\"\n return dcm_ds.pixel_array * dcm_ds.RescaleSlope + dcm_ds.RescaleIntercept\n\ndef show_images_for_patient(patient_id):\n patient_dir = os.path.join('../input/rsna-breast-cancer-detection/train_images', str(patient_id))\n num_images = len(glob.glob(f\"{patient_dir}/*\"))\n print(f\"Number of images for patient: {num_images}\")\n fig, axs = plt.subplots(2, 2, figsize=(24,15))\n axs = axs.flatten()\n for i, img_path in enumerate(list(Path(patient_dir).iterdir())):\n ds = pydicom.dcmread(img_path)\n axs[i].imshow(rescale_img_to_hu(ds), cmap=\"bone\")","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.698364Z","iopub.execute_input":"2022-12-01T17:47:10.698819Z","iopub.status.idle":"2022-12-01T17:47:10.710881Z","shell.execute_reply.started":"2022-12-01T17:47:10.698792Z","shell.execute_reply":"2022-12-01T17:47:10.709068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_for_patient(10011)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:10.71353Z","iopub.execute_input":"2022-12-01T17:47:10.714024Z","iopub.status.idle":"2022-12-01T17:47:15.244627Z","shell.execute_reply.started":"2022-12-01T17:47:10.713996Z","shell.execute_reply":"2022-12-01T17:47:15.242945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_images_for_patient(10025)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:15.246441Z","iopub.execute_input":"2022-12-01T17:47:15.246843Z","iopub.status.idle":"2022-12-01T17:47:31.552934Z","shell.execute_reply.started":"2022-12-01T17:47:15.246808Z","shell.execute_reply":"2022-12-01T17:47:31.551532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Tabular data","metadata":{}},{"cell_type":"markdown","source":"## Columns descriptions \n","metadata":{}},{"cell_type":"code","source":"train_csv = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.557258Z","iopub.execute_input":"2022-12-01T17:47:31.557613Z","iopub.status.idle":"2022-12-01T17:47:31.622648Z","shell.execute_reply.started":"2022-12-01T17:47:31.557587Z","shell.execute_reply":"2022-12-01T17:47:31.621229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.623941Z","iopub.execute_input":"2022-12-01T17:47:31.624296Z","iopub.status.idle":"2022-12-01T17:47:31.632209Z","shell.execute_reply.started":"2022-12-01T17:47:31.624244Z","shell.execute_reply":"2022-12-01T17:47:31.630341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.63343Z","iopub.execute_input":"2022-12-01T17:47:31.633777Z","iopub.status.idle":"2022-12-01T17:47:31.656553Z","shell.execute_reply.started":"2022-12-01T17:47:31.633751Z","shell.execute_reply":"2022-12-01T17:47:31.6548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
    Missing columns in test data\n
  1. cancer
    2. \n
  2. invasive
    3. \n
  3. density
    4. \n
  4. difficult_negative_case
    5. \n
","metadata":{}},{"cell_type":"markdown","source":"**Age distribution**","metadata":{}},{"cell_type":"code","source":"sns.displot(data=train_csv, x=\"age\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.657864Z","iopub.execute_input":"2022-12-01T17:47:31.658163Z","iopub.status.idle":"2022-12-01T17:47:31.953828Z","shell.execute_reply.started":"2022-12-01T17:47:31.658137Z","shell.execute_reply":"2022-12-01T17:47:31.952261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Labels distribution**","metadata":{}},{"cell_type":"code","source":"train_csv.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.955407Z","iopub.execute_input":"2022-12-01T17:47:31.955822Z","iopub.status.idle":"2022-12-01T17:47:31.965792Z","shell.execute_reply.started":"2022-12-01T17:47:31.955787Z","shell.execute_reply":"2022-12-01T17:47:31.964273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=train_csv, x='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:31.967456Z","iopub.execute_input":"2022-12-01T17:47:31.967872Z","iopub.status.idle":"2022-12-01T17:47:32.066694Z","shell.execute_reply.started":"2022-12-01T17:47:31.967838Z","shell.execute_reply":"2022-12-01T17:47:32.065161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**missing data**","metadata":{}},{"cell_type":"code","source":"import missingno as msno\nmsno.matrix(train_csv, figsize = (10,5))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:32.06894Z","iopub.execute_input":"2022-12-01T17:47:32.069418Z","iopub.status.idle":"2022-12-01T17:47:32.442493Z","shell.execute_reply.started":"2022-12-01T17:47:32.069379Z","shell.execute_reply":"2022-12-01T17:47:32.441041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"code","source":"train_csv[train_csv.cancer == 1]","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:32.443928Z","iopub.execute_input":"2022-12-01T17:47:32.44459Z","iopub.status.idle":"2022-12-01T17:47:32.472955Z","shell.execute_reply.started":"2022-12-01T17:47:32.444553Z","shell.execute_reply":"2022-12-01T17:47:32.471653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv[train_csv.cancer == 0]","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:32.474309Z","iopub.execute_input":"2022-12-01T17:47:32.474971Z","iopub.status.idle":"2022-12-01T17:47:32.506241Z","shell.execute_reply.started":"2022-12-01T17:47:32.474935Z","shell.execute_reply":"2022-12-01T17:47:32.504824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Show example with cancer","metadata":{}},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10130/1013166704.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"With cancer 10130 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:32.507603Z","iopub.execute_input":"2022-12-01T17:47:32.508231Z","iopub.status.idle":"2022-12-01T17:47:34.670769Z","shell.execute_reply.started":"2022-12-01T17:47:32.508204Z","shell.execute_reply":"2022-12-01T17:47:34.669981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10226/1943220805.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"With cancer 10226 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:34.671795Z","iopub.execute_input":"2022-12-01T17:47:34.672686Z","iopub.status.idle":"2022-12-01T17:47:36.594636Z","shell.execute_reply.started":"2022-12-01T17:47:34.672653Z","shell.execute_reply":"2022-12-01T17:47:36.59371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = pydicom.dcmread(\"/kaggle/input/rsna-breast-cancer-detection/train_images/10011/1031443799.dcm\").pixel_array\n \nimg = (img - img.min()) / (img.max() - img.min())\n\nplt.figure(figsize=(15, 15))\nplt.imshow(img, cmap=\"gray\")\nplt.title(f\"Without cancer 10006 id\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T17:47:36.595759Z","iopub.execute_input":"2022-12-01T17:47:36.596037Z","iopub.status.idle":"2022-12-01T17:47:37.712175Z","shell.execute_reply.started":"2022-12-01T17:47:36.596Z","shell.execute_reply":"2022-12-01T17:47:37.711347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thank you very much for reading!**\n**If you found this notebook useful, please upvote!**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}