{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA Fracture Detection - DICOM to 3D Numpy Volume\n\n![CT Image](https://storage.googleapis.com/kaggle-datasets-images/674071/1185670/0d449dc88ac1318321ae8f7de974f0fa/dataset-cover.jpg?t=2020-05-25-13-21-33)\n\n- **Author:** *Mariusz Wiśniewski*\n- **Competition:** *[RSNA 2022 Cervical Spine Fracture Detection](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection)*\n- **Date created:** *December 29th, 2022*\n- **Last modified:** *December 29th, 2022*\n\n## Overview\n\nIn this notebook we will prepare a dataset for object detection task by converting DICOM files to PNG format.\n\n### Libraries Used\n\n- [OpenCV 🖼️](https://opencv.org)\n- [Pydicom 🩻](https://pydicom.github.io)","metadata":{}},{"cell_type":"markdown","source":"# Notebook Steup","metadata":{}},{"cell_type":"markdown","source":"## Required Packages","metadata":{}},{"cell_type":"code","source":"# install pydicom requirements\n!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' --offline -yq\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' --offline -yq\n!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2 -q\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' --offline -yq\n!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' --offline -yq\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' --offline -yq\n!rm -rf gdcm/ gdcm.tar","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":false,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2022-12-29T01:30:48.681632Z","iopub.status.busy":"2022-12-29T01:30:48.680941Z","iopub.status.idle":"2022-12-29T01:31:33.062801Z","shell.execute_reply":"2022-12-29T01:31:33.061959Z","shell.execute_reply.started":"2022-12-29T01:30:48.681535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Statements","metadata":{}},{"cell_type":"code","source":"import gc\nimport glob\nimport os\nimport random\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pydicom\nfrom joblib import Parallel, delayed\nfrom matplotlib import animation, rc\nfrom scipy.ndimage import zoom\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.065498Z","iopub.status.busy":"2022-12-29T01:31:33.06467Z","iopub.status.idle":"2022-12-29T01:31:33.670657Z","shell.execute_reply":"2022-12-29T01:31:33.669991Z","shell.execute_reply.started":"2022-12-29T01:31:33.065462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Seed","metadata":{}},{"cell_type":"code","source":"seed = 42\nrandom.seed(seed)","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.672383Z","iopub.status.busy":"2022-12-29T01:31:33.671607Z","iopub.status.idle":"2022-12-29T01:31:33.676316Z","shell.execute_reply":"2022-12-29T01:31:33.675329Z","shell.execute_reply.started":"2022-12-29T01:31:33.672357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing DICOM Files","metadata":{}},{"cell_type":"markdown","source":"## Loading Data","metadata":{}},{"cell_type":"code","source":"TRAIN_IMG_PATH = '../input/rsna-2022-cervical-spine-fracture-detection/train_images/'\nTRAIN_CSV_PATH = '../input/rsna-2022-cervical-spine-fracture-detection/train.csv'\nTRAIN_BBOX_CSV_PATH = (\n    '../input/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv'\n)\n\ntrain_images = os.listdir(TRAIN_IMG_PATH)\n\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\ntrain_bbox_df = pd.read_csv(TRAIN_BBOX_CSV_PATH)","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.679981Z","iopub.status.busy":"2022-12-29T01:31:33.679744Z","iopub.status.idle":"2022-12-29T01:31:33.809654Z","shell.execute_reply":"2022-12-29T01:31:33.809028Z","shell.execute_reply.started":"2022-12-29T01:31:33.679959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cases = os.listdir(TRAIN_IMG_PATH)\nfractured_cases = list(\n    set(train_bbox_df['StudyInstanceUID'].unique()).intersection(cases))\nprint(\n    f'Number of fractured cases with bbox annotations: {len(fractured_cases)}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.810963Z","iopub.status.busy":"2022-12-29T01:31:33.810614Z","iopub.status.idle":"2022-12-29T01:31:33.822603Z","shell.execute_reply":"2022-12-29T01:31:33.821814Z","shell.execute_reply.started":"2022-12-29T01:31:33.810939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Output Directory","metadata":{}},{"cell_type":"code","source":"OUTPUT_PATH = 'train_images'\n\nif not os.path.exists(OUTPUT_PATH):\n    os.mkdir(OUTPUT_PATH)","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.830415Z","iopub.status.busy":"2022-12-29T01:31:33.829774Z","iopub.status.idle":"2022-12-29T01:31:33.838569Z","shell.execute_reply":"2022-12-29T01:31:33.837718Z","shell.execute_reply.started":"2022-12-29T01:31:33.830383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing Functions\n\nHere we define several helper functions to process the data.","metadata":{}},{"cell_type":"code","source":"def load_dicom_slice(path):\n    \"\"\"Read and load a single slice\"\"\"\n    dicom = pydicom.read_file(path)\n    data = dicom.pixel_array\n\n    return np.array(data).astype('float32')\n\n\ndef normalize(image):\n    \"\"\"Normalize the volume\"\"\"\n    image = (image - np.min(image)) / (np.max(image) - np.min(image))\n    # using uint8 to save memory\n    image = (image * 255).astype(np.uint8)\n\n    return image\n\n\ndef process_scan(path):\n    \"\"\"Read and normalize a slice\"\"\"\n    image = load_dicom_slice(path)\n    image = normalize(image)\n\n    return image","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:31:33.840198Z","iopub.status.busy":"2022-12-29T01:31:33.839659Z","iopub.status.idle":"2022-12-29T01:31:33.84689Z","shell.execute_reply":"2022-12-29T01:31:33.846323Z","shell.execute_reply.started":"2022-12-29T01:31:33.840171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving Slices as PNG","metadata":{}},{"cell_type":"markdown","source":"## Slices with a fracture","metadata":{}},{"cell_type":"code","source":"train_data = []\nfor case in fractured_cases:\n    bbox_slices = train_bbox_df[train_bbox_df['StudyInstanceUID']\n                                == case]['slice_number'].values\n    for slice_idx in bbox_slices:\n        slice_path = f'{TRAIN_IMG_PATH}/{case}/{slice_idx}.dcm'\n        image = process_scan(slice_path)\n\n        train_data.append({\n            'StudyInstanceUID': case,\n            'slice_number': slice_idx,\n            'width': image.shape[1],\n            'height': image.shape[0],\n            'label': 1\n        })\n\n        cv2.imwrite(f'{OUTPUT_PATH}/{case}_{slice_idx}.png', image)\n        del image\n\nprint(f'Number of fractured samples: {len(train_data)}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:42:34.903358Z","iopub.status.busy":"2022-12-29T01:42:34.903029Z","iopub.status.idle":"2022-12-29T01:42:34.982074Z","shell.execute_reply":"2022-12-29T01:42:34.981252Z","shell.execute_reply.started":"2022-12-29T01:42:34.903332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Slices without any fractures","metadata":{}},{"cell_type":"code","source":"unfractured_cases = train_df[train_df['patient_overall']\n                             == 0]['StudyInstanceUID']\nprint(f'Number of cases without a fracture: {len(unfractured_cases)}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:42:38.81347Z","iopub.status.busy":"2022-12-29T01:42:38.81316Z","iopub.status.idle":"2022-12-29T01:42:38.818185Z","shell.execute_reply":"2022-12-29T01:42:38.817551Z","shell.execute_reply.started":"2022-12-29T01:42:38.813447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unfractured_data = []\nfor case in unfractured_cases:\n    slices = [path.split('.')[0]\n              for path in os.listdir(f'{TRAIN_IMG_PATH}/{case}')]\n    for slice_idx in slices:\n        slice_path = f'{TRAIN_IMG_PATH}/{case}/{slice_idx}.dcm'\n        image = process_scan(slice_path)\n\n        unfractured_data.append({\n            'StudyInstanceUID': case,\n            'slice_number': slice_idx,\n            'width': image.shape[1],\n            'height': image.shape[0],\n            'label': 0\n        })\n\n        del image\n\nprint(f'Number of unfractured samples: {len(unfractured_data)}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:42:43.741218Z","iopub.status.busy":"2022-12-29T01:42:43.740576Z","iopub.status.idle":"2022-12-29T01:42:43.798896Z","shell.execute_reply":"2022-12-29T01:42:43.798123Z","shell.execute_reply.started":"2022-12-29T01:42:43.741189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To make the sets balanced, we'll take *n* randomly selected slices from `unfractured_paths`, where *n* corresponds to the number of slices with a fracture.","metadata":{}},{"cell_type":"code","source":"unfractured_data = random.sample(unfractured_data, len(train_data))\n\nfor item in unfractured_data:\n    slice_path = f'{TRAIN_IMG_PATH}/{item[\"StudyInstanceUID\"]}/{item[\"slice_number\"]}.dcm'\n    image = process_scan(slice_path)\n\n    cv2.imwrite(\n        f'{OUTPUT_PATH}/{item[\"StudyInstanceUID\"]}_{item[\"slice_number\"]}.png', image)\n    del image\n\nprint(\n    f'Number of unfractured samples after resampling: {len(unfractured_data)}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:42:46.842878Z","iopub.status.busy":"2022-12-29T01:42:46.842137Z","iopub.status.idle":"2022-12-29T01:42:46.854806Z","shell.execute_reply":"2022-12-29T01:42:46.854181Z","shell.execute_reply.started":"2022-12-29T01:42:46.842851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create CSV","metadata":{}},{"cell_type":"code","source":"train_data.extend(unfractured_data)\ntrain_data_df = pd.DataFrame(train_data)\ntrain_data_df.to_csv('train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-29T02:08:00.269997Z","iopub.execute_input":"2022-12-29T02:08:00.270475Z","iopub.status.idle":"2022-12-29T02:08:00.279317Z","shell.execute_reply.started":"2022-12-29T02:08:00.27045Z","shell.execute_reply":"2022-12-29T02:08:00.278416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary","metadata":{}},{"cell_type":"code","source":"print(\n    f'Number of scans without any fracture: {len(train_data_df[train_data_df[\"label\"] == 0])}')\nprint(\n    f'Number of scans with a fracture: {len(train_data_df[train_data_df[\"label\"] == 1])}')","metadata":{"execution":{"iopub.execute_input":"2022-12-29T02:05:02.553601Z","iopub.status.busy":"2022-12-29T02:05:02.552891Z","iopub.status.idle":"2022-12-29T02:05:02.559038Z","shell.execute_reply":"2022-12-29T02:05:02.558142Z","shell.execute_reply.started":"2022-12-29T02:05:02.553574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Zip Output","metadata":{}},{"cell_type":"code","source":"!tar -czf output.tar.gz --remove-files ./*","metadata":{"execution":{"iopub.execute_input":"2022-12-29T01:59:06.376212Z","iopub.status.busy":"2022-12-29T01:59:06.375907Z","iopub.status.idle":"2022-12-29T01:59:06.659694Z","shell.execute_reply":"2022-12-29T01:59:06.658741Z","shell.execute_reply.started":"2022-12-29T01:59:06.376186Z"},"trusted":true},"execution_count":null,"outputs":[]}]}