{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Some required installation for `PYDICOM` data**. As stated in the [data section](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data):\n\n> The **DICOM** image files are ≤ 1 mm slice thickness, axial orientation, and bone kernel. Note that some of the DICOM files are **JPEG compressed**. **You may require additional resources to read the pixel array of these files, such as GDCM and pylibjpeg.**","metadata":{}},{"cell_type":"code","source":"# Required for some dicom files.\nfrom IPython.display import clear_output\n!pip install -q ../input/for-pydicom/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install -q ../input/for-pydicom/python_gdcm-3.0.14-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install -q ../input/for-pydicom/pylibjpeg_libjpeg-1.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\nclear_output()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-23T14:03:25.73525Z","iopub.execute_input":"2022-08-23T14:03:25.735758Z","iopub.status.idle":"2022-08-23T14:05:02.367282Z","shell.execute_reply.started":"2022-08-23T14:03:25.735667Z","shell.execute_reply":"2022-08-23T14:05:02.366033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport itertools\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom random import sample\nfrom matplotlib import pyplot as plt\n\nimport pydicom\nimport nibabel as nib\nfrom glob import glob\nfrom tqdm import tqdm\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\npydicom.__version__, nib.__version__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-23T14:11:12.014968Z","iopub.execute_input":"2022-08-23T14:11:12.015443Z","iopub.status.idle":"2022-08-23T14:11:12.026722Z","shell.execute_reply.started":"2022-08-23T14:11:12.015404Z","shell.execute_reply":"2022-08-23T14:11:12.025761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility","metadata":{}},{"cell_type":"code","source":"def read_dicom(path, voi_lut = True, fix_monochrome = True):\n    '''ref: https://www.kaggle.com/code/raddar/convert-dicom-to-np-array-the-correct-way\n    '''\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n        \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n    return data, dicom","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:11:31.881947Z","iopub.execute_input":"2022-08-23T14:11:31.882758Z","iopub.status.idle":"2022-08-23T14:11:31.891011Z","shell.execute_reply.started":"2022-08-23T14:11:31.882711Z","shell.execute_reply":"2022-08-23T14:11:31.89003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_nibabel(path):\n    return nib.load(path).get_fdata()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:11:33.078153Z","iopub.execute_input":"2022-08-23T14:11:33.078727Z","iopub.status.idle":"2022-08-23T14:11:33.084083Z","shell.execute_reply.started":"2022-08-23T14:11:33.078677Z","shell.execute_reply":"2022-08-23T14:11:33.083276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def multi_dim_plot(multi_dim_array, id, num_slices=64):\n    fig = plt.figure(figsize=(30, 30))\n    plt.title(\n        f'Plotting first {num_slices} slices of {id}', \n        fontdict = {'fontsize' : 20}\n    )\n    plt.yticks([])\n    plt.xticks([])\n    \n    xy = int(np.sqrt(num_slices))\n    for i in range(num_slices):\n        ax = fig.add_subplot(xy, xy, i + 1)\n        plt.imshow(multi_dim_array[..., :num_slices][..., i])\n        plt.axis(\"off\")\n    plt.show()\n    \n    \ndef batch_plot(samples, cmap=None):\n    plt.figure(figsize=(30, 30))\n    \n    xy = int(np.sqrt(samples.shape[0]))\n    for i, sample in enumerate(samples):\n        try:\n            ax = plt.subplot(xy, xy, i + 1)\n        except:\n            ax = plt.subplot(8, 8, i + 1)\n        plt.axis('off')\n        plt.imshow(sample, cmap=cmap)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:22:23.462097Z","iopub.execute_input":"2022-08-23T14:22:23.462922Z","iopub.status.idle":"2022-08-23T14:22:23.473975Z","shell.execute_reply.started":"2022-08-23T14:22:23.462863Z","shell.execute_reply":"2022-08-23T14:22:23.473091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [Problem Framing](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data)\n\nThe goal of this competition is to **identify fractures in CT scans of the cervical spine (neck) at both the level of a single vertebrae and the entire patient.** Quickly detecting and determining the location of any vertebral fractures is essential to prevent neurologic deterioration and paralysis after trauma. This competition has provided many different types of annotation formatted data to solve the problem, such as meta information, segmentation mask, and bounding box.","metadata":{}},{"cell_type":"markdown","source":"# [Segmentation Data](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data?select=segmentations)\n\nPixel level annotations for a subset of the training set. This data is provided in the nifti file format. **A portion of the imaging datasets have been segmented automatically using a 3D UNET model, and radiologists modified and approved the segmentations.** The provided segmentation labels have values of 1 to 7 for C1 to C7 (seven cervical vertebrae) and 8 to 19 for T1 to T12 (twelve thoracic vertebrae are located in the center of your upper and middle back), and 0 for everything else. As we focused on the cervical spine, all scans have C1 to C7 labels but not all thoracic labels.\n\n\n> Please be aware that the **NIFTI** files consist of segmentation in the **sagittal plane**, while the **DICOM** files are in the **axial plane**. Please use the **NIFTI** header information to determine the appropriate orientation such that the **DICOM** images and segmentation match. Otherwise, you run the risk of having the segmentations flipped in the **Z axis** and mirrored in the **X axis**.\n\nRelevant and Important Resources:\n\n- [Connecting voxel spaces](https://www.kaggle.com/code/boojum/connecting-voxel-spaces)\n- [Determining MR Image Planes](https://www.kaggle.com/code/davidbroberts/determining-mr-image-planes)\n- [Normalized Voxels: Align Planes and Crop](https://www.kaggle.com/code/ren4yu/normalized-voxels-align-planes-and-crop)","metadata":{}},{"cell_type":"code","source":"segmentation_masks = sorted(\n    glob(\n        '../input/rsna-2022-cervical-spine-fracture-detection/segmentations/*.nii'\n    )\n)\n\n# num of segmentation mask\nlen(segmentation_masks)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:15:54.832439Z","iopub.execute_input":"2022-08-23T14:15:54.832853Z","iopub.status.idle":"2022-08-23T14:15:54.843009Z","shell.execute_reply.started":"2022-08-23T14:15:54.832818Z","shell.execute_reply":"2022-08-23T14:15:54.841621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for msk in sample(segmentation_masks, 2):\n    m = read_nibabel(msk)\n    multi_dim_plot(m, id=msk.split('/')[-1])","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:16:29.617498Z","iopub.execute_input":"2022-08-23T14:16:29.617927Z","iopub.status.idle":"2022-08-23T14:16:43.093491Z","shell.execute_reply.started":"2022-08-23T14:16:29.617893Z","shell.execute_reply":"2022-08-23T14:16:43.091776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [Data Frame](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data)\n\n**Metadata for the train test set.**","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/train.csv')\ntrain_df.head(15)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:10.49123Z","iopub.execute_input":"2022-08-23T14:17:10.492436Z","iopub.status.idle":"2022-08-23T14:17:10.540496Z","shell.execute_reply.started":"2022-08-23T14:17:10.492376Z","shell.execute_reply":"2022-08-23T14:17:10.539502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:13.318749Z","iopub.execute_input":"2022-08-23T14:17:13.319184Z","iopub.status.idle":"2022-08-23T14:17:13.347359Z","shell.execute_reply.started":"2022-08-23T14:17:13.319149Z","shell.execute_reply":"2022-08-23T14:17:13.34616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/test.csv')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:15.209287Z","iopub.execute_input":"2022-08-23T14:17:15.210104Z","iopub.status.idle":"2022-08-23T14:17:15.227472Z","shell.execute_reply.started":"2022-08-23T14:17:15.210064Z","shell.execute_reply":"2022-08-23T14:17:15.226346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [DICOM](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data)\n\n**The image data, organized with one folder per scan. Expect to see roughly 1,500 scans in the hidden test set. Each image is in the dicom file format.**\n\n> Please be aware that the **NIFTI** files consist of segmentation in the **sagittal plane**, while the **DICOM** files are in the **axial plane**. Please use the **NIFTI** header information to determine the appropriate orientation such that the **DICOM** images and segmentation match. Otherwise, you run the risk of having the segmentations flipped in the **Z axis** and mirrored in the **X axis**.\n\nRelevant and Important Resources:\n\n- [Determining MR Image Planes](https://www.kaggle.com/code/davidbroberts/determining-mr-image-planes)\n- [Connecting voxel spaces](https://www.kaggle.com/code/boojum/connecting-voxel-spaces)\n- [Normalized Voxels: Align Planes and Crop](https://www.kaggle.com/code/ren4yu/normalized-voxels-align-planes-and-crop)","metadata":{}},{"cell_type":"code","source":"dicom_files = glob(\n    \"../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.10062/*.dcm\"\n)\n\nlen(dicom_files)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:21.737221Z","iopub.execute_input":"2022-08-23T14:17:21.737668Z","iopub.status.idle":"2022-08-23T14:17:21.816035Z","shell.execute_reply.started":"2022-08-23T14:17:21.737634Z","shell.execute_reply":"2022-08-23T14:17:21.814804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read_dicom funciton would retrun 2 things.\n# In index 0: dicom.pixel data, index 1: raw read with meta info\nsample_dicom = dicom_files[0]\ntemp_read = read_dicom(sample_dicom)[1]\ntemp_read","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:24.466401Z","iopub.execute_input":"2022-08-23T14:17:24.466824Z","iopub.status.idle":"2022-08-23T14:17:24.492675Z","shell.execute_reply.started":"2022-08-23T14:17:24.466786Z","shell.execute_reply":"2022-08-23T14:17:24.491399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read and plot first 64 dicom files\ntemp = [read_dicom(i)[0][None, ...] for i in dicom_files[:64]]\ntemp = np.concatenate(temp, axis=0)\ntemp.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:27.454227Z","iopub.execute_input":"2022-08-23T14:17:27.455679Z","iopub.status.idle":"2022-08-23T14:17:28.522587Z","shell.execute_reply.started":"2022-08-23T14:17:27.455633Z","shell.execute_reply":"2022-08-23T14:17:28.521217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:29.726446Z","iopub.execute_input":"2022-08-23T14:17:29.726854Z","iopub.status.idle":"2022-08-23T14:17:35.127939Z","shell.execute_reply.started":"2022-08-23T14:17:29.72682Z","shell.execute_reply":"2022-08-23T14:17:35.126978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read and plot first 64 dicom files\ntemp = [read_dicom(i)[0][None, ...] for i in dicom_files[-64:]]\ntemp = np.concatenate(temp, axis=0)\ntemp.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:35.129718Z","iopub.execute_input":"2022-08-23T14:17:35.130619Z","iopub.status.idle":"2022-08-23T14:17:36.174609Z","shell.execute_reply.started":"2022-08-23T14:17:35.13057Z","shell.execute_reply":"2022-08-23T14:17:36.173402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp, cmap='bone')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:36.17629Z","iopub.execute_input":"2022-08-23T14:17:36.176791Z","iopub.status.idle":"2022-08-23T14:17:41.678894Z","shell.execute_reply.started":"2022-08-23T14:17:36.176741Z","shell.execute_reply":"2022-08-23T14:17:41.677855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [Bounding Box](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data?select=train_bounding_boxes.csv)\n\n**Bounding boxes for a subset of the training set.**","metadata":{}},{"cell_type":"code","source":"df_bbox = pd.read_csv(\n    '../input/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv'\n)\n# [train/test]_images/[StudyInstanceUID]/[slice_number].dcm\ndf_bbox.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:52.104573Z","iopub.execute_input":"2022-08-23T14:17:52.105028Z","iopub.status.idle":"2022-08-23T14:17:52.136883Z","shell.execute_reply.started":"2022-08-23T14:17:52.104989Z","shell.execute_reply":"2022-08-23T14:17:52.135757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bbox.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:53.848336Z","iopub.execute_input":"2022-08-23T14:17:53.848765Z","iopub.status.idle":"2022-08-23T14:17:53.865906Z","shell.execute_reply.started":"2022-08-23T14:17:53.84873Z","shell.execute_reply":"2022-08-23T14:17:53.864868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unique id with bbox labels\nlen(df_bbox.StudyInstanceUID), df_bbox.StudyInstanceUID.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:55.320527Z","iopub.execute_input":"2022-08-23T14:17:55.32169Z","iopub.status.idle":"2022-08-23T14:17:55.329147Z","shell.execute_reply.started":"2022-08-23T14:17:55.321646Z","shell.execute_reply":"2022-08-23T14:17:55.327859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_bbox.iloc[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:56.435322Z","iopub.execute_input":"2022-08-23T14:17:56.436011Z","iopub.status.idle":"2022-08-23T14:17:56.444738Z","shell.execute_reply.started":"2022-08-23T14:17:56.435974Z","shell.execute_reply":"2022-08-23T14:17:56.443548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = '../input/rsna-2022-cervical-spine-fracture-detection/train_images/'\nlen(os.listdir(train_images)) - df_bbox.StudyInstanceUID.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:17:58.316836Z","iopub.execute_input":"2022-08-23T14:17:58.317528Z","iopub.status.idle":"2022-08-23T14:17:58.440342Z","shell.execute_reply.started":"2022-08-23T14:17:58.317488Z","shell.execute_reply":"2022-08-23T14:17:58.438322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_512 = []\ntemp_768 = []\n\n#[train/test]_images/[StudyInstanceUID]/[slice_number].dcm\nfor index, row in tqdm(df_bbox.iterrows(), total=len(df_bbox)):\n    with_bbox_dicom_path = os.path.join(\n            train_images, str(row['StudyInstanceUID']), str(row['slice_number']) + '.dcm'\n        )\n    with_bbox_read_dicom, _ = read_dicom(with_bbox_dicom_path)\n    \n    xmin = int(row['x'])\n    ymin = int(row['y'])\n    xmax = xmin + int(row['width'])\n    ymax = ymin + int(row['height'])\n    cv2.rectangle(with_bbox_read_dicom, (xmin, ymin), (xmax, ymax), (0,255,255), 3)\n    \n    if with_bbox_read_dicom.shape[0] == 512:\n        temp_512.append(with_bbox_read_dicom[None, ...])\n    else:\n        temp_768.append(with_bbox_read_dicom[None, ...])\n    \ntemp_512 = np.concatenate(temp_512, axis=0)\ntemp_768 = np.concatenate(temp_768, axis=0)\ntemp_512.shape, temp_768.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:18:02.86034Z","iopub.execute_input":"2022-08-23T14:18:02.860742Z","iopub.status.idle":"2022-08-23T14:20:34.779342Z","shell.execute_reply.started":"2022-08-23T14:18:02.860708Z","shell.execute_reply":"2022-08-23T14:20:34.77776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp_512[:64], cmap='jet') # pick 64 slices","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:20:53.383842Z","iopub.execute_input":"2022-08-23T14:20:53.384313Z","iopub.status.idle":"2022-08-23T14:20:59.234201Z","shell.execute_reply.started":"2022-08-23T14:20:53.384261Z","shell.execute_reply":"2022-08-23T14:20:59.231083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp_512[-64:], cmap='inferno') # pick 64 slices","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:20:59.2368Z","iopub.execute_input":"2022-08-23T14:20:59.237511Z","iopub.status.idle":"2022-08-23T14:21:04.954851Z","shell.execute_reply.started":"2022-08-23T14:20:59.237469Z","shell.execute_reply":"2022-08-23T14:21:04.952511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp_512[64:128], cmap='bone') # pick 64 slices","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:21:04.956463Z","iopub.execute_input":"2022-08-23T14:21:04.957434Z","iopub.status.idle":"2022-08-23T14:21:10.618469Z","shell.execute_reply.started":"2022-08-23T14:21:04.957394Z","shell.execute_reply":"2022-08-23T14:21:10.61744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(\n    np.apply_along_axis(\n        cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8,8)).apply, 2, temp_512[128:192] # pick 64 slices\n    ), \n    cmap='gray'\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:21:10.620249Z","iopub.execute_input":"2022-08-23T14:21:10.621051Z","iopub.status.idle":"2022-08-23T14:21:18.325887Z","shell.execute_reply.started":"2022-08-23T14:21:10.621014Z","shell.execute_reply":"2022-08-23T14:21:18.323924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_plot(temp_768, cmap='bone')","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:22:29.505591Z","iopub.execute_input":"2022-08-23T14:22:29.506029Z","iopub.status.idle":"2022-08-23T14:22:32.189065Z","shell.execute_reply.started":"2022-08-23T14:22:29.505994Z","shell.execute_reply":"2022-08-23T14:22:32.187645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# [Submission](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/data?select=sample_submission.csv)","metadata":{}},{"cell_type":"code","source":"# ref. https://www.kaggle.com/code/paulorzp/rsna2022-cervical-baseline/\nmeans = train_df.mean(numeric_only=True).to_dict()\ntest_df['fractured'] = test_df['prediction_type'].map(means)\ntest_df[['row_id','fractured']].to_csv('submission.csv', index=False, float_format='%.1g')\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-23T14:21:31.484078Z","iopub.execute_input":"2022-08-23T14:21:31.484524Z","iopub.status.idle":"2022-08-23T14:21:31.511605Z","shell.execute_reply.started":"2022-08-23T14:21:31.484486Z","shell.execute_reply":"2022-08-23T14:21:31.510672Z"},"trusted":true},"execution_count":null,"outputs":[]}]}