{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA: Meta data","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nPATH_DATASET = \"/kaggle/input/rsna-2022-cervical-spine-fracture-detection\"","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:02.826037Z","iopub.execute_input":"2022-08-30T13:00:02.827158Z","iopub.status.idle":"2022-08-30T13:00:02.859526Z","shell.execute_reply.started":"2022-08-30T13:00:02.827052Z","shell.execute_reply":"2022-08-30T13:00:02.857859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(PATH_DATASET, \"train.csv\"))\ndisplay(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:02.863898Z","iopub.execute_input":"2022-08-30T13:00:02.864305Z","iopub.status.idle":"2022-08-30T13:00:02.901336Z","shell.execute_reply.started":"2022-08-30T13:00:02.864273Z","shell.execute_reply":"2022-08-30T13:00:02.900079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show the ration of positive/negative casses","metadata":{}},{"cell_type":"code","source":"ax = df_train[\"patient_overall\"].value_counts().plot.pie(ylabel=\"patient_overall\", autopct=\"%.1f%%\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:02.902783Z","iopub.execute_input":"2022-08-30T13:00:02.903901Z","iopub.status.idle":"2022-08-30T13:00:03.094962Z","shell.execute_reply.started":"2022-08-30T13:00:02.903854Z","shell.execute_reply":"2022-08-30T13:00:03.093087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution over spines\n\n![](https://upload.wikimedia.org/wikipedia/commons/5/54/Gray_111_-_Vertebral_column-coloured.png)","metadata":{}},{"cell_type":"code","source":"cx = [f\"C{i + 1}\" for i in range(7)]\nax = df_train[cx].sum().plot.bar(xlabel=\"spine\", ylabel=\"counts\", grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:03.099309Z","iopub.execute_input":"2022-08-30T13:00:03.100402Z","iopub.status.idle":"2022-08-30T13:00:03.348368Z","shell.execute_reply.started":"2022-08-30T13:00:03.100339Z","shell.execute_reply":"2022-08-30T13:00:03.347531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### How many times defect o multiple spines","metadata":{}},{"cell_type":"code","source":"ax = df_train[cx].sum(axis=1).value_counts().plot.bar(xlabel=\"occurances\", ylabel=\"counts\", grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:03.349587Z","iopub.execute_input":"2022-08-30T13:00:03.350098Z","iopub.status.idle":"2022-08-30T13:00:03.554191Z","shell.execute_reply.started":"2022-08-30T13:00:03.350064Z","shell.execute_reply":"2022-08-30T13:00:03.55336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multi-defect correlation","metadata":{}},{"cell_type":"code","source":"import seaborn as sn\ncorr = df_train[cx].corr()\nax = sn.heatmap(corr, cmap=\"Blues\", annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:03.555397Z","iopub.execute_input":"2022-08-30T13:00:03.555873Z","iopub.status.idle":"2022-08-30T13:00:05.408254Z","shell.execute_reply.started":"2022-08-30T13:00:03.555842Z","shell.execute_reply":"2022-08-30T13:00:05.406066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bounding box annotations","metadata":{}},{"cell_type":"code","source":"df_annot = pd.read_csv(os.path.join(PATH_DATASET, \"train_bounding_boxes.csv\"))\ndisplay(df_annot.head())","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:05.409628Z","iopub.execute_input":"2022-08-30T13:00:05.409963Z","iopub.status.idle":"2022-08-30T13:00:05.450085Z","shell.execute_reply.started":"2022-08-30T13:00:05.409934Z","shell.execute_reply":"2022-08-30T13:00:05.448946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\nmissing_annot = []\nspines = {f\"C{i + 1}\": [] for i in range(7)}\nfor _, row in tqdm(df_train.iterrows()):\n    study_id = row[\"StudyInstanceUID\"]\n    ls_imgs = glob.glob(os.path.join(PATH_DATASET, \"train_images\", study_id, \"*.dcm\"))\n    annot_slices = df_annot[df_annot[\"StudyInstanceUID\"] == study_id][\"slice_number\"].values.tolist()\n    if not annot_slices:\n        if row[\"patient_overall\"] == 1:\n            missing_annot.append(study_id)\n            # print(f\"missing annotation for: {study_id}\")\n        continue\n    nb = float(len(ls_imgs))\n    idx = [i / nb for i in annot_slices]\n    for c in spines:\n        if row[c] == 1:\n            spines[c] += idx","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:00:05.451513Z","iopub.execute_input":"2022-08-30T13:00:05.45203Z","iopub.status.idle":"2022-08-30T13:02:09.329046Z","shell.execute_reply.started":"2022-08-30T13:00:05.451974Z","shell.execute_reply":"2022-08-30T13:02:09.327776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Check how many bbox annot.","metadata":{}},{"cell_type":"code","source":"fractured = df_train[\"patient_overall\"].sum()\nax = pd.Series(\n    {\"missing\": len(missing_annot), \"annotated\": fractured - len(missing_annot)}\n).plot.pie(ylabel=\"\", autopct=\"%.1f%%\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:08:13.077588Z","iopub.execute_input":"2022-08-30T13:08:13.078172Z","iopub.status.idle":"2022-08-30T13:08:13.16849Z","shell.execute_reply.started":"2022-08-30T13:08:13.07812Z","shell.execute_reply":"2022-08-30T13:08:13.167509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cx = [f\"C{i + 1}\" for i in range(7)]\ndf_ = pd.DataFrame({\n    \"missing\": df_train[df_train[\"StudyInstanceUID\"].isin(missing_annot)][cx].sum(),\n    \"annotated\": df_train[~df_train[\"StudyInstanceUID\"].isin(missing_annot)][cx].sum()\n})\n_= df_.plot.bar(xlabel=\"spine\", ylabel=\"counts\", grid=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T13:16:42.404745Z","iopub.execute_input":"2022-08-30T13:16:42.405182Z","iopub.status.idle":"2022-08-30T13:16:42.650456Z","shell.execute_reply.started":"2022-08-30T13:16:42.40514Z","shell.execute_reply":"2022-08-30T13:16:42.649378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Annotations per fractured spine\n\nNote that it not exclusiove but include all cooccurances as annotation doe not tell detail about the spine nb","metadata":{}},{"cell_type":"code","source":"fig, axarr = plt.subplots(nrows=len(spines), figsize=(8, len(spines) * 2))\nfor i, c in enumerate(spines):\n    axarr[i].hist(spines[c], bins=50, range=(0., 1.))\n    axarr[i].grid(True)\n    axarr[i].set_title(c)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T15:15:43.469651Z","iopub.execute_input":"2022-08-01T15:15:43.470546Z","iopub.status.idle":"2022-08-01T15:15:44.793039Z","shell.execute_reply.started":"2022-08-01T15:15:43.4705Z","shell.execute_reply":"2022-08-01T15:15:44.791957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA: DICOM images","metadata":{}},{"cell_type":"code","source":"!pip download -q \"python-gdcm\" pydicom pylibjpeg \"opencv-python-headless\" --dest frozen_packages --prefer-binary\n!ls -l frozen_packages","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-01T14:00:05.226813Z","iopub.execute_input":"2022-08-01T14:00:05.227532Z","iopub.status.idle":"2022-08-01T14:00:14.48573Z","shell.execute_reply.started":"2022-08-01T14:00:05.227486Z","shell.execute_reply":"2022-08-01T14:00:14.484586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -qU \"python-gdcm\" pydicom pylibjpeg \"opencv-python-headless\" --find-links frozen_packages --no-index","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-08-01T14:00:14.487055Z","iopub.execute_input":"2022-08-01T14:00:14.487428Z","iopub.status.idle":"2022-08-01T14:00:28.582939Z","shell.execute_reply.started":"2022-08-01T14:00:14.487394Z","shell.execute_reply":"2022-08-01T14:00:28.581769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading DICOM image\n\nShow the DICOM image content with all metadata","metadata":{}},{"cell_type":"code","source":"import pydicom\nfrom pydicom.pixel_data_handlers import apply_voi_lut\n\nspl = df_annot.sample(1).iloc[0]\nprint(spl)\ndicom_path = os.path.join(PATH_DATASET, \"train_images\", spl[\"StudyInstanceUID\"], f'{spl[\"slice_number\"]}.dcm')\ndicom = pydicom.dcmread(dicom_path)\n# print(vars(dicom).keys())\nprint(dicom)\nimg = apply_voi_lut(dicom.pixel_array, dicom)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T14:00:28.585274Z","iopub.execute_input":"2022-08-01T14:00:28.586051Z","iopub.status.idle":"2022-08-01T14:00:28.839713Z","shell.execute_reply.started":"2022-08-01T14:00:28.585997Z","shell.execute_reply":"2022-08-01T14:00:28.838545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Show sample with Crack","metadata":{}},{"cell_type":"code","source":"from matplotlib.patches import Rectangle\n\nfig, ax = plt.subplots()\nax_im = ax.imshow(img, cmap=\"gray\")\nprint(f\"min: {img.min()} & max: {img.max()}\")\nprint(f\"size: {img.shape}\")\nrect = Rectangle((spl['x'], spl['y']), spl['width'], spl['height'], linewidth=1, edgecolor='r', facecolor='none')\n# Add the patch to the Axes\nax.add_patch(rect)\n\n_= plt.colorbar(ax_im)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T14:00:28.841274Z","iopub.execute_input":"2022-08-01T14:00:28.841818Z","iopub.status.idle":"2022-08-01T14:00:29.144363Z","shell.execute_reply.started":"2022-08-01T14:00:28.841785Z","shell.execute_reply":"2022-08-01T14:00:29.143209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load 3D volume","metadata":{}},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\ndef load_image_volume(img_dir):\n    img_paths = glob.glob(os.path.join(img_dir, f\"*.dcm\"))\n    img_paths = sorted(img_paths, key=lambda p: int(os.path.splitext(os.path.basename(p))[0]))\n    print(len(img_paths))\n    imgs = []\n    for p_img in tqdm(img_paths):\n        dicom = pydicom.dcmread(p_img)\n        imgs.append(apply_voi_lut(dicom.pixel_array, dicom).tolist())\n    vol = np.array(imgs)\n    return vol\n\nstudy_dir = os.path.dirname(dicom_path)\nvol = load_image_volume(study_dir)\nprint(vol.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T14:00:29.147751Z","iopub.execute_input":"2022-08-01T14:00:29.148116Z","iopub.status.idle":"2022-08-01T14:00:42.834631Z","shell.execute_reply.started":"2022-08-01T14:00:29.148083Z","shell.execute_reply":"2022-08-01T14:00:42.833461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ipywidgets import interact, IntSlider\nfrom matplotlib.patches import PathPatch\nfrom matplotlib.path import Path\n\n\ndef _draw_line(ax, coords, clr='g'):\n    line = Path(coords, [Path.MOVETO, Path.LINETO])\n    pp = PathPatch(line, linewidth=3, edgecolor=clr, facecolor='none')\n    ax.add_patch(pp)\n\ndef _set_axes_labels(ax, axes_x, axes_y):\n    ax.set_xlabel(axes_x)\n    ax.set_ylabel(axes_y)\n    ax.set_aspect('equal', 'box')\n\n_rec_prop = dict(linewidth=5, facecolor='none')\n\ndef show_volume(vol, annot, z, y, x, fig_size=(9, 9)):\n    fig, axarr = plt.subplots(nrows=2, ncols=2, figsize=fig_size)\n    v_z, v_y, v_x = vol.shape\n    axarr[0, 0].imshow(vol[z, :, :], cmap=\"gray\")\n    annot  = annot[annot[\"slice_number\"] == z]\n    if len(annot) == 1:\n        bbox = annot.iloc[0]\n        xy, w, h = (bbox['x'], bbox['y']), bbox['width'], bbox['height']\n        axarr[0, 0].add_patch(Rectangle(xy, w, h, linewidth=2, edgecolor='m', facecolor='none'))\n    axarr[0, 0].add_patch(Rectangle((-1, -1), v_x, v_y, edgecolor='r', **_rec_prop))\n    _draw_line(axarr[0, 0], [(x, 0), (x, v_y)], \"g\")\n    _draw_line(axarr[0, 0], [(0, y), (v_x, y)], \"b\")\n    _set_axes_labels(axarr[0, 0], \"X\", \"Y\")\n    axarr[0, 1].imshow(vol[:, :, x].T, cmap=\"gray\")\n    axarr[0, 1].add_patch(Rectangle((-1, -1), v_z, v_y, edgecolor='g', **_rec_prop))\n    _draw_line(axarr[0, 1], [(z, 0), (z, v_y)], \"r\")\n    _draw_line(axarr[0, 1], [(0, y), (v_x, y)], \"b\")\n    _set_axes_labels(axarr[0, 1], \"Z\", \"Y\")\n    axarr[1, 0].imshow(vol[:, y, :], cmap=\"gray\")\n    axarr[1, 0].add_patch(Rectangle((-1, -1), v_x, v_z, edgecolor='b', **_rec_prop))\n    _draw_line(axarr[1, 0], [(0, z), (v_x, z)], \"r\")\n    _draw_line(axarr[1, 0], [(x, 0), (x, v_y)], \"g\")\n    _set_axes_labels(axarr[1, 0], \"X\", \"Z\")\n    axarr[1, 1].set_axis_off()\n    fig.tight_layout()\n\n\ndef interactive_show(volume, annot):\n    vol_shape = volume.shape\n    init_z = annot.sample(1).iloc[0][\"slice_number\"] if len(annot) > 0 else None\n    interact(\n        lambda x, y, z: plt.show(show_volume(volume, annot, z, y, x)),\n        z=IntSlider(min=0, max=vol_shape[0], step=2, value=init_z or int(vol_shape[0] / 2)),\n        y=IntSlider(min=0, max=vol_shape[1], step=5, value=int(vol_shape[1] / 2)),\n        x=IntSlider(min=0, max=vol_shape[2], step=5, value=int(vol_shape[2] / 2)),\n    )","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-01T14:00:42.836095Z","iopub.execute_input":"2022-08-01T14:00:42.836461Z","iopub.status.idle":"2022-08-01T14:00:42.859414Z","shell.execute_reply.started":"2022-08-01T14:00:42.836428Z","shell.execute_reply":"2022-08-01T14:00:42.858447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annot = df_annot[df_annot[\"StudyInstanceUID\"] == os.path.basename(study_dir)]\n\ninteractive_show(vol, annot)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T14:00:42.860753Z","iopub.execute_input":"2022-08-01T14:00:42.861162Z","iopub.status.idle":"2022-08-01T14:00:43.516684Z","shell.execute_reply.started":"2022-08-01T14:00:42.861129Z","shell.execute_reply":"2022-08-01T14:00:43.51558Z"},"trusted":true},"execution_count":null,"outputs":[]}]}