{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Adapted from : https://www.kaggle.com/code/theoviel/dicom-resized-png-jpg\n\n**Dataset Links :**\n- Part 1 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-png-pt1\n- Part 2 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-png-pt2\n- Part 3 : https://www.kaggle.com/datasets/theoviel/rsna-2023-abdominal-trauma-detection-pngs-3-8\n- Part 4 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-png-pt4\n- Part 5 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-png-pt5\n- Part 6 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-png-pt6\n- Part 7 : https://www.kaggle.com/datasets/theoviel/rsna-abdominal-trauma-detection-pngs-pt7\n- Part 8 : https://www.kaggle.com/datasets/theoviel/rsna-2023-abdominal-trauma-detection-pngs-18\n\n**Changes :**\n- Apply `standardize_pixel_array` function\n- Update links\n- Remove `apply_voi_luit`\n- Add rescaling, thanks @sukharev !\n\n**TODO :**\n- Dicom processing on GPU\n- Figure out why example dicom is too dark","metadata":{}},{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:38:58.454535Z","iopub.execute_input":"2023-10-03T13:38:58.454891Z","iopub.status.idle":"2023-10-03T13:39:13.345642Z","shell.execute_reply.started":"2023-10-03T13:38:58.454865Z","shell.execute_reply":"2023-10-03T13:39:13.343984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nfrom glob import glob\nimport gdcm\nimport pydicom\nimport zipfile\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-03T13:39:13.347995Z","iopub.execute_input":"2023-10-03T13:39:13.348463Z","iopub.status.idle":"2023-10-03T13:39:14.763711Z","shell.execute_reply.started":"2023-10-03T13:39:13.348422Z","shell.execute_reply":"2023-10-03T13:39:14.762528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## From https://www.kaggle.com/code/haqishen/rsna-2022-1st-place-solution-inference/input\n# data_dir = '../input/rsna-2022-cervical-spine-fracture-detection/'\nimage_size_seg = (128, 128, 128)\n# msk_size = image_size_seg[0]\n# image_size_cls = 224\n# n_slice_per_c = 15\n# n_ch = 5\n\n# batch_size_seg = 1\n# num_workers = 2","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:39:14.765238Z","iopub.execute_input":"2023-10-03T13:39:14.765625Z","iopub.status.idle":"2023-10-03T13:39:14.771072Z","shell.execute_reply.started":"2023-10-03T13:39:14.765589Z","shell.execute_reply":"2023-10-03T13:39:14.77017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def standardize_pixel_array(dcm: pydicom.dataset.FileDataset) -> np.ndarray:\n    \"\"\"\n    Source : https://www.kaggle.com/competitions/rsna-2023-abdominal-trauma-detection/discussion/427217\n    \"\"\"\n    # Correct DICOM pixel_array if PixelRepresentation == 1.\n    pixel_array = dcm.pixel_array\n    if dcm.PixelRepresentation == 1:\n        bit_shift = dcm.BitsAllocated - dcm.BitsStored\n        dtype = pixel_array.dtype \n        pixel_array = (pixel_array << bit_shift).astype(dtype) >>  bit_shift\n#         pixel_array = pydicom.pixel_data_handlers.util.apply_modality_lut(new_array, dcm)\n\n    intercept = float(dcm.RescaleIntercept)\n    slope = float(dcm.RescaleSlope)\n    center = int(dcm.WindowCenter)\n    width = int(dcm.WindowWidth)\n    low = center - width / 2\n    high = center + width / 2    \n    \n    pixel_array = (pixel_array * slope) + intercept\n    pixel_array = np.clip(pixel_array, low, high)\n\n    return pixel_array","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:39:14.773618Z","iopub.execute_input":"2023-10-03T13:39:14.774306Z","iopub.status.idle":"2023-10-03T13:39:14.78925Z","shell.execute_reply.started":"2023-10-03T13:39:14.77427Z","shell.execute_reply":"2023-10-03T13:39:14.788192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_PATH = \"/kaggle/input/rsna-2023-abdominal-trauma-detection/train_images/\"\n\nprint('Number of training patients :', len(os.listdir(TRAIN_PATH)))","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:39:14.79118Z","iopub.execute_input":"2023-10-03T13:39:14.791908Z","iopub.status.idle":"2023-10-03T13:39:14.961422Z","shell.execute_reply.started":"2023-10-03T13:39:14.791865Z","shell.execute_reply":"2023-10-03T13:39:14.960163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\ndef study_path_to_3D_image(path, image_size_seg=(128, 128, 128), plot_image=False, z_df=None):\n    t_paths = sorted(glob(os.path.join(path, \"*\")), key=lambda x: int(x.split('/')[-1].split(\".\")[0]))\n    n_scans = len(t_paths)\n#     print(n_scans)\n    indices = np.quantile(list(range(n_scans)), np.linspace(0., 1., image_size_seg[2])).round().astype(int)\n    print(len(indices))\n    t_paths = [t_paths[i] for i in indices]\n\n    imgs = {}\n    pos_zs = []\n    for filename in t_paths:\n        dicom = pydicom.dcmread(filename)\n        pos_z = dicom[(0x20, 0x32)].value[-1]  # to retrieve the order of frames\n#         dicom_numbers.append(filename.split('/')[-1].split(\".\")[0])\n        pos_zs.append(pos_z)\n        img = standardize_pixel_array(dicom)\n        img = cv2.resize(img, (image_size_seg[0], image_size_seg[1]), interpolation = cv2.INTER_AREA)\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n            img = 1 - img\n        imgs[pos_z] = img\n        \n#     print(dicom_numbers, pos_zs)\n    if z_df is not None: \n        study = path.split('/')[-1]\n        z_df.loc[study, 'pos_z_ascending'] = pos_zs[-1] > pos_zs[0]\n        \n    images = []\n    cnt = Counter(pos_zs)\n    for i, k in enumerate(sorted(imgs.keys())):\n        img = imgs[k]\n#         images.append(img)\n        images.extend([img] * cnt[k]) # to make sure we have same dimensions\n        if not (i % 100) and plot_image:\n            plt.figure(figsize=(5, 5))\n            plt.imshow(img, cmap=\"gray\")\n            plt.title(f\"Patient {patient} - Study {study} - Frame {i}/{len(imgs)}\")\n            plt.axis(False)\n            plt.show()\n    images = np.stack(images, -1)\n    \n    images = images - np.min(images)\n    images = images / (np.max(images) + 1e-4)\n    images = (images * 255).astype(np.uint8)\n    \n    if plot_image: \n        print('plotting image normalized accross all images')\n        for i in range(images.shape[-1]): \n            if i % 100: continue\n            img = images[:, :, i]\n            plt.figure(figsize=(5, 5))\n            plt.imshow(img, cmap=\"gray\")\n            plt.title(f\"Patient {patient} - Study {study} - Frame {i}/{len(imgs)}\")\n            plt.axis(False)\n            plt.show()\n\n    return images","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:43:18.945336Z","iopub.execute_input":"2023-10-03T13:43:18.945686Z","iopub.status.idle":"2023-10-03T13:43:18.960295Z","shell.execute_reply.started":"2023-10-03T13:43:18.94566Z","shell.execute_reply":"2023-10-03T13:43:18.959323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"study_path_103 = '/kaggle/input/rsna-2023-abdominal-trauma-detection/train_images/10005/18667'\nimages = study_path_to_3D_image(study_path_103, image_size_seg, plot_image=False)\nimages.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:43:19.293832Z","iopub.execute_input":"2023-10-03T13:43:19.294723Z","iopub.status.idle":"2023-10-03T13:43:21.247765Z","shell.execute_reply.started":"2023-10-03T13:43:19.294688Z","shell.execute_reply":"2023-10-03T13:43:21.246641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z_df = pd.DataFrame()\nfor patient in sorted(os.listdir(TRAIN_PATH)):\n    for study in os.listdir(TRAIN_PATH + patient):\n        study_path = os.path.join(TRAIN_PATH + patient, study)\n        images = study_path_to_3D_image(study_path, image_size_seg, plot_image=True, z_df=z_df)\n    break\ndisplay(z_df)","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:44:20.528642Z","iopub.execute_input":"2023-10-03T13:44:20.529004Z","iopub.status.idle":"2023-10-03T13:44:26.100351Z","shell.execute_reply.started":"2023-10-03T13:44:20.528978Z","shell.execute_reply":"2023-10-03T13:44:26.099614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Save the processed data","metadata":{}},{"cell_type":"code","source":"ram_size_bytes = images.nbytes\n\n# Convert bytes to a more human-readable format (e.g., megabytes)\nram_size_megabytes = ram_size_bytes / (1024 * 1024)\n\nprint(f\"RAM size of the NumPy array: {ram_size_megabytes:.2f} megabytes\")\nprint(f\"approx RAM size of all NumPy array: {ram_size_megabytes * 4700:.2f} megabytes\")","metadata":{"execution":{"iopub.status.busy":"2023-10-02T13:55:46.600386Z","iopub.execute_input":"2023-10-02T13:55:46.6013Z","iopub.status.idle":"2023-10-02T13:55:46.610517Z","shell.execute_reply.started":"2023-10-02T13:55:46.601249Z","shell.execute_reply":"2023-10-02T13:55:46.608893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:45:44.976147Z","iopub.execute_input":"2023-10-03T13:45:44.976557Z","iopub.status.idle":"2023-10-03T13:45:44.981991Z","shell.execute_reply.started":"2023-10-03T13:45:44.976526Z","shell.execute_reply":"2023-10-03T13:45:44.980932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(patient, image_size_seg, save_folder=\"\", data_path=\"\", z_df=None):\n    for study in sorted(os.listdir(data_path + patient)):\n        study_path = os.path.join(data_path + patient, study)\n        images = study_path_to_3D_image(study_path, image_size_seg, z_df=z_df)\n        np.save(save_folder + f\"{patient}_{study}.npy\", images)\n        \n        \n#         if isinstance(save_folder, str):\n#             cv2.imwrite(save_folder + f\"{patient}_{study}_{i}.png\", (img * 255).astype(np.uint8))\n#         else:\n#             im = cv2.imencode('.png', (img * 255).astype(np.uint8))[1]\n#             save_folder.writestr(f'{patient}_{study}_{i:04d}.png', im)","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:45:47.520154Z","iopub.execute_input":"2023-10-03T13:45:47.521276Z","iopub.status.idle":"2023-10-03T13:45:47.527662Z","shell.execute_reply.started":"2023-10-03T13:45:47.521236Z","shell.execute_reply":"2023-10-03T13:45:47.526378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patients = os.listdir(TRAIN_PATH)\nos.makedirs('train_images')\nz_df = pd.DataFrame()\nfor patient in tqdm(patients):\n    process(patient, image_size_seg, save_folder='train_images', data_path=TRAIN_PATH, z_df=z_df)\nz_df.to_csv('z_df.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:45:53.309794Z","iopub.execute_input":"2023-10-03T13:45:53.310167Z","iopub.status.idle":"2023-10-03T13:46:02.607592Z","shell.execute_reply.started":"2023-10-03T13:45:53.310136Z","shell.execute_reply":"2023-10-03T13:46:02.60676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z_df.pos_z_ascending.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-03T13:47:03.796328Z","iopub.execute_input":"2023-10-03T13:47:03.796673Z","iopub.status.idle":"2023-10-03T13:47:03.809751Z","shell.execute_reply.started":"2023-10-03T13:47:03.796647Z","shell.execute_reply":"2023-10-03T13:47:03.808738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# patients = os.listdir(TRAIN_PATH)\n\n# # Chunking\n# patients = (\n# #     patients[:400],\n# #     patients[400:  800],\n# #     patients[800:  1200],\n# #     patients[1200: 1600],\n# #     patients[1600: 2000],\n# #     patients[2000: 2400],\n# #     patients[2400: 2800],\n#     patients[2800:],\n# )\n\n# patients = patients[0]  # [:10]  # subsample","metadata":{"execution":{"iopub.status.busy":"2023-07-27T13:24:43.851168Z","iopub.execute_input":"2023-07-27T13:24:43.851696Z","iopub.status.idle":"2023-07-27T13:24:43.861Z","shell.execute_reply.started":"2023-07-27T13:24:43.851656Z","shell.execute_reply":"2023-07-27T13:24:43.858729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with zipfile.ZipFile(\"output.zip\", 'w') as save_folder:\n#     for patient in tqdm(patients):\n#         process(patient, size=None, save_folder=save_folder, data_path=TRAIN_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-07-27T13:28:33.225763Z","iopub.execute_input":"2023-07-27T13:28:33.227353Z","iopub.status.idle":"2023-07-27T13:29:29.324828Z","shell.execute_reply.started":"2023-07-27T13:28:33.227263Z","shell.execute_reply":"2023-07-27T13:29:29.323243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Done ! ","metadata":{}}]}