{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is not my dicovery. @**kaggleqrdl** found it and discussed here: https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369684#2057282\n\nI created quick implementation how to speed up dicom processing significantly.\n\nGPU / 500 images (Parallel - 2 jobs):\n- pydicom -> 396.74 sec\n- dicomsdl -> 243.39 sec \n","metadata":{}},{"cell_type":"code","source":"!/usr/local/bin/python -m pip install --upgrade pip\n!pip install dicomsdl\n!pip install pylibjpeg\n!pip install python_gdcm\n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-11T04:21:34.549509Z","iopub.execute_input":"2023-01-11T04:21:34.549908Z","iopub.status.idle":"2023-01-11T04:21:50.211197Z","shell.execute_reply.started":"2023-01-11T04:21:34.549875Z","shell.execute_reply":"2023-01-11T04:21:50.210342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install opencv-python","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:22:50.717206Z","iopub.execute_input":"2023-01-11T04:22:50.717661Z","iopub.status.idle":"2023-01-11T04:22:57.179855Z","shell.execute_reply.started":"2023-01-11T04:22:50.717623Z","shell.execute_reply":"2023-01-11T04:22:57.178571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nimport cv2\nimport pydicom\n\nfrom joblib import Parallel, delayed\nimport glob\nimport time\nimport numpy as np\nimport os\nfrom matplotlib import pyplot as plt","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-01-11T04:22:58.905385Z","iopub.execute_input":"2023-01-11T04:22:58.906746Z","iopub.status.idle":"2023-01-11T04:23:00.01356Z","shell.execute_reply.started":"2023-01-11T04:22:58.906701Z","shell.execute_reply":"2023-01-11T04:23:00.01224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_dir_pydicom = '/kaggle/working/png_file_py/'\nimage_dir_dicomsdl = '/kaggle/working/png_file_dic/'\n\nos.makedirs(image_dir_pydicom, exist_ok=True)\nos.makedirs(image_dir_dicomsdl, exist_ok=True)\n\nIMAGES_TO_PROCESS = 500","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.125037Z","iopub.status.idle":"2023-01-11T04:19:04.125397Z","shell.execute_reply.started":"2023-01-11T04:19:04.125214Z","shell.execute_reply":"2023-01-11T04:19:04.125231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm\")\nlen(train_images)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.127152Z","iopub.status.idle":"2023-01-11T04:19:04.127505Z","shell.execute_reply.started":"2023-01-11T04:19:04.127323Z","shell.execute_reply":"2023-01-11T04:19:04.127339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512, save_folder=None, dicom_process = True, extension=\"png\"):\n    \n    patient = f.split('/')[-2]\n    image_name = f.split('/')[-1][:-4]\n    if dicom_process:\n        dicom = pydicom.dcmread(f)\n        img = dicom.pixel_array\n\n        img = (img - img.min()) / (img.max() - img.min())\n\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":  \n            img = 1 - img\n            \n        image = (img * 255).astype(np.uint8)\n    else:\n        \n        dicom = dicoml.open(f)\n        img = dicom.pixelData()\n\n        img = (img - img.min()) / (img.max() - img.min())\n\n        if dicom.getPixelDataInfo()['PhotometricInterpretation'] == \"MONOCHROME1\":\n            img = 1 - img\n\n        image = (img * 255).astype(np.uint8)\n    \n    img = cv2.resize(image, (size, size))\n\n    file_name = f'{save_folder}' + f\"{patient}_{image_name}.{extension}\"\n\n    cv2.imwrite(file_name, img)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.128542Z","iopub.status.idle":"2023-01-11T04:19:04.128938Z","shell.execute_reply.started":"2023-01-11T04:19:04.128734Z","shell.execute_reply":"2023-01-11T04:19:04.128752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### pydicom - speed test","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\n        \nParallel(n_jobs=96)(\n    delayed(process)(f, size = 512, save_folder = image_dir_pydicom, dicom_process = True)\n    for f in train_images[:IMAGES_TO_PROCESS]\n)\n\nprint(time.time() - start_time)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.130449Z","iopub.status.idle":"2023-01-11T04:19:04.130842Z","shell.execute_reply.started":"2023-01-11T04:19:04.130632Z","shell.execute_reply":"2023-01-11T04:19:04.130649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_files = glob.glob(f'{image_dir_pydicom}*.png')\nfor idx, i in enumerate(out_files[:4]):\n    im = cv2.imread(i)\n    print(f'file {i} size in bytes: {os.path.getsize(i)}, shape: {im.shape}')\n    plt.imshow(im)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.13255Z","iopub.status.idle":"2023-01-11T04:19:04.132895Z","shell.execute_reply.started":"2023-01-11T04:19:04.132711Z","shell.execute_reply":"2023-01-11T04:19:04.132726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### dicomsdl - speed test","metadata":{}},{"cell_type":"markdown","source":"We're going to decode every on of the 54K images, because why not?","metadata":{}},{"cell_type":"code","source":"import tqdm\nstart_time = time.time()\n        \nParallel(n_jobs=96)(\n    delayed(process)(f, size = 512, save_folder = image_dir_dicomsdl, dicom_process = False)\n    for f in tqdm.tqdm(train_images)\n)\n\nprint(time.time() - start_time)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.134187Z","iopub.status.idle":"2023-01-11T04:19:04.134537Z","shell.execute_reply.started":"2023-01-11T04:19:04.134357Z","shell.execute_reply":"2023-01-11T04:19:04.134373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Under 13 minutes to decode 54K dicom images.  Yimminy!","metadata":{}},{"cell_type":"code","source":"out_files = glob.glob(f'{image_dir_dicomsdl}*.png')\nfor i in out_files[6000:6004]:\n    im = cv2.imread(i)\n    print(f'file {i} size in bytes: {os.path.getsize(i)}, shape: {im.shape}')\n    plt.imshow(im)\n    plt.show() ","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:19:04.13546Z","iopub.status.idle":"2023-01-11T04:19:04.135787Z","shell.execute_reply.started":"2023-01-11T04:19:04.135612Z","shell.execute_reply":"2023-01-11T04:19:04.135627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see there is improvement in speed but further investigation in image quality is needed.","metadata":{}}]}