{"cells":[{"metadata":{},"cell_type":"markdown","source":"> ## DICOM to JPEG converter\n\nThis script allows the conversion of the DICOM data files to JPEG format. \n\nPersonally I prefer to handle images, because you can benefit from the image compression formats and from the ease of visualization."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pydicom as dicom\nimport glob\nimport os\nfrom tqdm import tqdm\nfrom PIL import Image\n\ntraindir = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/stage_1_train_images\"\ntestdir = \"/kaggle/input/rsna-intracranial-hemorrhage-detection/stage_1_test_images\"\n\ntrain_dicom_files = glob.glob(f\"{traindir}/*.dcm\")\nprint(f\"Number of train files: {len(train_dicom_files)}\")\n\ntest_dicom_files = glob.glob(f\"{testdir}/*.dcm\")\nprint(f\"Number of test files: {len(test_dicom_files)}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Helper function adapted from: \n# https://www.raddq.com/dicom-processing-segmentation-visualization-in-python/\n\ndef get_pixels_hu(scan):\n    image = np.stack(scan.pixel_array)\n    \n    # Convert to int16 (from sometimes int16), \n    # should be possible as values should always be low enough (<32k)\n    image = image.astype(np.int16)\n\n    # Set outside-of-scan pixels to 1\n    # The intercept is usually -1024, so air is approximately 0\n    image[image == -2000] = 0\n    \n    # Convert to Hounsfield units (HU)\n    intercept = scan.RescaleIntercept\n    slope = scan.RescaleSlope\n    \n    if slope != 1:\n        image = slope * image.astype(np.float64)\n        image = image.astype(np.int16)\n        \n    image += np.int16(intercept)\n    \n    return np.array(image, dtype=np.int16)\n\ndef hu2pil(img):\n    # as we want to detect hemorrages (HU between 0 to 100), we can normalizeas 0\n    # -1000: air\n    # -500: lung\n    # (-100, -50): Fat\n    # 0: Water\n    # (+30, +70): Blood\n    # (+10, +40): Muscle\n    # (+40, +60): Liver\n    # (+600, +3000): Bone\n    \n    # remove values greater than 3000 (out of scale)\n    img = np.less_equal(img, 3000)*img\n    # remove values greater lower than -1000 (out of scale)\n    img = np.greater_equal(img, -1000)*img\n    \n    # convert to positive values (0 to 4000 range)\n    img = img + 1000\n    \n    img = img/4000 # (0 to 1 scaling)\n    \n    img = Image.fromarray(np.uint8(img*255.0))\n    \n    return img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"outputdir = \"./output\"\nos.system(f\"mkdir {outputdir} {outputdir}/train {outputdir}/test\")\n\nfor file in tqdm(train_dicom_files):\n    id = os.path.splitext(os.path.basename(file))[0]\n    patient = dicom.read_file(file)\n    img = get_pixels_hu(patient)\n    img = hu2pil(img)\n    img.save(f\"{outputdir}/train/{id}.jpg\")\n    break # REMOVE WHEN USING IT (IN THIS SCRIPT THE LOOP IS DISABLED)\n    \nfor file in tqdm(test_dicom_files):\n    id = os.path.splitext(os.path.basename(file))[0]\n    patient = dicom.read_file(file)\n    img = get_pixels_hu(patient)\n    img = hu2pil(img)\n    img.save(f\"{outputdir}/test/{id}.jpg\")\n    break # REMOVE WHEN USING IT (IN THIS SCRIPT THE LOOP IS DISABLED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Detecting duplicate files"},{"metadata":{"trusted":true},"cell_type":"code","source":"import hashlib, os\n\nunique = dict()\nduplicate = []\nfor filename in glob.glob(f\"{outputdir}/*/*.jpg\"):\n    if os.path.isfile(filename):\n        filehash = hashlib.md5(open(filename, 'rb').read()).hexdigest()\n\n        if filehash not in unique: \n            unique[filehash] = filename\n        else:\n            print (f\"{filename} is a duplicate of {unique[filehash]}\")\n            duplicate.append(filename)\nprint(f\"Number of duplicates: {len(duplicate)}\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing duplicate files"},{"metadata":{"trusted":true},"cell_type":"code","source":"for file in duplicate:\n    print(f\"{file}\")\n    os.system(f\"rm {file}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"After running this script, you obtain an easier to handle BW 512x512 JPEG image dataset of about 30GB."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}