{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports :","metadata":{}},{"cell_type":"code","source":"# Skipping version problems :\n! pip install --upgrade numpy\n# Fixing non-readable DCIM images issue :\n\n! pip install python-gdcm\n! pip install pylibjpeg pylibjpeg-libjpeg pydicom","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:36:26.972964Z","iopub.execute_input":"2022-10-21T07:36:26.973435Z","iopub.status.idle":"2022-10-21T07:37:04.744392Z","shell.execute_reply.started":"2022-10-21T07:36:26.973341Z","shell.execute_reply":"2022-10-21T07:37:04.7429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport gc\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import ndimage\nimport tensorflow as tf\n\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\n\nimport pydicom as dicom\nimport gdcm\n\n\nimport nibabel as nib","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:04.748335Z","iopub.execute_input":"2022-10-21T07:37:04.748916Z","iopub.status.idle":"2022-10-21T07:37:10.685949Z","shell.execute_reply.started":"2022-10-21T07:37:04.748854Z","shell.execute_reply":"2022-10-21T07:37:10.684571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining directories :","metadata":{}},{"cell_type":"code","source":"# Checking current dir :\n\nINITIAL_PATH = os.getcwd()\nprint(INITIAL_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.687388Z","iopub.execute_input":"2022-10-21T07:37:10.68805Z","iopub.status.idle":"2022-10-21T07:37:10.694869Z","shell.execute_reply.started":"2022-10-21T07:37:10.688011Z","shell.execute_reply":"2022-10-21T07:37:10.693089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Defining each useful path :\n\n# Where the images are :\nTRAIN_IMAGES_PATH = '/kaggle/input/rsna-2022-cervical-spine-fracture-detection/train_images/'\nTEST_IMAGES_PATH = '/kaggle/input/rsna-2022-cervical-spine-fracture-detection/test_images/'\n\n# Pathes of the dataframes :\nTRAIN_CSV_PATH = '/kaggle/input/rsna-2022-cervical-spine-fracture-detection/train.csv'\nTEST_CSV_PATH = '/kaggle/input/rsna-2022-cervical-spine-fracture-detection/test.csv'\n\n# Where we are going to save the preprocessed data :\nTRAIN_OUTPUT_PATH = './train_arrays/'\nTEST_OUTPUT_PATH = './test_arrays/'\nif not os.path.exists(TRAIN_OUTPUT_PATH): os.mkdir(TRAIN_OUTPUT_PATH)\nif not os.path.exists(TEST_OUTPUT_PATH): os.mkdir(TEST_OUTPUT_PATH)  ","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.697533Z","iopub.execute_input":"2022-10-21T07:37:10.697927Z","iopub.status.idle":"2022-10-21T07:37:10.708528Z","shell.execute_reply.started":"2022-10-21T07:37:10.697891Z","shell.execute_reply":"2022-10-21T07:37:10.707298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining other useful features :\n","metadata":{}},{"cell_type":"code","source":"# Defining desired width and depth of the output arrays :\n\ndesired_width = 64\ndesired_height = 64\ndesired_depth = 64","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.710027Z","iopub.execute_input":"2022-10-21T07:37:10.711031Z","iopub.status.idle":"2022-10-21T07:37:10.718801Z","shell.execute_reply.started":"2022-10-21T07:37:10.710992Z","shell.execute_reply":"2022-10-21T07:37:10.717467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Putting images paths into lists : \n\ntrain_images = os.listdir(TRAIN_IMAGES_PATH)\ntest_images = os.listdir(TEST_IMAGES_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.720201Z","iopub.execute_input":"2022-10-21T07:37:10.720579Z","iopub.status.idle":"2022-10-21T07:37:10.831772Z","shell.execute_reply.started":"2022-10-21T07:37:10.720535Z","shell.execute_reply":"2022-10-21T07:37:10.830679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading dataframes :\n\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\ntest_df = pd.read_csv(TEST_CSV_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.83344Z","iopub.execute_input":"2022-10-21T07:37:10.833911Z","iopub.status.idle":"2022-10-21T07:37:10.860573Z","shell.execute_reply.started":"2022-10-21T07:37:10.833864Z","shell.execute_reply":"2022-10-21T07:37:10.859547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List of IDs of every patients :\n\ntrain_patients = train_df.StudyInstanceUID.to_list()","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.862182Z","iopub.execute_input":"2022-10-21T07:37:10.863416Z","iopub.status.idle":"2022-10-21T07:37:10.87465Z","shell.execute_reply.started":"2022-10-21T07:37:10.863367Z","shell.execute_reply":"2022-10-21T07:37:10.873402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the preparation steps :","metadata":{}},{"cell_type":"code","source":"### Creating a function to load dicom files, even if compressed :\n\ndef load_dicom(path: str):\n    \"\"\"Load a dicom file (.dcm) even if it is compressed.\"\"\"\n    try:\n        file_as_array = dicom.dcmread(path).pixel_array\n    except:\n        decompressed_file = gdcm.ImageReader().SetFileName(path).Read()\n        file_as_array = decompressed_file.pixel_array\n    return(file_as_array)","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.876845Z","iopub.execute_input":"2022-10-21T07:37:10.877349Z","iopub.status.idle":"2022-10-21T07:37:10.885032Z","shell.execute_reply.started":"2022-10-21T07:37:10.877304Z","shell.execute_reply":"2022-10-21T07:37:10.884023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating the preprocessing function including loading dicom normalization and resize :\n\ndef preprocessing_slice(slice_path: str):\n    \"\"\"Load dicom of a slice, normalize and resize it\"\"\"\n    # Loading dicom :\n    slice_array = load_dicom(slice_path)\n    slice_array = slice_array.astype(np.uint8)\n    \n    # Normalization :\n    slice_array = slice_array - np.min(slice_array)\n    if np.max(slice_array) != 0:\n        slice_array = slice_array / np.max(slice_array)\n    slice_array = (slice_array * 255).astype(np.uint8)\n        \n    # Resize (2D) :  \n    width_factor = desired_width / slice_array.shape[0]\n    height_factor = desired_height / slice_array.shape[1]\n    \n    slice_array = ndimage.zoom(slice_array, (width_factor, height_factor), order=3) # resize with spline interpolation of order 3\n    \n    return(slice_array)","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.889065Z","iopub.execute_input":"2022-10-21T07:37:10.889418Z","iopub.status.idle":"2022-10-21T07:37:10.901507Z","shell.execute_reply.started":"2022-10-21T07:37:10.889386Z","shell.execute_reply":"2022-10-21T07:37:10.900576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resize_depth(numpy_volume: np.array, desired_depth=desired_depth):\n    \"\"\"Resize across z-axis\"\"\"\n    ## Get current depth\n    current_depth = numpy_volume.shape[0]\n    ## Compute depth factor\n    depth_factor = desired_depth / current_depth\n    \n    ## Resize across z-axis\n    # Rotate\n    numpy_volume = ndimage.rotate(numpy_volume, 90, reshape=False)\n    # Resize\n    volume = ndimage.zoom(numpy_volume, (depth_factor, 1, 1), order=1) # resize with spline interpolation of order 1\n    return volume","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.902885Z","iopub.execute_input":"2022-10-21T07:37:10.903353Z","iopub.status.idle":"2022-10-21T07:37:10.912026Z","shell.execute_reply.started":"2022-10-21T07:37:10.903317Z","shell.execute_reply":"2022-10-21T07:37:10.911113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating a function to parallelize most of the charge of loading the dicom files :\n\ndef load_and_stack_dicom_parallel(scan_path: str):\n    \"\"\"Load all dicom files from a scan and stack them all in a numpy array\"\"\"\n    # Defining slice paths :\n    slice_paths = sorted(glob.glob(os.path.join(scan_path, \"*\")),\n                         key=lambda x: int(x.split('/')[-1].split(\".\")[0]))\n    \n    # Preprocessing slices :\n    images = Parallel(n_jobs=-1)(delayed(preprocessing_slice)(filename) for filename in slice_paths)\n    \n    # Returning stacked slices as a resized on depth (3rd dimension) volume :\n    return(tf.expand_dims(resize_depth(np.array(images)), axis=3))","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.91361Z","iopub.execute_input":"2022-10-21T07:37:10.914176Z","iopub.status.idle":"2022-10-21T07:37:10.926581Z","shell.execute_reply.started":"2022-10-21T07:37:10.914131Z","shell.execute_reply":"2022-10-21T07:37:10.92546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating preprocessed data :","metadata":{}},{"cell_type":"code","source":"# Function to create and save the 3D volumes corresponding to a scan :\n\ndef save_3D_arrays(scan_path: str, output_path: str):\n    \"\"\"Create and save the 3D arrays corresponding to a scan\"\"\"\n    \n    # Preprocessing and creation :\n    volume = load_and_stack_dicom_parallel(scan_path=scan_path)\n    \n    # Saving the numpy array :\n    volume_file_name = output_path + scan_path.split('/')[-1] + '.npy'\n    np.save(volume_file_name, volume)\n    \n    # Deleting in memory :\n    del volume\n    \n    return None","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.928252Z","iopub.execute_input":"2022-10-21T07:37:10.929067Z","iopub.status.idle":"2022-10-21T07:37:10.944969Z","shell.execute_reply.started":"2022-10-21T07:37:10.929026Z","shell.execute_reply":"2022-10-21T07:37:10.943776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creation of the preprocessed array volumes :\n\nfor i in tqdm(range(len(train_patients))):\n    case_path = TRAIN_IMAGES_PATH + train_patients[i]\n    save_3D_arrays(case_path, TRAIN_OUTPUT_PATH)\n\n\n# Free up memory :\n\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-10-21T07:37:10.946234Z","iopub.execute_input":"2022-10-21T07:37:10.946987Z","iopub.status.idle":"2022-10-21T09:28:35.178217Z","shell.execute_reply.started":"2022-10-21T07:37:10.946944Z","shell.execute_reply":"2022-10-21T09:28:35.17682Z"},"trusted":true},"execution_count":null,"outputs":[]}]}