{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Introduction\n\nThis kernels uses the recent pip wheel of DALI for decoding dicoms using GPU. It works for all JPEG2000 and most of the JPEG-lossless formated images.\n\nThe decoding work strongly is based on the kernels of Theo Viel (@theoviel) and David Austin (@tivfrvqhs5)\n\n***WARNING***: Allthough the GPU decoding works for all train images, a few of the JPEG-lossless formated DICOMS (TransferSyntaxUID == '1.2.840.10008.1.2.4.70') of the hidden test set cannot be decoded. So its crucial to have a CPU fallback in place so the notebook wont throw an exception in the submission re-run","metadata":{"execution":{"iopub.execute_input":"2023-02-05T06:44:19.900699Z","iopub.status.busy":"2023-02-05T06:44:19.900319Z","iopub.status.idle":"2023-02-05T06:44:19.920765Z","shell.execute_reply":"2023-02-05T06:44:19.919665Z","shell.execute_reply.started":"2023-02-05T06:44:19.900617Z"},"papermill":{"duration":0.016446,"end_time":"2023-02-06T12:29:18.4474","exception":false,"start_time":"2023-02-06T12:29:18.430954","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Requirements","metadata":{"papermill":{"duration":0.016063,"end_time":"2023-02-06T12:29:18.479831","exception":false,"start_time":"2023-02-06T12:29:18.463768","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"We start with installing pip requirements.","metadata":{"papermill":{"duration":0.007743,"end_time":"2023-02-06T12:29:18.500637","exception":false,"start_time":"2023-02-06T12:29:18.492894","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q timm==0.6.5 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q albumentations==1.2.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q pylibjpeg-libjpeg==1.3.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q pydicom==2.0.0 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q python-gdcm==3.0.20 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q dicomsdl==0.109.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":66.474137,"end_time":"2023-02-06T12:30:24.982634","exception":false,"start_time":"2023-02-06T12:29:18.508497","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:00:45.663935Z","iopub.execute_input":"2023-02-11T04:00:45.664319Z","iopub.status.idle":"2023-02-11T04:01:48.922844Z","shell.execute_reply.started":"2023-02-11T04:00:45.664224Z","shell.execute_reply":"2023-02-11T04:01:48.921551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we install the latest DALI packaging which we will use for GPU decoding","metadata":{"papermill":{"duration":0.01318,"end_time":"2023-02-06T12:30:25.009684","exception":false,"start_time":"2023-02-06T12:30:24.996504","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q /kaggle/input/nvidia-dali-nightly-cuda110-1230dev/nvidia_dali_nightly_cuda110-1.23.0.dev20230203-7187866-py3-none-manylinux2014_x86_64.whl","metadata":{"papermill":{"duration":42.356106,"end_time":"2023-02-06T12:31:07.378705","exception":false,"start_time":"2023-02-06T12:30:25.022599","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:01:48.925243Z","iopub.execute_input":"2023-02-11T04:01:48.925974Z","iopub.status.idle":"2023-02-11T04:02:30.69986Z","shell.execute_reply.started":"2023-02-11T04:01:48.92593Z","shell.execute_reply":"2023-02-11T04:02:30.698654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we import all the packages we need and patch a function to allow for INT16 support","metadata":{"papermill":{"duration":0.008194,"end_time":"2023-02-06T12:31:07.395534","exception":false,"start_time":"2023-02-06T12:31:07.38734","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import timm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom copy import copy\nimport gc\nimport shutil \n\nimport glob\nfrom scipy.special import expit\n\nimport albumentations as A\nimport cv2\ncv2.setNumThreads(0)\n\nimport dicomsdl\nimport pydicom\nfrom pydicom.filebase import DicomBytesIO\n\nfrom os.path import join\n\nfrom tqdm import tqdm\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\n\nfrom types import SimpleNamespace\nfrom typing import Any, Dict\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import GradScaler, autocast\n\n\nimport nvidia.dali.fn as fn\nimport nvidia.dali.types as types\nfrom nvidia.dali import pipeline_def\nfrom nvidia.dali.types import DALIDataType","metadata":{"papermill":{"duration":5.037552,"end_time":"2023-02-06T12:31:12.441309","exception":false,"start_time":"2023-02-06T12:31:07.403757","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:07.291472Z","iopub.execute_input":"2023-02-11T04:09:07.292628Z","iopub.status.idle":"2023-02-11T04:09:11.74913Z","shell.execute_reply.started":"2023-02-11T04:09:07.292577Z","shell.execute_reply":"2023-02-11T04:09:11.748048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we need to patch DALI for Int16 support\n\n\nfrom nvidia.dali.backend import TensorGPU, TensorListGPU\nfrom nvidia.dali.pipeline import Pipeline\nimport nvidia.dali.ops as ops\nfrom nvidia.dali import types\nfrom nvidia.dali.plugin.base_iterator import _DaliBaseIterator\nfrom nvidia.dali.plugin.base_iterator import LastBatchPolicy\nimport torch\nimport torch.utils.dlpack as torch_dlpack\nimport ctypes\nimport numpy as np\nimport torch.nn.functional as F\nimport pydicom\n\nto_torch_type = {\n    types.DALIDataType.FLOAT:   torch.float32,\n    types.DALIDataType.FLOAT64: torch.float64,\n    types.DALIDataType.FLOAT16: torch.float16,\n    types.DALIDataType.UINT8:   torch.uint8,\n    types.DALIDataType.INT8:    torch.int8,\n    types.DALIDataType.UINT16:  torch.int16,\n    types.DALIDataType.INT16:   torch.int16,\n    types.DALIDataType.INT32:   torch.int32,\n    types.DALIDataType.INT64:   torch.int64\n}\n\n\ndef feed_ndarray(dali_tensor, arr, cuda_stream=None):\n    \"\"\"\n    Copy contents of DALI tensor to PyTorch's Tensor.\n\n    Parameters\n    ----------\n    `dali_tensor` : nvidia.dali.backend.TensorCPU or nvidia.dali.backend.TensorGPU\n                    Tensor from which to copy\n    `arr` : torch.Tensor\n            Destination of the copy\n    `cuda_stream` : torch.cuda.Stream, cudaStream_t or any value that can be cast to cudaStream_t.\n                    CUDA stream to be used for the copy\n                    (if not provided, an internal user stream will be selected)\n                    In most cases, using pytorch's current stream is expected (for example,\n                    if we are copying to a tensor allocated with torch.zeros(...))\n    \"\"\"\n    dali_type = to_torch_type[dali_tensor.dtype]\n\n    assert dali_type == arr.dtype, (\"The element type of DALI Tensor/TensorList\"\n                                    \" doesn't match the element type of the target PyTorch Tensor: \"\n                                    \"{} vs {}\".format(dali_type, arr.dtype))\n    assert dali_tensor.shape() == list(arr.size()), \\\n        (\"Shapes do not match: DALI tensor has size {0}, but PyTorch Tensor has size {1}\".\n            format(dali_tensor.shape(), list(arr.size())))\n    cuda_stream = types._raw_cuda_stream(cuda_stream)\n\n    # turn raw int to a c void pointer\n    c_type_pointer = ctypes.c_void_p(arr.data_ptr())\n    if isinstance(dali_tensor, (TensorGPU, TensorListGPU)):\n        stream = None if cuda_stream is None else ctypes.c_void_p(cuda_stream)\n        dali_tensor.copy_to_external(c_type_pointer, stream, non_blocking=True)\n    else:\n        dali_tensor.copy_to_external(c_type_pointer)\n    return arr\n\n\n\n","metadata":{"papermill":{"duration":0.02571,"end_time":"2023-02-06T12:31:12.47592","exception":false,"start_time":"2023-02-06T12:31:12.45021","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:11.751271Z","iopub.execute_input":"2023-02-11T04:09:11.751634Z","iopub.status.idle":"2023-02-11T04:09:11.768961Z","shell.execute_reply.started":"2023-02-11T04:09:11.7516Z","shell.execute_reply":"2023-02-11T04:09:11.76804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next I set major variables which handle the public run and the re-run on the hidden test set, and also allow for simulating the size of the hidden test set by setting RAM_CHECK = True","metadata":{"papermill":{"duration":0.008037,"end_time":"2023-02-06T12:31:12.49277","exception":false,"start_time":"2023-02-06T12:31:12.484733","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Params\n\nCOMP_FOLDER = '/kaggle/input/rsna-breast-cancer-detection/'\nDATA_FOLDER = COMP_FOLDER + 'train_images/'\n\nsample_submission = pd.read_csv(COMP_FOLDER + 'train.csv')\n\nPUBLIC_RUN = len(sample_submission) == 2\n\nN_CORES = mp.cpu_count()\nMIXED_PRECISION = False\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nRAM_CHECK = True\nDEBUG = True\n\ntest_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntest_df['cancer'] = 0 #dummy value\n\n\nif PUBLIC_RUN is False:\n    RAM_CHECK = False\n    DEBUG = False\n\n# if RAM_CHECK is True:\n#     test_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n#     patient_filter = list(sorted((set(test_df.patient_id.unique()))))[:8000]\n#     test_df = test_df[test_df.patient_id.isin(patient_filter)]\n#     DATA_FOLDER = DATA_FOLDER.replace('test','train')\n\n# if DEBUG is True:\n#     test_df = test_df.head(500)\n\ntest_df","metadata":{"papermill":{"duration":0.23946,"end_time":"2023-02-06T12:31:12.740724","exception":false,"start_time":"2023-02-06T12:31:12.501264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:11.77062Z","iopub.execute_input":"2023-02-11T04:09:11.770985Z","iopub.status.idle":"2023-02-11T04:09:12.041222Z","shell.execute_reply.started":"2023-02-11T04:09:11.770951Z","shell.execute_reply":"2023-02-11T04:09:12.040252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Len df : {len(test_df)}')\ntest_df['patient_id'].nunique()","metadata":{"papermill":{"duration":0.022631,"end_time":"2023-02-06T12:31:12.773582","exception":false,"start_time":"2023-02-06T12:31:12.750951","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:12.04333Z","iopub.execute_input":"2023-02-11T04:09:12.043952Z","iopub.status.idle":"2023-02-11T04:09:12.057396Z","shell.execute_reply.started":"2023-02-11T04:09:12.043914Z","shell.execute_reply":"2023-02-11T04:09:12.056442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"fns\"] = test_df['patient_id'].astype(str) + '/' + test_df['image_id'].astype(str) + '.dcm'","metadata":{"papermill":{"duration":0.019117,"end_time":"2023-02-06T12:31:12.801584","exception":false,"start_time":"2023-02-06T12:31:12.782467","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:27.017071Z","iopub.execute_input":"2023-02-11T04:09:27.018041Z","iopub.status.idle":"2023-02-11T04:09:27.094986Z","shell.execute_reply.started":"2023-02-11T04:09:27.018005Z","shell.execute_reply":"2023-02-11T04:09:27.094012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we define the function for GPU-based decoding using DALI and processing the dicom images","metadata":{"papermill":{"duration":0.008603,"end_time":"2023-02-06T12:31:12.818867","exception":false,"start_time":"2023-02-06T12:31:12.810264","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def convert_dicom_to_jpg(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")  #<---- the jpeg2000 header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n            \n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.70':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\xff\\xd8\\xff\\xe0\")  #<---- the jpeg lossless header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n            \n@pipeline_def\ndef jpg_decode_pipeline(jpgfiles):\n    jpegs, _ = fn.readers.file(files=jpgfiles)\n    images = fn.experimental.decoders.image(jpegs, device='mixed', output_type=types.ANY_DATA, dtype=DALIDataType.UINT16)\n    return images\n\ndef parse_window_element(elem):\n    if type(elem)==list:\n        return float(elem[0])\n    if type(elem)==str:\n        return float(elem)\n    if type(elem)==float:\n        return elem\n    if type(elem)==pydicom.dataelem.DataElement:\n        try:\n            return float(elem[0])\n        except:\n            return float(elem.value)\n    return None\n\ndef linear_window(data, center, width):\n    lower, upper = center - width // 2, center + width // 2\n    data = torch.clamp(data, min=lower, max=upper)\n    return data \n\ndef process_dicom(img, dicom):\n    try:\n        invert = getattr(dicom, \"PhotometricInterpretation\", None) == \"MONOCHROME1\"\n    except:\n        invert = False\n        \n    center = parse_window_element(dicom[\"WindowCenter\"]) \n    width = parse_window_element(dicom[\"WindowWidth\"])\n        \n    if (center is not None) & (width is not None):\n        img = linear_window(img, center, width)\n\n    img = (img - img.min()) / (img.max() - img.min())\n    if invert:\n        img = 1 - img\n    return img","metadata":{"papermill":{"duration":0.026255,"end_time":"2023-02-06T12:31:12.853956","exception":false,"start_time":"2023-02-06T12:31:12.827701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:33.150816Z","iopub.execute_input":"2023-02-11T04:09:33.15119Z","iopub.status.idle":"2023-02-11T04:09:33.166685Z","shell.execute_reply.started":"2023-02-11T04:09:33.151157Z","shell.execute_reply":"2023-02-11T04:09:33.165488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.008783,"end_time":"2023-02-06T12:31:12.87152","exception":false,"start_time":"2023-02-06T12:31:12.862737","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = SimpleNamespace(**{})\ncfg.img_size = 1024\ncfg.backbone = 'seresnext50_32x4d'\ncfg.pretrained=False\ncfg.in_channels = 1\ncfg.classes = ['cancer']\ncfg.batch_size = 8\ncfg.data_folder = \"./\"\ncfg.val_aug = A.CenterCrop(always_apply=False, p=1.0, height=cfg.img_size, width=cfg.img_size)\ncfg.device = DEVICE","metadata":{"papermill":{"duration":0.018519,"end_time":"2023-02-06T12:31:12.898729","exception":false,"start_time":"2023-02-06T12:31:12.88021","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:37.009973Z","iopub.execute_input":"2023-02-11T04:09:37.01036Z","iopub.status.idle":"2023-02-11T04:09:37.017375Z","shell.execute_reply.started":"2023-02-11T04:09:37.010328Z","shell.execute_reply":"2023-02-11T04:09:37.016247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will process the dicoms in chunks so the disk space does not become an issue. ","metadata":{"execution":{"iopub.execute_input":"2023-02-05T07:14:35.534267Z","iopub.status.busy":"2023-02-05T07:14:35.533224Z","iopub.status.idle":"2023-02-05T07:14:35.545681Z","shell.execute_reply":"2023-02-05T07:14:35.544691Z","shell.execute_reply.started":"2023-02-05T07:14:35.534231Z"},"papermill":{"duration":0.008545,"end_time":"2023-02-06T12:31:12.916133","exception":false,"start_time":"2023-02-06T12:31:12.907588","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SAVE_SIZE = int(cfg.img_size * 1.125)\nSAVE_FOLDER = cfg.data_folder\nos.makedirs(SAVE_FOLDER, exist_ok=True)\nN_CHUNKS = len(test_df[\"fns\"]) // 2000 if len(test_df[\"fns\"]) > 2000 else 1\nCHUNKS = [(len(test_df[\"fns\"]) / N_CHUNKS * k, len(test_df[\"fns\"]) / N_CHUNKS * (k + 1)) for k in range(N_CHUNKS)]\nCHUNKS = np.array(CHUNKS).astype(int)\nJPG_FOLDER = \"/tmp/jpg/\"","metadata":{"papermill":{"duration":0.018556,"end_time":"2023-02-06T12:31:12.943654","exception":false,"start_time":"2023-02-06T12:31:12.925098","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:41.128945Z","iopub.execute_input":"2023-02-11T04:09:41.129347Z","iopub.status.idle":"2023-02-11T04:09:41.137186Z","shell.execute_reply.started":"2023-02-11T04:09:41.129314Z","shell.execute_reply":"2023-02-11T04:09:41.136002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfor ttt, chunk in enumerate(CHUNKS):\n    print(f'chunk {ttt} of {len(CHUNKS)} chunks')\n    os.makedirs(JPG_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_jpg)(f'{DATA_FOLDER}/{img}', save_folder=JPG_FOLDER)\n        for img in test_df[\"fns\"].tolist()[chunk[0]: chunk[1]]\n    )\n    \n    jpgfiles = glob.glob(JPG_FOLDER + \"*.jpg\")\n\n\n    pipe = jpg_decode_pipeline(jpgfiles, batch_size=1, num_threads=2, device_id=0)\n    pipe.build()\n\n    for i, f in enumerate(tqdm(jpgfiles)):\n        \n        patient, dicom_id = f.split('/')[-1][:-4].split('_')\n        dicom = pydicom.dcmread(DATA_FOLDER + f\"/{patient}/{dicom_id}.dcm\")\n        try:\n            out = pipe.run()\n            # Dali -> Torch\n            img = out[0][0]\n            img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n            feed_ndarray(img, img_torch, cuda_stream=torch.cuda.current_stream(device=0))\n            img = img_torch.float()\n\n            \n\n\n            #apply dicom preprocessing\n            img = process_dicom(img, dicom)\n\n            #resize the torch image\n            img = F.interpolate(img.view(1, 1, img.size(0), img.size(1)), (SAVE_SIZE, SAVE_SIZE), mode=\"bilinear\")[0, 0]\n\n            img = (img * 255).clip(0,255).to(torch.uint8).cpu().numpy()\n            out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n            cv2.imwrite(out_file_name, img)\n    \n        except Exception as e:\n            print(i, e)\n            pipe = jpg_decode_pipeline(jpgfiles[i+1:], batch_size=1, num_threads=2, device_id=0)\n            pipe.build()\n            continue\n\n    shutil.rmtree(JPG_FOLDER)\nprint(f'DALI Raw image load complete')","metadata":{"papermill":{"duration":102.364817,"end_time":"2023-02-06T12:32:55.317413","exception":false,"start_time":"2023-02-06T12:31:12.952596","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-11T04:09:43.268297Z","iopub.execute_input":"2023-02-11T04:09:43.269009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns = glob.glob(f'{SAVE_FOLDER}/*.png')\nn_saved = len(fns)\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.03725,"end_time":"2023-02-06T12:32:55.38013","exception":false,"start_time":"2023-02-06T12:32:55.34288","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A few hidden test set images might not be decoded via DALI, so we fallback to CPU for those","metadata":{"papermill":{"duration":0.024386,"end_time":"2023-02-06T12:32:55.429853","exception":false,"start_time":"2023-02-06T12:32:55.405467","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gpu_processed_files = [fn.split('/')[-1].replace('_','/').replace('png','dcm') for fn in fns]\nto_process = [f for f in test_df[\"fns\"].values if f not in gpu_processed_files]\nlen(gpu_processed_files), len(to_process)","metadata":{"papermill":{"duration":0.039261,"end_time":"2023-02-06T12:32:55.495229","exception":false,"start_time":"2023-02-06T12:32:55.455968","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef process(f, save_folder=\"\"):\n    patient = f.split('/')[-2]\n    dicom_id = f.split('/')[-1][:-4]\n    \n    dicom = dicomsdl.open(f)\n    img = dicom.pixelData()\n    img = torch.from_numpy(img)\n    img = process_dicom(img, dicom)\n    \n    img = F.interpolate(img.view(1, 1, img.size(0), img.size(1)), (SAVE_SIZE, SAVE_SIZE), mode=\"bilinear\")[0, 0]\n\n    img = (img * 255).clip(0,255).to(torch.uint8).cpu().numpy()\n    out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n    cv2.imwrite(out_file_name, img)\n    return out_file_name","metadata":{"papermill":{"duration":0.037403,"end_time":"2023-02-06T12:32:55.557572","exception":false,"start_time":"2023-02-06T12:32:55.520169","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_processed_filenames = Parallel(n_jobs=2)(\n    delayed(process)(f'{DATA_FOLDER}/{img}', save_folder=SAVE_FOLDER)\n    for img in tqdm(to_process)\n)\ncpu_processed_filenames = [f for f in cpu_processed_filenames if f]\nprint(f'CPU Raw image load complete with {len(cpu_processed_filenames)} loaded')","metadata":{"papermill":{"duration":0.045144,"end_time":"2023-02-06T12:32:55.628204","exception":false,"start_time":"2023-02-06T12:32:55.58306","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.265635,"end_time":"2023-02-06T12:32:55.919677","exception":false,"start_time":"2023-02-06T12:32:55.654042","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_saved = len(glob.glob(f'{SAVE_FOLDER}/*.png'))\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.038085,"end_time":"2023-02-06T12:32:55.983371","exception":false,"start_time":"2023-02-06T12:32:55.945286","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert n_saved == len(test_df)","metadata":{"papermill":{"duration":0.033708,"end_time":"2023-02-06T12:32:56.043435","exception":false,"start_time":"2023-02-06T12:32:56.009727","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We finished with preprocessing all the dicoms to images. So next, we set-up the dataloading and model","metadata":{"papermill":{"duration":0.025094,"end_time":"2023-02-06T12:32:56.09348","exception":false,"start_time":"2023-02-06T12:32:56.068386","status":"completed"},"tags":[]}}]}