{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Easy load the image with nvJPEG2000\n\nThis code try to load the dicom file with [nvJPEG2000](https://docs.nvidia.com/cuda/nvjpeg2000/userguide.html), it's implemented with a Python extension, 5x faster than normal load with Kaggle's P100, and is very easy to use.\n\nThe extensin source code is opensourced here: https://github.com/louis-she/nvjpeg2k-python. The extension in the dataset is built against Python 3.7, if using with another Python version it will failed. But one can always compile it from source.\n\nThanks to the trick posted here: https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/371534. I'm not a big fan of dali so I write this.\n\n<span style=\"color: red\">This can not be used with concorrent case(e.g. multi processing or PyTorch's dataloader with num_workers > 1</span>","metadata":{}},{"cell_type":"code","source":"!cp /kaggle/input/nvjpeg2k/nvjpeg2k.so ./\n!pip install -q --disable-pip-version-check /kaggle/input/rsna-2022-whl/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install -q --disable-pip-version-check /kaggle/input/rsna-2022-whl/python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:25:28.577223Z","iopub.execute_input":"2022-12-15T07:25:28.57762Z","iopub.status.idle":"2022-12-15T07:25:49.530579Z","shell.execute_reply.started":"2022-12-15T07:25:28.577577Z","shell.execute_reply":"2022-12-15T07:25:49.529379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pydicom.filebase import DicomBytesIO\nimport pydicom\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nimport nvjpeg2k\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:26:21.141523Z","iopub.execute_input":"2022-12-15T07:26:21.141889Z","iopub.status.idle":"2022-12-15T07:26:21.148231Z","shell.execute_reply.started":"2022-12-15T07:26:21.141859Z","shell.execute_reply":"2022-12-15T07:26:21.147241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"decoder = nvjpeg2k.Decoder()\n\ndef load_dicom(path, force_slow=False):\n    dcmfile = pydicom.dcmread(path)\n    if not force_slow and dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(path, 'rb') as f:\n            raw = DicomBytesIO(f.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        return decoder.decode(hackedbitstream)\n    else:\n        return dcmfile.pixel_array","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:26:21.444144Z","iopub.execute_input":"2022-12-15T07:26:21.444472Z","iopub.status.idle":"2022-12-15T07:26:21.580974Z","shell.execute_reply.started":"2022-12-15T07:26:21.444442Z","shell.execute_reply":"2022-12-15T07:26:21.579683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_path(item):\n    return Path(\"/kaggle/input/rsna-breast-cancer-detection/train_images\") / item[\"patient_id\"] / (item[\"image_id\"] + \".dcm\")","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:26:21.958183Z","iopub.execute_input":"2022-12-15T07:26:21.958517Z","iopub.status.idle":"2022-12-15T07:26:21.963641Z","shell.execute_reply.started":"2022-12-15T07:26:21.958487Z","shell.execute_reply":"2022-12-15T07:26:21.962641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seperate the df into 2k and non 2K parts, and do the benchmark.\n# for demonstration only, we only use the first 200 samples.\ndf = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\", dtype={\"patient_id\": str, \"image_id\": str})\ndf200 = df.head(200).reset_index()\nfor i, item in tqdm(df200.iterrows(), total=200):\n    path = get_path(item)\n    dcmfile = pydicom.dcmread(path)\n    df200.loc[i, \"is2k\"] = dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90' ","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:26:22.436806Z","iopub.execute_input":"2022-12-15T07:26:22.437789Z","iopub.status.idle":"2022-12-15T07:26:23.559005Z","shell.execute_reply.started":"2022-12-15T07:26:22.437727Z","shell.execute_reply":"2022-12-15T07:26:23.558032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df200_2k = df200.loc[df200.is2k]\ndf200_non2k = df200.loc[df200.is2k == False]\n\nprint(df200_2k.shape, df200_non2k.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:26:25.137844Z","iopub.execute_input":"2022-12-15T07:26:25.138215Z","iopub.status.idle":"2022-12-15T07:26:25.148803Z","shell.execute_reply.started":"2022-12-15T07:26:25.138184Z","shell.execute_reply":"2022-12-15T07:26:25.147821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _, row in tqdm(df200_non2k.iterrows(), total=len(df200_non2k)):\n    load_dicom(get_path(row))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T06:56:32.067626Z","iopub.execute_input":"2022-12-15T06:56:32.068658Z","iopub.status.idle":"2022-12-15T06:57:12.824509Z","shell.execute_reply.started":"2022-12-15T06:56:32.068609Z","shell.execute_reply":"2022-12-15T06:57:12.823496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for _, row in tqdm(df200_2k.iterrows(), total=len(df200_2k)):\n    load_dicom(get_path(row))","metadata":{"execution":{"iopub.status.busy":"2022-12-15T06:57:16.152453Z","iopub.execute_input":"2022-12-15T06:57:16.152842Z","iopub.status.idle":"2022-12-15T06:57:26.259039Z","shell.execute_reply.started":"2022-12-15T06:57:16.152808Z","shell.execute_reply":"2022-12-15T06:57:26.258028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is about 4x faster than normal load. Now lets compare if the output values are all same.","metadata":{}},{"cell_type":"code","source":"for _, row in tqdm(df200_2k.iterrows(), total=len(df200_2k)):\n    path = get_path(row)\n    load_slow = load_dicom(path, force_slow=True)\n    load_fast = load_dicom(path)\n    \n    assert np.array_equal(load_slow, load_fast)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:02:25.722247Z","iopub.execute_input":"2022-12-15T07:02:25.722909Z","iopub.status.idle":"2022-12-15T07:04:30.011081Z","shell.execute_reply.started":"2022-12-15T07:02:25.722873Z","shell.execute_reply":"2022-12-15T07:04:30.01Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Yeah! all the tensor is matched. In the final, let's see how much memory it takes.\n\nI'm pretty sure there is no gpu memory leak since that if you load 10 images or 100 images, the gpu memory usage is almost the same.","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-12-15T07:11:16.646412Z","iopub.execute_input":"2022-12-15T07:11:16.646825Z","iopub.status.idle":"2022-12-15T07:11:17.665986Z","shell.execute_reply.started":"2022-12-15T07:11:16.646774Z","shell.execute_reply":"2022-12-15T07:11:17.664827Z"},"trusted":true},"execution_count":null,"outputs":[]}]}