{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA: Loading the DICOM Images into a PyTorch DataLoader\n(Initially keeping it simple for beginners like me)","metadata":{}},{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-31T18:18:10.528508Z","iopub.execute_input":"2023-01-31T18:18:10.528959Z","iopub.status.idle":"2023-01-31T18:18:34.823329Z","shell.execute_reply.started":"2023-01-31T18:18:10.528916Z","shell.execute_reply":"2023-01-31T18:18:34.82203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import packages","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport glob\nimport pydicom\nimport gdcm\nimport cv2\nimport torchvision\nimport torch\nfrom torchvision.io import read_image\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:34.826428Z","iopub.execute_input":"2023-01-31T18:18:34.826846Z","iopub.status.idle":"2023-01-31T18:18:34.836919Z","shell.execute_reply.started":"2023-01-31T18:18:34.826803Z","shell.execute_reply":"2023-01-31T18:18:34.835233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load files","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\ntrain_paths = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/train_images/*/*.dcm\")\ntest_paths = glob.glob(\"/kaggle/input/rsna-breast-cancer-detection/test_images/*/*.dcm\")","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:34.838462Z","iopub.execute_input":"2023-01-31T18:18:34.8391Z","iopub.status.idle":"2023-01-31T18:18:45.136106Z","shell.execute_reply.started":"2023-01-31T18:18:34.83906Z","shell.execute_reply":"2023-01-31T18:18:45.135082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### Only using a few images for demonstrative purposes","metadata":{}},{"cell_type":"code","source":"# Only using N images for demonstrative purposes\nN_SAMPLES = 102\ntrain_paths.sort()\ntrain_paths = train_paths[:N_SAMPLES]\ntrain = train[:N_SAMPLES]\nprint([sample[56:-4].split('/') for sample in train_paths][-5:])\ntrain.tail(5)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-01-31T18:18:45.139793Z","iopub.execute_input":"2023-01-31T18:18:45.140704Z","iopub.status.idle":"2023-01-31T18:18:45.175632Z","shell.execute_reply.started":"2023-01-31T18:18:45.14067Z","shell.execute_reply":"2023-01-31T18:18:45.17453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just checking that the same sample images are saved.","metadata":{"jupyter":{"source_hidden":true}}},{"cell_type":"code","source":"glob.glob(\"/kaggle/working/output/test/*\")","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-01-31T18:18:45.179316Z","iopub.execute_input":"2023-01-31T18:18:45.180059Z","iopub.status.idle":"2023-01-31T18:18:45.198419Z","shell.execute_reply.started":"2023-01-31T18:18:45.180022Z","shell.execute_reply":"2023-01-31T18:18:45.196902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['img_path'] = '/kaggle/working/output/train/' + train.patient_id.astype('string') + \"_\" + train.image_id.astype('string') + '.png'\ntest['img_path'] = '/kaggle/working/output/test/' + test.patient_id.astype('string') + \"_\" + test.image_id.astype('string') + '.png'\ntrain[['patient_id', 'image_id', 'img_path']].head()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:45.201445Z","iopub.execute_input":"2023-01-31T18:18:45.202295Z","iopub.status.idle":"2023-01-31T18:18:45.222637Z","shell.execute_reply.started":"2023-01-31T18:18:45.202258Z","shell.execute_reply":"2023-01-31T18:18:45.221308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving DICOM images into PNGs\n[With a lot of help from this notebook](https://www.kaggle.com/code/theoviel/dicom-resized-png-jpg)","metadata":{}},{"cell_type":"code","source":"def convert(path, train=True, show=False, size=256):\n    subfolder = 'train' if train else 'test'\n    patient = path.split('/')[-2]\n    image_number = path.split('/')[-1][:-4] # removes the .dcm\n    \n    dicom = pydicom.dcmread(path)\n    img = dicom.pixel_array\n    \n    img = (img - img.min())/(img.max() - img.min()) # Normalisation\n    \n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img # some images are inverted\n    \n    img = cv2.resize(img, (size,size))\n    img = (img * 255).astype(np.uint8)\n    \n#     if show:\n#         plt.figure(figsize=(5,5))\n#         plt.imshow(img, cmap='gray')\n#         plt.title(f'Patient {patient}, Image {image_number}')\n#         plt.show()\n        \n    cv2.imwrite(f'output/{subfolder}/{patient}_{image_number}.png', img)\n    \n    return img","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:45.22389Z","iopub.execute_input":"2023-01-31T18:18:45.224333Z","iopub.status.idle":"2023-01-31T18:18:45.231926Z","shell.execute_reply.started":"2023-01-31T18:18:45.224297Z","shell.execute_reply":"2023-01-31T18:18:45.230624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Making the new folders to put our new images","metadata":{}},{"cell_type":"code","source":"os.makedirs('output/train/', exist_ok=True)\nos.makedirs('output/test/', exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:45.233132Z","iopub.execute_input":"2023-01-31T18:18:45.233526Z","iopub.status.idle":"2023-01-31T18:18:45.24418Z","shell.execute_reply.started":"2023-01-31T18:18:45.233492Z","shell.execute_reply":"2023-01-31T18:18:45.243308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using the GPU to do the work","metadata":{}},{"cell_type":"code","source":"_ = Parallel(n_jobs=4)(\n    delayed(convert)(x, train=True)\n    for x in tqdm(train_paths)\n)\n_ = Parallel(n_jobs=4)(\n    delayed(convert)(x, train=False)\n    for x in tqdm(test_paths)\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:18:45.245201Z","iopub.execute_input":"2023-01-31T18:18:45.245499Z","iopub.status.idle":"2023-01-31T18:20:21.55566Z","shell.execute_reply.started":"2023-01-31T18:18:45.245475Z","shell.execute_reply":"2023-01-31T18:20:21.55452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a PyTorch Dataset\n[Datasets & DataLoaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)","metadata":{}},{"cell_type":"code","source":"class RSNADataset(Dataset):\n    def __init__(self, df, transform=None, is_test=False):\n        self.df = df\n        self.transform = transform\n        self.is_test = is_test\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        img_path = self.df.img_path[idx]\n        image = read_image(img_path)\n        if self.transform:\n            image = self.transform(image)\n        if self.is_test:\n            return image\n        label = self.df.cancer[idx]\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:20:21.557596Z","iopub.execute_input":"2023-01-31T18:20:21.557964Z","iopub.status.idle":"2023-01-31T18:20:21.569199Z","shell.execute_reply.started":"2023-01-31T18:20:21.557931Z","shell.execute_reply":"2023-01-31T18:20:21.568184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating a DataLoader\n[Datasets & DataLoaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)","metadata":{}},{"cell_type":"code","source":"train_dataset = RSNADataset(df=train)\ntrain_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n\n# Display image and label.\ntrain_features, train_labels = next(iter(train_dataloader))\nprint(f\"Feature batch shape: {train_features.size()}\")\nprint(f\"Labels batch shape: {train_labels.size()}\")\n\nfig = plt.figure(figsize=(10,8))\n\nfor i in range(4):\n    fig.add_subplot(2,2,i+1)\n    plt.axis('off')\n    plt.title(train_labels[i].item())\n    plt.imshow(train_features[i].squeeze(), cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:20:21.57117Z","iopub.execute_input":"2023-01-31T18:20:21.571764Z","iopub.status.idle":"2023-01-31T18:20:21.898241Z","shell.execute_reply.started":"2023-01-31T18:20:21.571724Z","shell.execute_reply":"2023-01-31T18:20:21.897258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = RSNADataset(df=test, is_test=True)\ntest_dataloader = DataLoader(test_dataset, batch_size=4)\n\n# Display image and label.\ntest_pics = next(iter(test_dataloader))\nfig = plt.figure(figsize=(10,8))\n\nfor i in range(4):\n    fig.add_subplot(2,2,i+1)\n    plt.axis('off')\n    plt.imshow(test_pics[i].squeeze(), cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-01-31T18:20:21.899481Z","iopub.execute_input":"2023-01-31T18:20:21.900274Z","iopub.status.idle":"2023-01-31T18:20:22.21011Z","shell.execute_reply.started":"2023-01-31T18:20:21.90021Z","shell.execute_reply":"2023-01-31T18:20:22.209185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Thank you!","metadata":{}}]}