{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install openslide-python\n!pip install staintools\n!pip install albumentations\n!pip install pytorch-lightning","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-24T09:26:39.09287Z","iopub.execute_input":"2023-03-24T09:26:39.093418Z","iopub.status.idle":"2023-03-24T09:27:30.41198Z","shell.execute_reply.started":"2023-03-24T09:26:39.093372Z","shell.execute_reply":"2023-03-24T09:27:30.410194Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"Requirement already satisfied: openslide-python in /opt/conda/lib/python3.7/site-packages (1.2.0)\nRequirement already satisfied: Pillow in /opt/conda/lib/python3.7/site-packages (from openslide-python) (9.4.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mCollecting staintools\n  Downloading staintools-2.1.2.tar.gz (7.0 kB)\n  Preparing metadata (setup.py) ... \u001b[?25ldone\n\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from staintools) (1.21.6)\nRequirement already satisfied: matplotlib in /opt/conda/lib/python3.7/site-packages (from staintools) (3.5.3)\nRequirement already satisfied: opencv-python in /opt/conda/lib/python3.7/site-packages (from staintools) (4.5.4.60)\nRequirement already satisfied: pyparsing>=2.2.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (3.0.9)\nRequirement already satisfied: python-dateutil>=2.7 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (2.8.2)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (23.0)\nRequirement already satisfied: pillow>=6.2.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (9.4.0)\nRequirement already satisfied: cycler>=0.10 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (0.11.0)\nRequirement already satisfied: fonttools>=4.22.0 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (4.38.0)\nRequirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from matplotlib->staintools) (1.4.4)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from kiwisolver>=1.0.1->matplotlib->staintools) (4.4.0)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7->matplotlib->staintools) (1.16.0)\nBuilding wheels for collected packages: staintools\n  Building wheel for staintools (setup.py) ... \u001b[?25ldone\n\u001b[?25h  Created wheel for staintools: filename=staintools-2.1.2-py3-none-any.whl size=14072 sha256=87c15ee9f1e3d5842dc560aa3f56ad788f6863fbecd07d201b60874c836291b6\n  Stored in directory: /root/.cache/pip/wheels/04/f5/bb/ed0491391a02a485821a575c51ff95c9595678e2c230cab6e7\nSuccessfully built staintools\nInstalling collected packages: staintools\nSuccessfully installed staintools-2.1.2\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mRequirement already satisfied: albumentations in /opt/conda/lib/python3.7/site-packages (1.3.0)\nRequirement already satisfied: numpy>=1.11.1 in /opt/conda/lib/python3.7/site-packages (from albumentations) (1.21.6)\nRequirement already satisfied: scipy in /opt/conda/lib/python3.7/site-packages (from albumentations) (1.7.3)\nRequirement already satisfied: qudida>=0.0.4 in /opt/conda/lib/python3.7/site-packages (from albumentations) (0.0.4)\nRequirement already satisfied: PyYAML in /opt/conda/lib/python3.7/site-packages (from albumentations) (6.0)\nRequirement already satisfied: opencv-python-headless>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from albumentations) (4.5.4.60)\nRequirement already satisfied: scikit-image>=0.16.1 in /opt/conda/lib/python3.7/site-packages (from albumentations) (0.19.3)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from qudida>=0.0.4->albumentations) (4.4.0)\nRequirement already satisfied: scikit-learn>=0.19.1 in /opt/conda/lib/python3.7/site-packages (from qudida>=0.0.4->albumentations) (1.0.2)\nRequirement already satisfied: tifffile>=2019.7.26 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (2021.11.2)\nRequirement already satisfied: PyWavelets>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (1.3.0)\nRequirement already satisfied: networkx>=2.2 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (2.6.3)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (23.0)\nRequirement already satisfied: imageio>=2.4.1 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (2.25.0)\nRequirement already satisfied: pillow!=7.1.0,!=7.1.1,!=8.3.0,>=6.1.0 in /opt/conda/lib/python3.7/site-packages (from scikit-image>=0.16.1->albumentations) (9.4.0)\nRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn>=0.19.1->qudida>=0.0.4->albumentations) (1.2.0)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn>=0.19.1->qudida>=0.0.4->albumentations) (3.1.0)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0mRequirement already satisfied: pytorch-lightning in /opt/conda/lib/python3.7/site-packages (1.9.3)\nRequirement already satisfied: tqdm>=4.57.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (4.64.1)\nRequirement already satisfied: typing-extensions>=4.0.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (4.4.0)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (1.13.0+cpu)\nRequirement already satisfied: numpy>=1.17.2 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (1.21.6)\nRequirement already satisfied: fsspec[http]>2021.06.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (2023.1.0)\nRequirement already satisfied: PyYAML>=5.4 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (6.0)\nRequirement already satisfied: torchmetrics>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (0.11.1)\nRequirement already satisfied: packaging>=17.1 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (23.0)\nRequirement already satisfied: lightning-utilities>=0.6.0.post0 in /opt/conda/lib/python3.7/site-packages (from pytorch-lightning) (0.7.1)\nRequirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /opt/conda/lib/python3.7/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning) (3.8.3)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from fsspec[http]>2021.06.0->pytorch-lightning) (2.28.2)\nRequirement already satisfied: importlib-metadata>=4.0.0 in /opt/conda/lib/python3.7/site-packages (from lightning-utilities>=0.6.0.post0->pytorch-lightning) (4.11.4)\nRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (2.1.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (22.2.0)\nRequirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (4.0.2)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.8.2)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (1.3.3)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (6.0.4)\nRequirement already satisfied: asynctest==0.13.0 in /opt/conda/lib/python3.7/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch-lightning) (0.13.0)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.0.0->lightning-utilities>=0.6.0.post0->pytorch-lightning) (3.11.0)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (1.26.14)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (2022.12.7)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->fsspec[http]>2021.06.0->pytorch-lightning) (3.4)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"!pip install spams\n\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport openslide\nimport staintools\nimport albumentations as A\nfrom skimage.restoration import denoise_nl_means, estimate_sigma\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-03-24T09:27:42.997463Z","iopub.execute_input":"2023-03-24T09:27:42.997938Z","iopub.status.idle":"2023-03-24T09:27:57.851951Z","shell.execute_reply.started":"2023-03-24T09:27:42.997889Z","shell.execute_reply":"2023-03-24T09:27:57.850082Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"Requirement already satisfied: spams in /opt/conda/lib/python3.7/site-packages (2.6.5.4)\nRequirement already satisfied: scipy>=1.0 in /opt/conda/lib/python3.7/site-packages (from spams) (1.7.3)\nRequirement already satisfied: Pillow>=6.0 in /opt/conda/lib/python3.7/site-packages (from spams) (9.4.0)\nRequirement already satisfied: numpy>=1.12 in /opt/conda/lib/python3.7/site-packages (from spams) (1.21.6)\n\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n\u001b[0m","output_type":"stream"}]},{"cell_type":"code","source":"# Set the paths to the dataset\ndata_path = \"../input/prostate-cancer-grade-assessment/\"\ntrain_images_path = os.path.join(data_path, \"train_images\")\n\n# Load the labels\ntrain_labels = pd.read_csv(os.path.join(data_path, \"train.csv\"))","metadata":{"execution":{"iopub.status.busy":"2023-03-24T09:28:12.930267Z","iopub.execute_input":"2023-03-24T09:28:12.930792Z","iopub.status.idle":"2023-03-24T09:28:12.957544Z","shell.execute_reply.started":"2023-03-24T09:28:12.930734Z","shell.execute_reply":"2023-03-24T09:28:12.956041Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"def reinhard_color_normalization(img):\n    normalizer = staintools.ReinhardColorNormalizer()\n    normalizer.fit(img)\n    return normalizer.transform(img)\n\ndef stain_augmentation(img):\n    augmenter = staintools.StainAugmentor(method=\"random\", sigma1=0.2, sigma2=0.2)\n    augmenter.fit(img)\n    return augmenter.transform(img)\n\ndef nl_means_denoising(img):\n    sigma_estimated = estimate_sigma(img, multichannel=True)\n    denoised_img = denoise_nl_means(img, h=1.15 * sigma_estimated, fast_mode=True, patch_size=5, patch_distance=6, multichannel=True)\n    return denoised_img\n","metadata":{"execution":{"iopub.status.busy":"2023-03-24T09:28:21.321583Z","iopub.execute_input":"2023-03-24T09:28:21.322344Z","iopub.status.idle":"2023-03-24T09:28:21.331621Z","shell.execute_reply.started":"2023-03-24T09:28:21.322297Z","shell.execute_reply":"2023-03-24T09:28:21.32947Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"def sliding_window(image, patch_size=(256, 256), stride=128):\n    tiles = []\n    for y in range(0, image.shape[0], stride):\n        for x in range(0, image.shape[1], stride):\n            tile = image[y:y + patch_size[1], x:x + patch_size[0]]\n            if tile.shape[0] == patch_size[0] and tile.shape[1] == patch_size[1]:\n                tiles.append(tile)\n    return tiles\n\ndef augment_tile(tile):\n    aug = A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.Rotate(limit=90, p=0.5),\n        A.RandomResizedCrop(256, 256, scale=(0.9, 1.1), ratio=(0.9, 1.1), p=0.5)\n    ])\n    return aug(image=tile)['image']\n\ndef filter_tiles(tiles, tissue_threshold=0.1):\n    filtered_tiles = []\n    for tile in tiles:\n        gray_tile = cv2.cvtColor(tile, cv2.COLOR_BGR2GRAY)\n        _, binary_tile = cv2.threshold(gray_tile, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)\n        tissue_ratio = np.sum(binary_tile == 0) / (256 * 256)\n        if tissue_ratio > tissue_threshold:\n            filtered_tiles.append(tile)\n            \ndef save_tiles_to_disk(image_id, tiles, output_dir):\n    image_dir = os.path.join(output_dir, image_id)\n    os.makedirs(image_dir, exist_ok=True)\n    for i, tile in enumerate(tiles):\n        tile_path = os.path.join(image_dir, f\"{image_id}_{i}.png\")\n        cv2.imwrite(tile_path, tile)\n    return filtered_tiles","metadata":{"execution":{"iopub.status.busy":"2023-03-24T09:28:51.697546Z","iopub.execute_input":"2023-03-24T09:28:51.698686Z","iopub.status.idle":"2023-03-24T09:28:51.712391Z","shell.execute_reply.started":"2023-03-24T09:28:51.698632Z","shell.execute_reply":"2023-03-24T09:28:51.710775Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"def process_slide(slide_path, tissue_threshold=0.1):\n    slide = openslide.OpenSlide(slide_path)\n    img = np.array(slide.read_region((0, 0), 0, slide.level_dimensions[0]))[:, :, :3]\n\n    # Apply preprocessing\n    img_normalized = reinhard_color_normalization(img)\n    img_augmented = stain_augmentation(img_normalized)\n    img_denoised = nl_means_denoising(img_augmented)\n\n    # Extract tiles\n    tiles = sliding_window(img_denoised)\n    filtered_tiles = filter_tiles(tiles, tissue_threshold)\n\n    # Apply data augmentation to tiles\n    augmented_tiles = [augment_tile(tile) for tile in filtered_tiles]\n\n    return augmented_tiles","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_dir = \"preprocessed_tiles\"\nos.makedirs(output_dir, exist_ok=True)\n\nfor idx, row in tqdm(train_labels.iterrows(), total=len(train_labels)):\n    slide_path = os.path.join(train_images_path, f\"{row['image_id']}.tiff\")\n    tiles = process_slide(slide_path)\n    save_tiles_to_disk(row['image_id'], tiles, output_dir)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n\nwith open('preprocessed_data.pkl', 'wb') as f:\n    pickle.dump(preprocessed_data, f)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\n\nclass PANDADataset(Dataset):\n    def init(self, image_ids, labels, tiles_dir, transform=None):\n        self.image_ids = image_ids\n        self.labels = labels\n        self.tiles_dir = tiles_dir\n        self.transform = transform\n\ntransform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\ndataset = PANDADataset(preprocessed_data, transform=transform)\ntrain_size = int(0.8 * len(dataset))\nval_size = len(dataset) - train_size\ntrain_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False, num_workers=4)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pytorch_lightning as pl\nimport torch.nn as nn\nimport torch.optim as optim\n\nclass PANDAModel(pl.LightningModule):\n    def _init_(self):\n        super(PANDAModel, self).__init__()\n        self.model = nn.Sequential(\n        nn.Conv2d(3, 32, kernel_size=3, padding=1),\n        nn.ReLU(inplace=True),\n        nn.MaxPool2d(kernel_size=2, stride=2),\n        # ... Add more layers as needed ...\n        nn.AdaptiveAvgPool2d((1, 1)),\n        nn.Flatten(),\n        nn.Linear(32, 6)  # 6 classes for ISUP grades\n        )\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def forward(self, x):\n    return self.model(x)\n\ndef training_step(self, batch, batch_idx):\n    _, x, y = batch\n    logits = self(x)\n    loss = nn.CrossEntropyLoss()(logits, y)\n    self.log(\"train_loss\", loss)\n    return loss\n\ndef validation_step(self, batch, batch_idx):\n    _, x, y = batch\n    logits = self(x)\n    loss = nn.CrossEntropyLoss()(logits, y)\n    self.log(\"val_loss\", loss)\n\ndef configure_optimizers(self):\n    optimizer = optim.Adam(self.parameters(), lr=1e-3)\n    return optimizer\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = PANDAModel()\ntrainer = pl.Trainer(max_epochs=10, gpus=1) # Adjust max_epochs and gpus as needed\ntrainer.fit(model, train_loader, val_loader)","metadata":{},"execution_count":null,"outputs":[]}]}