{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\n# import glob\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torchvision\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.transforms as transforms\nimport pytorch_lightning as pl\n# import torch.nn.functional as F\n\nimport openslide\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport timm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-05T21:44:41.556419Z","iopub.execute_input":"2023-06-05T21:44:41.557064Z","iopub.status.idle":"2023-06-05T21:44:41.575369Z","shell.execute_reply.started":"2023-06-05T21:44:41.557007Z","shell.execute_reply":"2023-06-05T21:44:41.574045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create an empty submission.csv for Kaggle to recognise\nwith open('../working/submission.csv', 'w') as submis:\n    pass","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.577766Z","iopub.execute_input":"2023-06-05T21:44:41.578698Z","iopub.status.idle":"2023-06-05T21:44:41.588278Z","shell.execute_reply.started":"2023-06-05T21:44:41.578622Z","shell.execute_reply":"2023-06-05T21:44:41.587232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv')\ntest = pd.read_csv('../input/prostate-cancer-grade-assessment/test.csv')\nsample = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.589748Z","iopub.execute_input":"2023-06-05T21:44:41.590563Z","iopub.status.idle":"2023-06-05T21:44:41.621493Z","shell.execute_reply.started":"2023-06-05T21:44:41.590528Z","shell.execute_reply":"2023-06-05T21:44:41.620584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Tiler():\n    def __init__(self, N_tiles= 36, tile_size = 2**8, level=1):\n        self.N_tiles = N_tiles\n        self.tile_size = tile_size\n        self.level = level\n        \n    # This function takes an openslide object and returns the top left coordinates of N tiles (of a given size) with the most tissue pixels. \n    # Note: slide.level_dimensions[level] = (width,height).\n    # Note: padding is done to the right and bottom, this is to keep it simple while having at most 1 tile in memory at a time.\n    def _get_tile_locations_from_slide(self, slide):\n        tiles = []\n        required_padding = False\n        xlocs, ylocs = np.arange(0, slide.level_dimensions[self.level][0], self.tile_size), np.arange(0, slide.level_dimensions[self.level][1], self.tile_size) # Get the coordinates of the top left corners of the tiles.\n        for x_i, xloc in enumerate(xlocs):\n            for y_i, yloc in enumerate(ylocs):\n                region = np.copy(slide.read_region((xloc*(4**self.level),yloc*(4**self.level)), self.level, (self.tile_size,self.tile_size))) # The position is wrt. level 0, so must convert to level 0 coordinates by multiplying by the downsampling factor.\n                region_arr = np.asarray(region)[:,:,:3] # Ignore the alpha channel\n                if xloc+self.tile_size > slide.level_dimensions[self.level][0] or yloc+self.tile_size > slide.level_dimensions[self.level][1]: # if the tile goes out of bounds\n                    region_arr[region_arr==0] = 255\n                    required_padding = True\n                pixel_sum = region_arr.sum()\n                tiles.append({'xloc': xloc, 'yloc': yloc, 'pixel_sum': pixel_sum, 'required_padding': required_padding}) # store top left corner location and the tile's pixel_sum\n                required_padding = False\n        sorted_tiles = sorted(tiles, key= lambda d: d['pixel_sum']) # Sort tiles based on their pixel_sum field\n        sorted_tiles = sorted_tiles[:self.N_tiles] # Get top N tiles\n        return sorted_tiles\n    \n    # Return the tensor of individual tiles\n    def get_individual_tiles(self, slide, transform=None):\n        tiles_info = self._get_tile_locations_from_slide(slide)\n        tiles = torch.empty((self.N_tiles,3, self.tile_size, self.tile_size))\n        for i, tile in enumerate(tiles_info):\n            img = slide.read_region((tile['xloc']*(4**self.level),tile['yloc']*(4**self.level)), self.level, (self.tile_size,self.tile_size))\n            img = torch.clone(transforms.PILToTensor()(img)) # The position is wrt. level 0, so must convert to level 0 coordinates by multiplying by the downsampling factor.\n            img = img[:3,:,:] # Ignore the alpha channel\n            if tile['required_padding']:\n                img[img==0] = 255\n            if transforms: # SHOULD BE FASLE FOR TEST\n                img = img.float()/255.0 # Necessary for the transformations, images are expected to be a tensor with elements between [0,1]\n                img = transform(img)\n            tiles[i,...] = img\n        return tiles","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.623736Z","iopub.execute_input":"2023-06-05T21:44:41.624444Z","iopub.status.idle":"2023-06-05T21:44:41.641352Z","shell.execute_reply.started":"2023-06-05T21:44:41.62441Z","shell.execute_reply":"2023-06-05T21:44:41.640259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(pl.LightningModule):\n    def __init__(self, df, dir_name):\n        super().__init__()\n        self.df = df\n        self.img_dir = f'/kaggle/input/prostate-cancer-grade-assessment/{dir_name}/'\n        self.tiler = Tiler(N_tiles = 36, tile_size=2**8, level=1)\n\n        self.normalize = transforms.Normalize(\n                mean=[0.485, 0.456, 0.406],\n                std=[0.229, 0.224, 0.225])\n            \n    # Returns the individual tiles\n    def __getitem__(self, idx):\n            item_name = self.df.iloc[idx].loc['image_id'] # Get the name of the sample\n            file_path = os.path.join(self.img_dir, f'{item_name}.tiff')\n            slide = openslide.OpenSlide(file_path)\n            tiles = self.tiler.get_individual_tiles(slide, transform=self.normalize)\n            return tiles\n        \n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.643306Z","iopub.execute_input":"2023-06-05T21:44:41.643591Z","iopub.status.idle":"2023-06-05T21:44:41.65737Z","shell.execute_reply.started":"2023-06-05T21:44:41.643568Z","shell.execute_reply":"2023-06-05T21:44:41.656533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(model, dataloader, device):\n    model.eval()\n    model.to(device)\n    preds = []\n    sigm = nn.Sigmoid()\n    threshold = 0.5\n    for i, img in enumerate(dataloader):\n        img = img.to(device).float()\n        with torch.no_grad():\n            output = model(img)\n            output = sigm(output)\n            output = torch.where(output>threshold, 1, 0)\n            pred = torch.sum(output, axis=1)\n        preds.append(pred.to('cpu').numpy()) # add the predictions of this batch to the overall list\n    preds = np.concatenate(preds) # Make it a single list of predictions over all batches\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.689919Z","iopub.execute_input":"2023-06-05T21:44:41.691041Z","iopub.status.idle":"2023-06-05T21:44:41.706692Z","shell.execute_reply.started":"2023-06-05T21:44:41.690979Z","shell.execute_reply":"2023-06-05T21:44:41.70559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submit(model, sample, dir_name='test_images'):\n    if os.path.exists(f'../input/prostate-cancer-grade-assessment/{dir_name}'):\n        print('run inference')\n        test_dataset = TestDataset(sample, dir_name)\n        test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False)\n        preds = inference(model, test_loader, device)\n        sample['isup_grade'] = preds\n    return sample","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:44:41.709019Z","iopub.execute_input":"2023-06-05T21:44:41.709445Z","iopub.status.idle":"2023-06-05T21:44:41.716296Z","shell.execute_reply.started":"2023-06-05T21:44:41.709414Z","shell.execute_reply":"2023-06-05T21:44:41.715362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check using train_images\n# model = torch.jit.load('/kaggle/input/ismi-group3-panda-trained-models/convnext_pico_submission.pt')\n# submission = submit(model, train.head(), dir_name='train_images')\n# submission['isup_grade'] = submission['isup_grade'].astype(int)\n# submission.to_csv('submission.csv', index=False)\n# submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:46:17.528892Z","iopub.execute_input":"2023-06-05T21:46:17.529253Z","iopub.status.idle":"2023-06-05T21:46:30.507638Z","shell.execute_reply.started":"2023-06-05T21:46:17.529224Z","shell.execute_reply":"2023-06-05T21:46:30.506661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test submission\nmodel = torch.jit.load('/kaggle/input/ismi-group3-panda-trained-models/densenetblur121d_submission.pt')\nsubmission = submit(model, sample, dir_name='test_images')\nsubmission['isup_grade'] = submission['isup_grade'].astype(int)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T21:45:03.488358Z","iopub.execute_input":"2023-06-05T21:45:03.488782Z","iopub.status.idle":"2023-06-05T21:45:04.561679Z","shell.execute_reply.started":"2023-06-05T21:45:03.488749Z","shell.execute_reply":"2023-06-05T21:45:04.560743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}