{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Inference Kernel for SE_ResNeXT_50 Model\n\nI have trained 5 models using the **Concat Tile Pooling** approach as described in earlier notebooks on 5 folds of our 11,000 whole-slide images digitized H&E-stained biopsies. In this notebook, I describe the inference process and will submit the notebook to the competition.\n\nTraining Kernel can be found [here](https://www.kaggle.com/norrsken/detailed-implementation-of-se-resnext50/).\n\nTiles Kernel can be found [here](https://www.kaggle.com/norrsken/tiles/notebook)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"'''Imports'''\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport sys\nimport time\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport albumentations\nimport cv2\nimport openslide\nimport PIL\nimport skimage.io\nfrom sklearn import model_selection\nfrom tqdm.notebook import tqdm\n\n'''Download link for github repo: https://github.com/Cadene/pretrained-models.pytorch'''\npretrained_models = '/kaggle/input/pretrainedmodels/pretrained-models.pytorch-master/'\nsys.path.insert(0, pretrained_models)\nimport pretrainedmodels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMG_PATH = '../input/prostate-cancer-grade-assessment/train_images/'\nMSK_PATH = '../input/prostate-cancer-grade-assessment/train_label_masks/'\nSAVE_PATH = '/kaggle/Resized/'\nTEST_PATH = '../input/prostate-cancer-grade-assessment/test_images/'\n\nsz = 64 # Size of individual tile\nN = 49 # Number of Tiles\n\nTRAIN_BAT_SIZE = 32\nVALID_BAT_SIZE = 16\nDEVICE = 'cuda'\n\nMODEL_MEAN=(0.485, 0.456, 0.406)\nMODEL_STD=(0.229, 0.224, 0.225)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tile(img):\n    result = []\n    shape = img.shape\n    \n    '''Calculating the padding required in Horizontal and Vertical direction respectively'''\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    \n    '''np.pad is used to pad the image. the second positional argument represents number of rows/columns to be padded, ie.\n        [[top_pad, bottom_pad], [left_pad, right_pad],[front_pad, back_pad]], with value 255'''\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    \n    '''Reshaping the image into blocks of 128* 128\n    So, the img becomes 5 dimensional, and dimensions representthe following:\n    (num_vertical_blocks, height_each_block, num_horizontal_blocks, width_each_block, dimension)'''\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    \n    '''Reshaping them into (num_horizontal_blocks x num_vertical_blocks) blocks of 128x128 in 3 dimensions'''\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    \n    '''If the number of blocks is less than the 16 or N, then pad remaining blocks with value 255 (White pixels).'''\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n        \n    '''Now get top N blocks from the image which have the lowest sum (which means they have fewer white pixels and hence more information).'''\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    \n    '''Filter out the N blocks and then stack them into a single image of dimensions (512x512x3) with N tiles concatenated together.'''\n    img = img[idxs]\n    img = np.vstack((np.hstack((img[:7])),np.hstack((img[7:14])),np.hstack((img[14:21])),np.hstack((img[21:28])),np.hstack((img[28:35])),np.hstack((img[35:42])),np.hstack((img[42:49]))))\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Dataset\nclass PANDADatasetTest:\n    def __init__(self, mean, std):\n        df = pd.read_csv('../input/prostate-cancer-grade-assessment/test.csv')\n        self.image_ids = df.image_id.values\n\n        self.aug = albumentations.Compose([\n            albumentations.Normalize(mean, std, always_apply = True)\n            ])\n        \n    def __len__(self):\n        return len(self.image_ids)\n\n    def __getitem__(self, item):\n        img = skimage.io.MultiImage(os.path.join(TEST_PATH,self.image_ids[item]+'.tiff'))[-1] # Get the lowest resolution image from the MultiImage. You can use any of the three resolutions.\n        tiles = torch.Tensor(tile(img))\n        tiles = self.aug(image = np.array(tiles))['image']\n        tiles = torch.Tensor(tiles).permute(2, 0, 1)\n        return {\n            'image': tiles,\n            'image_id': self.image_ids[item]\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nCode borrowed from\nhttps://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py\n\"\"\"\nfrom __future__ import print_function, division, absolute_import\nfrom collections import OrderedDict\nimport math\n\nimport torch.nn as nn\nfrom torch.utils import model_zoo\n\n__all__ = ['SENet', 'senet154', 'se_resnet50', 'se_resnet101', 'se_resnet152',\n           'se_resnext50_32x4d', 'se_resnext101_32x4d']\n\npretrained_settings = {\n    'senet154': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet50': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet101': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnet152': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnext50_32x4d': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n    'se_resnext101_32x4d': {\n        'imagenet': {\n            'url': 'http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth',\n            'input_space': 'RGB',\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],\n            'num_classes': 1000\n        }\n    },\n}\n\n\nclass SEModule(nn.Module):\n\n    def __init__(self, channels, reduction):\n        super(SEModule, self).__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,\n                             padding=0)\n        self.relu = nn.ReLU(inplace=True)\n        self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,\n                             padding=0)\n        self.sigmoid = nn.Sigmoid()\n\n    def forward(self, x):\n        module_input = x\n        x = self.avg_pool(x)\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = self.fc2(x)\n        x = self.sigmoid(x)\n        return module_input * x\n\n\nclass Bottleneck(nn.Module):\n    \"\"\"\n    Base class for bottlenecks that implements `forward()` method.\n    \"\"\"\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out = self.se_module(out) + residual\n        out = self.relu(out)\n\n        return out\n\n\nclass SEBottleneck(Bottleneck):\n    \"\"\"\n    Bottleneck for SENet154.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes * 2)\n        self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,\n                               stride=stride, padding=1, groups=groups,\n                               bias=False)\n        self.bn2 = nn.BatchNorm2d(planes * 4)\n        self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,\n                               bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNetBottleneck(Bottleneck):\n    \"\"\"\n    ResNet bottleneck with a Squeeze-and-Excitation module. It follows Caffe\n    implementation and uses `stride=stride` in `conv1` and not in `conv2`\n    (the latter is used in the torchvision implementation of ResNet).\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None):\n        super(SEResNetBottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,\n                               stride=stride)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,\n                               groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SEResNeXtBottleneck(Bottleneck):\n    \"\"\"\n    ResNeXt bottleneck type C with a Squeeze-and-Excitation module.\n    \"\"\"\n    expansion = 4\n\n    def __init__(self, inplanes, planes, groups, reduction, stride=1,\n                 downsample=None, base_width=4):\n        super(SEResNeXtBottleneck, self).__init__()\n        width = math.floor(planes * (base_width / 64)) * groups\n        self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,\n                               stride=1)\n        self.bn1 = nn.BatchNorm2d(width)\n        self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,\n                               padding=1, groups=groups, bias=False)\n        self.bn2 = nn.BatchNorm2d(width)\n        self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.se_module = SEModule(planes * 4, reduction=reduction)\n        self.downsample = downsample\n        self.stride = stride\n\n\nclass SENet(nn.Module):\n\n    def __init__(self, block, layers, groups, reduction, dropout_p=0.2,\n                 inplanes=128, input_3x3=True, downsample_kernel_size=3,\n                 downsample_padding=1, num_classes=1000):\n        \"\"\"\n        Parameters\n        ----------\n        block (nn.Module): Bottleneck class.\n            - For SENet154: SEBottleneck\n            - For SE-ResNet models: SEResNetBottleneck\n            - For SE-ResNeXt models:  SEResNeXtBottleneck\n        layers (list of ints): Number of residual blocks for 4 layers of the\n            network (layer1...layer4).\n        groups (int): Number of groups for the 3x3 convolution in each\n            bottleneck block.\n            - For SENet154: 64\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models:  32\n        reduction (int): Reduction ratio for Squeeze-and-Excitation modules.\n            - For all models: 16\n        dropout_p (float or None): Drop probability for the Dropout layer.\n            If `None` the Dropout layer is not used.\n            - For SENet154: 0.2\n            - For SE-ResNet models: None\n            - For SE-ResNeXt models: None\n        inplanes (int):  Number of input channels for layer1.\n            - For SENet154: 128\n            - For SE-ResNet models: 64\n            - For SE-ResNeXt models: 64\n        input_3x3 (bool): If `True`, use three 3x3 convolutions instead of\n            a single 7x7 convolution in layer0.\n            - For SENet154: True\n            - For SE-ResNet models: False\n            - For SE-ResNeXt models: False\n        downsample_kernel_size (int): Kernel size for downsampling convolutions\n            in layer2, layer3 and layer4.\n            - For SENet154: 3\n            - For SE-ResNet models: 1\n            - For SE-ResNeXt models: 1\n        downsample_padding (int): Padding for downsampling convolutions in\n            layer2, layer3 and layer4.\n            - For SENet154: 1\n            - For SE-ResNet models: 0\n            - For SE-ResNeXt models: 0\n        num_classes (int): Number of outputs in `last_linear` layer.\n            - For all models: 1000\n        \"\"\"\n        super(SENet, self).__init__()\n        self.inplanes = inplanes\n        if input_3x3:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,\n                                    bias=False)),\n                ('bn1', nn.BatchNorm2d(64)),\n                ('relu1', nn.ReLU(inplace=True)),\n                ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn2', nn.BatchNorm2d(64)),\n                ('relu2', nn.ReLU(inplace=True)),\n                ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,\n                                    bias=False)),\n                ('bn3', nn.BatchNorm2d(inplanes)),\n                ('relu3', nn.ReLU(inplace=True)),\n            ]\n        else:\n            layer0_modules = [\n                ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,\n                                    padding=3, bias=False)),\n                ('bn1', nn.BatchNorm2d(inplanes)),\n                ('relu1', nn.ReLU(inplace=True)),\n            ]\n        # To preserve compatibility with Caffe weights `ceil_mode=True`\n        # is used instead of `padding=1`.\n        layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,\n                                                    ceil_mode=True)))\n        self.layer0 = nn.Sequential(OrderedDict(layer0_modules))\n        self.layer1 = self._make_layer(\n            block,\n            planes=64,\n            blocks=layers[0],\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=1,\n            downsample_padding=0\n        )\n        self.layer2 = self._make_layer(\n            block,\n            planes=128,\n            blocks=layers[1],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer3 = self._make_layer(\n            block,\n            planes=256,\n            blocks=layers[2],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        self.layer4 = self._make_layer(\n            block,\n            planes=512,\n            blocks=layers[3],\n            stride=2,\n            groups=groups,\n            reduction=reduction,\n            downsample_kernel_size=downsample_kernel_size,\n            downsample_padding=downsample_padding\n        )\n        # =====================> Changing this from AvgPool to AdaptiveAvgPool, so that all-dimensional inputs can be used. <=================================\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None\n        self.last_linear = nn.Linear(512 * block.expansion, num_classes)\n\n    def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,\n                    downsample_kernel_size=1, downsample_padding=0):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=downsample_kernel_size, stride=stride,\n                          padding=downsample_padding, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, groups, reduction, stride,\n                            downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes, groups, reduction))\n\n        return nn.Sequential(*layers)\n\n    def features(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n        return x\n\n    def logits(self, x):\n        x = self.avg_pool(x)\n        if self.dropout is not None:\n            x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.last_linear(x)\n        return x\n\n    def forward(self, x):\n        x = self.features(x)\n        x = self.logits(x)\n        return x\n\n\ndef initialize_pretrained_model(model, num_classes, settings):\n    assert num_classes == settings['num_classes'], \\\n        'num_classes should be {}, but is {}'.format(\n            settings['num_classes'], num_classes)\n    model.load_state_dict(model_zoo.load_url(settings['url']))\n    model.input_space = settings['input_space']\n    model.input_size = settings['input_size']\n    model.input_range = settings['input_range']\n    model.mean = settings['mean']\n    model.std = settings['std']\n\n\ndef senet154(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEBottleneck, [3, 8, 36, 3], groups=64, reduction=16,\n                  dropout_p=0.2, num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['senet154'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet50(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 4, 6, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet50'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet101(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 4, 23, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet101'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnet152(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNetBottleneck, [3, 8, 36, 3], groups=1, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnet152'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnext50_32x4d'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model\n\n\ndef se_resnext101_32x4d(num_classes=1000, pretrained='imagenet'):\n    model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,\n                  dropout_p=None, inplanes=64, input_3x3=False,\n                  downsample_kernel_size=1, downsample_padding=0,\n                  num_classes=num_classes)\n    if pretrained is not None:\n        settings = pretrained_settings['se_resnext101_32x4d'][pretrained]\n        initialize_pretrained_model(model, num_classes, settings)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''The same model architecture as used in training.'''\nclass Flatten(nn.Module):\n    def forward(self, input):\n        return input.view(input.size(0), -1)\n\nclass SEResNext50(nn.Module):\n    def __init__(self, h1=512, out = 1, d=0.5, freeze = False, pretrained = True):\n        super().__init__()\n        self.model = se_resnext50_32x4d(pretrained = None)\n        self.model = nn.Sequential(*list(self.model.children()))[:-2]\n        inp = 2048\n        if freeze:\n            for param in self.model.parameters():\n                param.requires_grad = False\n        self.ap = nn.AdaptiveAvgPool2d((1,1))\n        self.mp = nn.AdaptiveMaxPool2d((1,1))\n        self.fla = Flatten()\n        self.bn0 = nn.BatchNorm1d(2 * inp,eps=1e-05, momentum=0.1, affine=True)\n        self.dropout0 = nn.Dropout(d)\n        self.fc1 = nn.Linear(inp*2, h1)\n        self.bn1 = nn.BatchNorm1d(h1,eps=1e-05, momentum=0.1, affine=True)\n        self.dropout1 = nn.Dropout(d)\n        self.fc2 = nn.Linear(h1, out)\n        \n    def forward(self, x):\n        shape = x[0].shape\n        x = self.model(x)\n        ap = self.ap(x)\n        mp = self.mp(x)\n        x = torch.cat((ap,mp),dim=1)\n        x = self.fla(x)\n        x = self.bn0(x)\n        x = self.dropout0(x)\n        x = F.relu(self.fc1(x))\n        x = self.bn1(x)\n        x = self.dropout1(x)         \n        x = self.fc2(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Working of Optimized Rounder\nWe'll be giving equal weightage to the outputs of all the 5 models we have trained. We'll take the mean of all the predictions and then output the ISUP_Score using the result.<br>\n\nThe outputs of our models are continuous values and we need to convert them into one of the 6 ISUP Scores (ie. 0,1,2,3,4,5). For this conversion, we'll use optimized rounder. The optimized rounder works by creating ranges for each class and all the values lying in the certain range are given corresponding values.<br>\nComponents of Optimized Rounder:\n**coefficients**: These are the coefficients that help define the value range for each class.<br>\nfor eg: we might set initial coefficients as \\[0.5, 1.5, 2.5, 3.5, 4.5\\], what this means is:<br>\nRange     => Class<br>\n-inf - 0.5 => 0<br>\n0.5  - 1.5 => 1<br>\n1.5  - 2.5 => 2<br>\n... and so on.<br>\n\n**fit**: We use the fit() to optimize our coefficients, so that the ranges are better selected according to our data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')\n\n\nclass OptimizedRounder():\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n\n        ll = quadratic_weighted_kappa(y, X_p)\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(coefficients):\n    submission = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')\n    submission = submission[['image_id', 'isup_grade']]\n\n    dataset = PANDADatasetTest(\n        mean = MODEL_MEAN,\n        std = MODEL_STD\n    )\n    \n    test_dataloader = torch.utils.data.DataLoader(\n        dataset = dataset,\n        batch_size = 8,\n        shuffle = True,\n        num_workers = 4\n    )\n    \n    for bi, d in tqdm(enumerate(test_dataloader), total = int(len(dataset) / test_dataloader.batch_size)):\n        image = d['image']\n        img_id = d['image_id']\n        \n        image = image.to(DEVICE, dtype = torch.float)\n        isups = []\n        for x in range(5):\n            model = SEResNext50()\n            model.eval()\n            model.load_state_dict(torch.load(f'../input/se-resnext-50-training/fold{x}_se_resnext50.pth'))\n            model.to(DEVICE)\n            with torch.no_grad():\n                isup = model(image)\n            isups.append(isup.to('cpu').numpy())\n        isup = isup.to('cpu').numpy()\n        del model\n        isup = np.mean(isups, axis = 0)\n        optimized_rounder = OptimizedRounder()\n        preds = optimized_rounder.predict(isup, coefficients)\n        \n        for i, score in zip(img_id, preds):\n            submission.loc[submission['image_id'] == i, 'isup_grade'] = score\n    submission['isup_grade'] = submission['isup_grade'].astype(int)\n    submission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')\n'''Coefficients are selected from the training.'''\ncoefficients = [0.63941445, 1.54103949, 2.2556802, 3.06807197, 3.77971836]\n'''The test files are only visible while submission, so the below if statement will run only during the submission runs'''\nif os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):\n    predict(coefficients)\nelse:\n    submission.to_csv('submission.csv', index = False)\nprint('Done')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}