{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Fam inference","metadata":{}},{"cell_type":"code","source":"import collections\nimport math\nimport os\nimport re\nimport sys\nfrom collections import OrderedDict\nfrom functools import partial\n\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport scipy as sp\nimport skimage\nimport skimage.io\nimport torch\nfrom albumentations import Compose\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.metrics import cohen_kappa_score\nfrom torch import nn as nn\nfrom torch.autograd import Variable\nfrom torch.nn import functional as F\nfrom torch.nn.modules.batchnorm import _BatchNorm\nfrom torch.nn.parameter import Parameter\nfrom torch.utils import model_zoo\nfrom torch.utils.data import DataLoader, Dataset\nfrom tqdm import tqdm\n\nsys.path = [\n    '../input/ttach-kaggle/ttach/',\n] + sys.path\nimport ttach as tta","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:40.034411Z","iopub.execute_input":"2021-07-21T19:39:40.035024Z","iopub.status.idle":"2021-07-21T19:39:43.804764Z","shell.execute_reply.started":"2021-07-21T19:39:40.034962Z","shell.execute_reply":"2021-07-21T19:39:43.801809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocess\n############################################################################################\n\ndef get_tiles(img, tile_size, n_tiles, mode=0):\n    h, w, c = img.shape\n    pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n    pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n\n    img2 = np.pad(\n        img,\n        [[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2, pad_w - pad_w // 2], [0, 0]],\n        constant_values=255,\n    )\n    img3 = img2.reshape(\n        img2.shape[0] // tile_size, tile_size, img2.shape[1] // tile_size, tile_size, 3\n    )\n\n    img3 = img3.transpose(0, 2, 1, 3, 4).reshape(-1, tile_size, tile_size, 3)\n    n_tiles_with_info = (\n        img3.reshape(img3.shape[0], -1).sum(1) < tile_size ** 2 * 3 * 255\n    ).sum()\n    if len(img3) < n_tiles:\n        img3 = np.pad(\n            img3,\n            [[0, n_tiles - len(img3)], [0, 0], [0, 0], [0, 0]],\n            constant_values=255,\n        )\n    idxs = np.argsort(img3.reshape(img3.shape[0], -1).sum(-1))[:n_tiles]\n    img3 = img3[idxs]\n    return img3, n_tiles_with_info >= n_tiles\n\n\ndef concat_tiles(tiles, tile_shape):\n    image = cv2.hconcat([cv2.vconcat(tiles[ts]) for ts in tile_shape])\n    return image","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.805766Z","iopub.status.idle":"2021-07-21T19:39:43.806193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset\n###################################################################\n\nclass TestDataset(Dataset):\n    def __init__(\n        self,\n        df,\n        image_dir,\n        num_tile,\n        img_size,\n        tile_size_r,\n        tile_size_k,\n        tile_mode,\n        transform,\n    ):\n        self.df = df\n        self.img_ids = self.df.image_id\n        self.image_dir = image_dir\n        self.transform = transform\n        self.num_tiles = num_tile\n        self.img_size = img_size\n        self.tile_mode = tile_mode\n        self.tile_size_k = tile_size_k\n        self.tile_size_r = tile_size_r\n\n        self.is_karolinska = (self.df.data_provider == \"karolinska\").values\n        self.tiff_level = 1\n        n = int(self.num_tiles ** 0.5)\n        self.tile_shape = np.array(range(self.num_tiles)).reshape((n, n))\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_name = self.img_ids[idx]\n        file_path = os.path.join(self.image_dir, f\"{file_name}.tiff\")\n\n        # RGB\n        img_tmp = skimage.io.MultiImage(file_path)[self.tiff_level]\n        tiles, _ = get_tiles(\n            img_tmp,\n            tile_size=self.tile_size_k,\n            n_tiles=self.num_tiles,\n            mode=self.tile_mode,\n        )\n\n        tiles = np.array(tiles)\n        image = concat_tiles(tiles, tile_shape=self.tile_shape)\n        image = image.astype(np.float32)\n        image = 255 - image  # this task images has many whites(255)\n        image /= 255  # ToTensorV2 has no normalize !\n#         image = cv2.resize(image, (self.img_size, self.img_size))\n\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented[\"image\"]\n        return image","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.807492Z","iopub.status.idle":"2021-07-21T19:39:43.807976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# EfficientNet\n\n# Parameters for the entire model (stem, all blocks, and head)\nGlobalParams = collections.namedtuple(\n    \"GlobalParams\",\n    [\n        \"batch_norm_momentum\",\n        \"batch_norm_epsilon\",\n        \"dropout_rate\",\n        \"num_classes\",\n        \"width_coefficient\",\n        \"depth_coefficient\",\n        \"depth_divisor\",\n        \"min_depth\",\n        \"drop_connect_rate\",\n        \"image_size\",\n    ],\n)\n\n# Parameters for an individual model block\nBlockArgs = collections.namedtuple(\n    \"BlockArgs\",\n    [\n        \"kernel_size\",\n        \"num_repeat\",\n        \"input_filters\",\n        \"output_filters\",\n        \"expand_ratio\",\n        \"id_skip\",\n        \"stride\",\n        \"se_ratio\",\n    ],\n)\n\n# Change namedtuple defaults\nGlobalParams.__new__.__defaults__ = (None,) * len(GlobalParams._fields)\nBlockArgs.__new__.__defaults__ = (None,) * len(BlockArgs._fields)\n\n\nclass SwishImplementation(torch.autograd.Function):\n    @staticmethod\n    def forward(ctx, i):\n        result = i * torch.sigmoid(i)\n        ctx.save_for_backward(i)\n        return result\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        i = ctx.saved_variables[0]\n        sigmoid_i = torch.sigmoid(i)\n        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))\n\n\nclass MemoryEfficientSwish(nn.Module):\n    def forward(self, x):\n        return SwishImplementation.apply(x)\n\n\nclass Swish(nn.Module):\n    def forward(self, x):\n        return x * torch.sigmoid(x)\n\n\ndef round_filters(filters, global_params):\n    \"\"\" Calculate and round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.width_coefficient\n    if not multiplier:\n        return filters\n    divisor = global_params.depth_divisor\n    min_depth = global_params.min_depth\n    filters *= multiplier\n    min_depth = min_depth or divisor\n    new_filters = max(min_depth, int(filters + divisor / 2) // divisor * divisor)\n    if new_filters < 0.9 * filters:  # prevent rounding by more than 10%\n        new_filters += divisor\n    return int(new_filters)\n\n\ndef round_repeats(repeats, global_params):\n    \"\"\" Round number of filters based on depth multiplier. \"\"\"\n    multiplier = global_params.depth_coefficient\n    if not multiplier:\n        return repeats\n    return int(math.ceil(multiplier * repeats))\n\n\ndef drop_connect(inputs, p, training):\n    \"\"\" Drop connect. \"\"\"\n    if not training:\n        return inputs\n    batch_size = inputs.shape[0]\n    keep_prob = 1 - p\n    random_tensor = keep_prob\n    random_tensor += torch.rand(\n        [batch_size, 1, 1, 1], dtype=inputs.dtype, device=inputs.device\n    )\n    binary_tensor = torch.floor(random_tensor)\n    output = inputs / keep_prob * binary_tensor\n    return output\n\n\ndef get_same_padding_conv2d(image_size=None):\n    \"\"\" Chooses static padding if you have specified an image size, and dynamic padding otherwise.\n        Static padding is necessary for ONNX exporting of models. \"\"\"\n    if image_size is None:\n        return Conv2dDynamicSamePadding\n    else:\n        return partial(Conv2dStaticSamePadding, image_size=image_size)\n\n\nclass Conv2dDynamicSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a dynamic image size \"\"\"\n\n    def __init__(\n        self,\n        in_channels,\n        out_channels,\n        kernel_size,\n        stride=1,\n        dilation=1,\n        groups=1,\n        bias=True,\n    ):\n        super().__init__(\n            in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias\n        )\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n    def forward(self, x):\n        ih, iw = x.size()[-2:]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            x = F.pad(\n                x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]\n            )\n        return F.conv2d(\n            x,\n            self.weight,\n            self.bias,\n            self.stride,\n            self.padding,\n            self.dilation,\n            self.groups,\n        )\n\n\nclass Conv2dStaticSamePadding(nn.Conv2d):\n    \"\"\" 2D Convolutions like TensorFlow, for a fixed image size\"\"\"\n\n    def __init__(\n        self, in_channels, out_channels, kernel_size, image_size=None, **kwargs\n    ):\n        super().__init__(in_channels, out_channels, kernel_size, **kwargs)\n        self.stride = self.stride if len(self.stride) == 2 else [self.stride[0]] * 2\n\n        # Calculate padding based on image size and save it\n        assert image_size is not None\n        ih, iw = image_size if type(image_size) == list else [image_size, image_size]\n        kh, kw = self.weight.size()[-2:]\n        sh, sw = self.stride\n        oh, ow = math.ceil(ih / sh), math.ceil(iw / sw)\n        pad_h = max((oh - 1) * self.stride[0] + (kh - 1) * self.dilation[0] + 1 - ih, 0)\n        pad_w = max((ow - 1) * self.stride[1] + (kw - 1) * self.dilation[1] + 1 - iw, 0)\n        if pad_h > 0 or pad_w > 0:\n            self.static_padding = nn.ZeroPad2d(\n                (pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)\n            )\n        else:\n            self.static_padding = Identity()\n\n    def forward(self, x):\n        x = self.static_padding(x)\n        x = F.conv2d(\n            x,\n            self.weight,\n            self.bias,\n            self.stride,\n            self.padding,\n            self.dilation,\n            self.groups,\n        )\n        return x\n\n\nclass Identity(nn.Module):\n    def __init__(self,):\n        super(Identity, self).__init__()\n\n    def forward(self, input):\n        return input\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.809164Z","iopub.status.idle":"2021-07-21T19:39:43.809593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"########################################################################\n############## HELPERS FUNCTIONS FOR LOADING MODEL PARAMS ##############\n########################################################################\n\n\ndef efficientnet_params(model_name):\n    \"\"\" Map EfficientNet model name to parameter coefficients. \"\"\"\n    params_dict = {\n        # Coefficients:   width,depth,res,dropout\n        \"efficientnet-b0\": (1.0, 1.0, 224, 0.2),\n        \"efficientnet-b1\": (1.0, 1.1, 240, 0.2),\n        \"efficientnet-b2\": (1.1, 1.2, 260, 0.3),\n        \"efficientnet-b3\": (1.2, 1.4, 300, 0.3),\n        \"efficientnet-b4\": (1.4, 1.8, 380, 0.4),\n        \"efficientnet-b5\": (1.6, 2.2, 456, 0.4),\n        \"efficientnet-b6\": (1.8, 2.6, 528, 0.5),\n        \"efficientnet-b7\": (2.0, 3.1, 600, 0.5),\n        \"efficientnet-b8\": (2.2, 3.6, 672, 0.5),\n        \"efficientnet-l2\": (4.3, 5.3, 800, 0.5),\n    }\n    return params_dict[model_name]\n\n\nclass BlockDecoder(object):\n    \"\"\" Block Decoder for readability, straight from the official TensorFlow repository \"\"\"\n\n    @staticmethod\n    def _decode_block_string(block_string):\n        \"\"\" Gets a block through a string notation of arguments. \"\"\"\n        assert isinstance(block_string, str)\n\n        ops = block_string.split(\"_\")\n        options = {}\n        for op in ops:\n            splits = re.split(r\"(\\d.*)\", op)\n            if len(splits) >= 2:\n                key, value = splits[:2]\n                options[key] = value\n\n        # Check stride\n        assert (\"s\" in options and len(options[\"s\"]) == 1) or (\n            len(options[\"s\"]) == 2 and options[\"s\"][0] == options[\"s\"][1]\n        )\n\n        return BlockArgs(\n            kernel_size=int(options[\"k\"]),\n            num_repeat=int(options[\"r\"]),\n            input_filters=int(options[\"i\"]),\n            output_filters=int(options[\"o\"]),\n            expand_ratio=int(options[\"e\"]),\n            id_skip=(\"noskip\" not in block_string),\n            se_ratio=float(options[\"se\"]) if \"se\" in options else None,\n            stride=[int(options[\"s\"][0])],\n        )\n\n    @staticmethod\n    def _encode_block_string(block):\n        \"\"\"Encodes a block to a string.\"\"\"\n        args = [\n            \"r%d\" % block.num_repeat,\n            \"k%d\" % block.kernel_size,\n            \"s%d%d\" % (block.strides[0], block.strides[1]),\n            \"e%s\" % block.expand_ratio,\n            \"i%d\" % block.input_filters,\n            \"o%d\" % block.output_filters,\n        ]\n        if 0 < block.se_ratio <= 1:\n            args.append(\"se%s\" % block.se_ratio)\n        if block.id_skip is False:\n            args.append(\"noskip\")\n        return \"_\".join(args)\n\n    @staticmethod\n    def decode(string_list):\n        \"\"\"\n        Decodes a list of string notations to specify blocks inside the network.\n\n        :param string_list: a list of strings, each string is a notation of block\n        :return: a list of BlockArgs namedtuples of block args\n        \"\"\"\n        assert isinstance(string_list, list)\n        blocks_args = []\n        for block_string in string_list:\n            blocks_args.append(BlockDecoder._decode_block_string(block_string))\n        return blocks_args\n\n    @staticmethod\n    def encode(blocks_args):\n        \"\"\"\n        Encodes a list of BlockArgs to a list of strings.\n\n        :param blocks_args: a list of BlockArgs namedtuples of block args\n        :return: a list of strings, each string is a notation of block\n        \"\"\"\n        block_strings = []\n        for block in blocks_args:\n            block_strings.append(BlockDecoder._encode_block_string(block))\n        return block_strings\n\n\ndef efficientnet(\n    width_coefficient=None,\n    depth_coefficient=None,\n    dropout_rate=0.2,\n    drop_connect_rate=0.2,\n    image_size=None,\n    num_classes=1000,\n):\n    \"\"\" Creates a efficientnet model. \"\"\"\n\n    blocks_args = [\n        \"r1_k3_s11_e1_i32_o16_se0.25\",\n        \"r2_k3_s22_e6_i16_o24_se0.25\",\n        \"r2_k5_s22_e6_i24_o40_se0.25\",\n        \"r3_k3_s22_e6_i40_o80_se0.25\",\n        \"r3_k5_s11_e6_i80_o112_se0.25\",\n        \"r4_k5_s22_e6_i112_o192_se0.25\",\n        \"r1_k3_s11_e6_i192_o320_se0.25\",\n    ]\n    blocks_args = BlockDecoder.decode(blocks_args)\n\n    global_params = GlobalParams(\n        batch_norm_momentum=0.99,\n        batch_norm_epsilon=1e-3,\n        dropout_rate=dropout_rate,\n        drop_connect_rate=drop_connect_rate,\n        # data_format='channels_last',  # removed, this is always true in PyTorch\n        num_classes=num_classes,\n        width_coefficient=width_coefficient,\n        depth_coefficient=depth_coefficient,\n        depth_divisor=8,\n        min_depth=None,\n        image_size=image_size,\n    )\n\n    return blocks_args, global_params\n\n\ndef get_model_params(model_name, override_params):\n    \"\"\" Get the block args and global params for a given model \"\"\"\n    if model_name.startswith(\"efficientnet\"):\n        w, d, s, p = efficientnet_params(model_name)\n        # note: all models have drop connect rate = 0.2\n        blocks_args, global_params = efficientnet(\n            width_coefficient=w, depth_coefficient=d, dropout_rate=p, image_size=s\n        )\n    else:\n        raise NotImplementedError(\"model name is not pre-defined: %s\" % model_name)\n    if override_params:\n        # ValueError will be raised here if override_params has fields not included in global_params.\n        global_params = global_params._replace(**override_params)\n    return blocks_args, global_params\n\n\nurl_map = {\"no-url\"}\nurl_map_advprop = {\"no-url\"}\n\n\ndef load_pretrained_weights(model, model_name, load_fc=True, advprop=False):\n    \"\"\" Loads pretrained weights, and downloads if loading for the first time. \"\"\"\n    # AutoAugment or Advprop (different preprocessing)\n    url_map_ = url_map_advprop if advprop else url_map\n    state_dict = model_zoo.load_url(url_map_[model_name])\n    if load_fc:\n        model.load_state_dict(state_dict)\n    else:\n        state_dict.pop(\"_fc.weight\")\n        state_dict.pop(\"_fc.bias\")\n        res = model.load_state_dict(state_dict, strict=False)\n        assert set(res.missing_keys) == set(\n            [\"_fc.weight\", \"_fc.bias\"]\n        ), \"issue loading pretrained weights\"\n    print(\"Loaded pretrained weights for {}\".format(model_name))\n\n\nclass MBConvBlock(nn.Module):\n    \"\"\"\n    Mobile Inverted Residual Bottleneck Block\n\n    Args:\n        block_args (namedtuple): BlockArgs, see above\n        global_params (namedtuple): GlobalParam, see above\n\n    Attributes:\n        has_se (bool): Whether the block contains a Squeeze and Excitation layer.\n    \"\"\"\n\n    def __init__(self, block_args, global_params):\n        super().__init__()\n        self._block_args = block_args\n        self._bn_mom = 1 - global_params.batch_norm_momentum\n        self._bn_eps = global_params.batch_norm_epsilon\n        self.has_se = (self._block_args.se_ratio is not None) and (\n            0 < self._block_args.se_ratio <= 1\n        )\n        self.id_skip = block_args.id_skip  # skip connection and drop connect\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Expansion phase\n        inp = self._block_args.input_filters  # number of input channels\n        oup = (\n            self._block_args.input_filters * self._block_args.expand_ratio\n        )  # number of output channels\n        if self._block_args.expand_ratio != 1:\n            self._expand_conv = Conv2d(\n                in_channels=inp, out_channels=oup, kernel_size=1, bias=False\n            )\n            self._bn0 = nn.BatchNorm2d(\n                num_features=oup, momentum=self._bn_mom, eps=self._bn_eps\n            )\n\n        # Depthwise convolution phase\n        k = self._block_args.kernel_size\n        s = self._block_args.stride\n        self._depthwise_conv = Conv2d(\n            in_channels=oup,\n            out_channels=oup,\n            groups=oup,  # groups makes it depthwise\n            kernel_size=k,\n            stride=s,\n            bias=False,\n        )\n        self._bn1 = nn.BatchNorm2d(\n            num_features=oup, momentum=self._bn_mom, eps=self._bn_eps\n        )\n\n        # Squeeze and Excitation layer, if desired\n        if self.has_se:\n            num_squeezed_channels = max(\n                1, int(self._block_args.input_filters * self._block_args.se_ratio)\n            )\n            self._se_reduce = Conv2d(\n                in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1\n            )\n            self._se_expand = Conv2d(\n                in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1\n            )\n\n        # Output phase\n        final_oup = self._block_args.output_filters\n        self._project_conv = Conv2d(\n            in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False\n        )\n        self._bn2 = nn.BatchNorm2d(\n            num_features=final_oup, momentum=self._bn_mom, eps=self._bn_eps\n        )\n        self._swish = MemoryEfficientSwish()\n\n    def forward(self, inputs, drop_connect_rate=None):\n        \"\"\"\n        :param inputs: input tensor\n        :param drop_connect_rate: drop connect rate (float, between 0 and 1)\n        :return: output of block\n        \"\"\"\n\n        # Expansion and Depthwise Convolution\n        x = inputs\n        if self._block_args.expand_ratio != 1:\n            x = self._swish(self._bn0(self._expand_conv(inputs)))\n        x = self._swish(self._bn1(self._depthwise_conv(x)))\n\n        # Squeeze and Excitation\n        if self.has_se:\n            x_squeezed = F.adaptive_avg_pool2d(x, 1)\n            x_squeezed = self._se_expand(self._swish(self._se_reduce(x_squeezed)))\n            x = torch.sigmoid(x_squeezed) * x\n\n        x = self._bn2(self._project_conv(x))\n\n        # Skip connection and drop connect\n        input_filters, output_filters = (\n            self._block_args.input_filters,\n            self._block_args.output_filters,\n        )\n        if (\n            self.id_skip\n            and self._block_args.stride == 1\n            and input_filters == output_filters\n        ):\n            if drop_connect_rate:\n                x = drop_connect(x, p=drop_connect_rate, training=self.training)\n            x = x + inputs  # skip connection\n        return x\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.810957Z","iopub.status.idle":"2021-07-21T19:39:43.811395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EfficientNet(nn.Module):\n    \"\"\"\n    An EfficientNet model. Most easily loaded with the .from_name or .from_pretrained methods\n\n    Args:\n        blocks_args (list): A list of BlockArgs to construct blocks\n        global_params (namedtuple): A set of GlobalParams shared between blocks\n\n    Example:\n        model = EfficientNet.from_pretrained('efficientnet-b0')\n\n    \"\"\"\n\n    def __init__(self, blocks_args=None, global_params=None):\n        super().__init__()\n        assert isinstance(blocks_args, list), \"blocks_args should be a list\"\n        assert len(blocks_args) > 0, \"block args must be greater than 0\"\n        self._global_params = global_params\n        self._blocks_args = blocks_args\n\n        # Get static or dynamic convolution depending on image size\n        Conv2d = get_same_padding_conv2d(image_size=global_params.image_size)\n\n        # Batch norm parameters\n        bn_mom = 1 - self._global_params.batch_norm_momentum\n        bn_eps = self._global_params.batch_norm_epsilon\n\n        # Stem\n        in_channels = 3  # rgb\n        out_channels = round_filters(\n            32, self._global_params\n        )  # number of output channels\n        self._conv_stem = Conv2d(\n            in_channels, out_channels, kernel_size=3, stride=2, bias=False\n        )\n        self._bn0 = nn.BatchNorm2d(\n            num_features=out_channels, momentum=bn_mom, eps=bn_eps\n        )\n\n        # Build blocks\n        self._blocks = nn.ModuleList([])\n        for block_args in self._blocks_args:\n\n            # Update block input and output filters based on depth multiplier.\n            block_args = block_args._replace(\n                input_filters=round_filters(\n                    block_args.input_filters, self._global_params\n                ),\n                output_filters=round_filters(\n                    block_args.output_filters, self._global_params\n                ),\n                num_repeat=round_repeats(block_args.num_repeat, self._global_params),\n            )\n\n            # The first block needs to take care of stride and filter size increase.\n            self._blocks.append(MBConvBlock(block_args, self._global_params))\n            if block_args.num_repeat > 1:\n                block_args = block_args._replace(\n                    input_filters=block_args.output_filters, stride=1\n                )\n            for _ in range(block_args.num_repeat - 1):\n                self._blocks.append(MBConvBlock(block_args, self._global_params))\n\n        # Head\n        in_channels = block_args.output_filters  # output of final block\n        out_channels = round_filters(1280, self._global_params)\n        self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self._bn1 = nn.BatchNorm2d(\n            num_features=out_channels, momentum=bn_mom, eps=bn_eps\n        )\n\n        # Final linear layer\n        self._avg_pooling = nn.AdaptiveAvgPool2d(1)\n        self._dropout = nn.Dropout(self._global_params.dropout_rate)\n        self._fc = nn.Linear(out_channels, self._global_params.num_classes)\n        self._swish = MemoryEfficientSwish()\n\n    def set_swish(self, memory_efficient=True):\n        \"\"\"Sets swish function as memory efficient (for training) or standard (for export)\"\"\"\n        self._swish = MemoryEfficientSwish() if memory_efficient else Swish()\n        for block in self._blocks:\n            block.set_swish(memory_efficient)\n\n    def extract_features(self, inputs):\n        \"\"\" Returns output of the final convolution layer \"\"\"\n\n        # Stem\n        x = self._swish(self._bn0(self._conv_stem(inputs)))\n\n        # Blocks\n        for idx, block in enumerate(self._blocks):\n            drop_connect_rate = self._global_params.drop_connect_rate\n            if drop_connect_rate:\n                drop_connect_rate *= float(idx) / len(self._blocks)\n            x = block(x, drop_connect_rate=drop_connect_rate)\n\n        # Head\n        x = self._swish(self._bn1(self._conv_head(x)))\n\n        return x\n\n    def forward(self, inputs):\n        \"\"\" Calls extract_features to extract features, applies final linear layer, and returns logits. \"\"\"\n        bs = inputs.size(0)\n        # Convolution layers\n        x = self.extract_features(inputs)\n\n        # Pooling and final linear layer\n        x = self._avg_pooling(x)\n        x = x.view(bs, -1)\n        x = self._dropout(x)\n        x = self._fc(x)\n        return x\n\n    @classmethod\n    def from_name(cls, model_name, override_params=None):\n        cls._check_model_name_is_valid(model_name)\n        blocks_args, global_params = get_model_params(model_name, override_params)\n        return cls(blocks_args, global_params)\n\n    @classmethod\n    def from_pretrained(\n        cls, model_name, advprop=False, num_classes=1000, in_channels=3\n    ):\n        model = cls.from_name(model_name, override_params={\"num_classes\": num_classes})\n        load_pretrained_weights(\n            model, model_name, load_fc=(num_classes == 1000), advprop=advprop\n        )\n        if in_channels != 3:\n            Conv2d = get_same_padding_conv2d(image_size=model._global_params.image_size)\n            out_channels = round_filters(32, model._global_params)\n            model._conv_stem = Conv2d(\n                in_channels, out_channels, kernel_size=3, stride=2, bias=False\n            )\n        return model\n\n    @classmethod\n    def get_image_size(cls, model_name):\n        cls._check_model_name_is_valid(model_name)\n        _, _, res, _ = efficientnet_params(model_name)\n        return res\n\n    @classmethod\n    def _check_model_name_is_valid(cls, model_name):\n        \"\"\" Validates model name. \"\"\"\n        valid_models = [\"efficientnet-b\" + str(i) for i in range(9)]\n        if model_name not in valid_models:\n            raise ValueError(\"model_name should be one of: \" + \", \".join(valid_models))\n\n\nclass CustomEfficientNet(nn.Module):\n    def __init__(\n        self,\n        base=\"efficientnet-b0\",\n        pool_type=\"gem\",\n        in_ch=3,\n        out_ch=1,\n        pretrained=False,\n    ):\n        super(CustomEfficientNet, self).__init__()\n        assert base in {\n            \"efficientnet-b0\",\n            \"efficientnet-b1\",\n            \"efficientnet-b2\",\n            \"efficientnet-b3\",\n            \"efficientnet-b4\",\n        }\n        assert pool_type in {\"concat\", \"avg\", \"gem\"}\n\n        self.base = base\n        self.in_ch = in_ch\n        self.out_ch = out_ch\n        self.pretrained = pretrained\n\n        if pretrained:\n            self.net = EfficientNet.from_pretrained(base)\n        else:\n            self.net = EfficientNet.from_name(base)\n\n        out_shape = self.net._fc.in_features\n        if pool_type == \"concat\":\n            self.net._avg_pooling = AdaptiveConcatPool2d()\n            out_shape = out_shape * 2\n        elif pool_type == \"gem\":\n            self.net._avg_pooling = GeM()\n            out_shape = out_shape\n        self.net._fc = nn.Sequential(\n            Flatten(), SEBlock(out_shape), nn.Dropout(), nn.Linear(out_shape, out_ch)\n        )\n\n    def forward(self, x):\n        x = self.net(x)\n        return x\n\n\nclass AdaptiveConcatPool2d(nn.Module):\n    def __init__(self, sz=None):\n        super().__init__()\n        sz = sz or (1, 1)\n        self.ap = nn.AdaptiveAvgPool2d(sz)\n        self.mp = nn.AdaptiveMaxPool2d(sz)\n\n    def forward(self, x):\n        return torch.cat([self.mp(x), self.ap(x)], 1)\n\n\nclass SEBlock(nn.Module):\n    def __init__(self, in_ch, r=8):\n        super(SEBlock, self).__init__()\n\n        self.linear_1 = nn.Linear(in_ch, in_ch // r)\n        self.linear_2 = nn.Linear(in_ch // r, in_ch)\n\n    def forward(self, x):\n        input_x = x\n\n        x = F.relu(self.linear_1(x), inplace=True)\n        x = self.linear_2(x)\n        x = torch.sigmoid(x)\n\n        x = input_x * x\n\n        return x\n\n\nclass Flatten(nn.Module):\n    def forward(self, input):\n        return input.view(input.size(0), -1)\n\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = Parameter(torch.ones(1) * p)\n        self.eps = eps\n\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)\n\n    def __repr__(self):\n        return (\n            self.__class__.__name__\n            + f\"(p={self.p.data.tolist()[0]:.4f}, eps={str(self.eps)})\"\n        )\n\n\n######### Inference\n#################################################################\ndef inference(\n    dataloader, model, device, out_ch, tta=False, fp16=False, dataloader2=None,\n):\n    probs = []\n\n    def extract_preds(logits_tmp):\n        if out_ch == 1:\n            preds_tmp = logits_tmp\n        elif out_ch == 5:\n            preds_tmp = logits_tmp.sigmoid().sum(1)\n        elif out_ch > 5:\n            preds_tmp = logits_tmp[:, :5].sigmoid()\n            preds_tmp = preds_tmp.sum(1)\n        return preds_tmp.detach()\n\n    for i, images in tqdm(enumerate(dataloader), total=len(dataloader)):\n        if fp16:\n            images = images.half()\n        images = images.to(device)\n\n        with torch.no_grad():\n            logits = model(images)\n            preds = extract_preds(logits)\n\n            if tta:\n                logits_hflip = model(images.flip(3))\n                preds_hflip = extract_preds(logits_hflip)\n                preds = (preds + preds_hflip) / 2.0\n        probs.append(preds.to(\"cpu\").numpy())\n    probs = np.concatenate(probs)\n\n    if dataloader2 is not None:\n        probs2 = inference(\n            dataloader2, model, device, out_ch, tta, fp16=fp16, dataloader2=None\n        )\n        return (probs + probs2) / 2.0, probs\n    return probs\n\n\ndef quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights=\"quadratic\")\n\n\nclass MyOptimizedRounder:\n    def __init__(self):\n        self.coef = None\n        self.initial_coef = np.array([0.5, 1.5, 2.5, 3.5, 4.5])\n\n    def judge_each_thres(self, thres, x):\n        tmp = thres.reshape((1, -1)) <= x.reshape((-1, 1))\n        result = tmp.sum(axis=1)\n        return result\n\n    def _kappa_loss(self, coef, x, y):\n        x_p = self.judge_each_thres(coef, np.copy(x))\n        ll = quadratic_weighted_kappa(y, x_p)\n        return -ll\n\n    def fit(self, x, y):\n        loss_partial = partial(self._kappa_loss, x=x, y=y)\n        self.coef = sp.optimize.minimize(\n            loss_partial, self.initial_coef, method=\"nelder-mead\"\n        )\n\n    def predict(self, x, coef=None):\n        if coef is None:\n            coef = self.coef[\"x\"]\n        return self.judge_each_thres(coef, np.copy(x))\n\n    def predict_default_thres(self, x):\n        coef = self.initial_coef.copy()\n        return self.judge_each_thres(coef, np.copy(x))\n\n    def coefficients(self):\n        return self.coef[\"x\"]\n\n\n# main\n#################################################################\n\n\ndef print_eval_local(df, preds):\n    ground_truth = df.isup_grade.values\n    score = quadratic_weighted_kappa(ground_truth, preds)\n    print(f\"Local CV kappa score(all): {score}\")\n\n    indx_val_karolinska = df.data_provider == \"karolinska\"\n    indx_val_radboud = df.data_provider == \"radboud\"\n    indx = indx_val_karolinska\n\n    v_acc_k = (preds[indx] == ground_truth[indx]).mean() * 100.0\n    v_kappa_k = quadratic_weighted_kappa(ground_truth[indx], preds[indx])\n    print(f\"Local CV kappa score(karolinska): {v_kappa_k}\")\n    print(f\"Local CV accuracy(karolinska): {v_acc_k}\")\n    indx = indx_val_radboud\n    v_acc_r = (preds[indx] == ground_truth[indx]).mean() * 100.0\n    v_kappa_r = quadratic_weighted_kappa(ground_truth[indx], preds[indx])\n    print(f\"Local CV kappa score(radboud): {v_kappa_r}\")\n    print(f\"Local CV accuracy(radboud): {v_acc_r}\")\n","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.812561Z","iopub.status.idle":"2021-07-21T19:39:43.813045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    def __init__(self, on_kernel=True, kfold=1, debug=False):\n        self.on_kernel = on_kernel\n        self.num_tile = 64\n        self.tile_size_k = 192\n        self.tile_size_r = 192\n        self.img_size = 1536\n        self.fp16 = False\n        self.batch_size = 4\n        if self.fp16:\n            self.batch_size = int(self.batch_size * 2)\n        self.tta = False\n        self.ada_bn = False  # public lb 0.67 if True and fp16 is True\n\n        # Model config\n        self.in_ch = 3\n        self.out_ch = 10\n        self.base = \"efficientnet-b1\"\n\n        # pseudo label config\n        self.pseudo_use_threshold = 0.2\n        self.pseudo_soft_label = False\n\n        # weight config\n#         self.weight_name = \"097_efficientnet-b1_kfold_{}_latest.pt\"\n        self.weight_name = \"final_2_efficientnet-b1_kfold_{}_latest.pt\"\n        self.weight_name = self.weight_name.format(kfold)\n\n        if on_kernel:\n            # Image folder\n            self.data_dir = \"../input/prostate-cancer-grade-assessment\"\n            test_image_folder = os.path.join(self.data_dir, \"test_images\")\n            train_image_folder = os.path.join(self.data_dir, \"train_images\")\n            is_test = os.path.exists(test_image_folder)\n            self.image_folder = test_image_folder if is_test else train_image_folder\n\n            df_train = pd.read_csv(os.path.join(self.data_dir, \"train.csv\"))\n            sample = pd.read_csv(os.path.join(self.data_dir, \"test.csv\"))\n            self.df = sample if is_test else df_train.loc[:16]\n        else:\n            # Local PC\n            self.data_dir = \"../input\"\n            self.image_folder = \"../input/train_images\"\n            df = pd.read_csv(\"../input/train-5kfold.csv\")\n            self.kfold = 1\n            self.df = df[df[\"kfold\"] == self.kfold].reset_index(drop=True)\n            if debug:\n                self.df = self.df[:16].reset_index(drop=True)\n\n    def get_weight_path(self):\n        if self.on_kernel:\n            return os.path.join(\"../input/030-weight\", self.weight_name)\n        else:\n            dir_name = self.weight_name.split(\"_\")[0]\n            return os.path.join(\"../output/model\", dir_name, self.weight_name)\n\n    def get_args_ds(self):\n        return {\n            \"df\": self.df,\n            \"image_dir\": self.image_folder,\n            \"num_tile\": self.num_tile,\n            \"img_size\": self.img_size,\n            \"tile_size_k\": self.tile_size_k,\n            \"tile_size_r\": self.tile_size_r,\n            \"transform\": Compose([ToTensorV2()]),\n        }\n\n    def get_args_dl(self):\n        return {\"batch_size\": self.batch_size, \"shuffle\": False, \"num_workers\": 4}\n\n\ndef get_model(cfg_model):\n    net = None\n    if \"seresnext\" in cfg_model.base:\n        net = CustomSEResNeXt(\n            in_ch=cfg_model.in_ch,\n            out_ch=cfg_model.out_ch,\n            pool_type=\"gem\",\n            pretrained=False,\n        )\n    elif \"efficientnet\" in cfg_model.base:\n        net = CustomEfficientNet(\n            base=cfg_model.base,\n            in_ch=cfg_model.in_ch,\n            out_ch=cfg_model.out_ch,\n            pretrained=False,\n        )\n    assert net is not None\n    return net\n\n\ndef inference_from_cfg(cfg, model=None):\n    print(\"Make frame and path\")\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    # Make Dataloader\n    test_dataset = TestDataset(tile_mode=0, **cfg.get_args_ds())\n    test_loader = DataLoader(test_dataset, **cfg.get_args_dl())\n    test_dataset2 = TestDataset(tile_mode=2, **cfg.get_args_ds())\n    test_loader2 = DataLoader(test_dataset2, **cfg.get_args_dl())\n    print(\"Make data loader\")\n\n    if model is not None:\n        print(f\"Use existed Model: {cfg.base}\")\n    else:\n        print(f\"Get Model: {cfg.base}\")\n        model = get_model(cfg)\n        w_path = cfg.get_weight_path()\n        print(f\"Load weight...: {w_path}\")\n        state_dict = torch.load(w_path)[\"state_dict\"]\n        state_dict = {k[4:]: v for k, v in state_dict.items()}\n        model.load_state_dict(state_dict)\n\n    if cfg.ada_bn:\n        # Apply AdaBN\n        for m in model.modules():\n            if isinstance(m, _BatchNorm):\n                m.track_running_stats = False\n\n    if cfg.fp16:\n        model = model.half()\n    model.eval()\n\n    model = model.to(device)\n    \n    transforms = tta.Compose(\n        [tta.HorizontalFlip(), tta.VerticalFlip(), tta.Rotate90(angles=[0, 90, 180])]\n    )\n    model = tta.ClassificationTTAWrapper(model, transforms)\n\n    # Inference and round\n    print(\"Inference start\")\n    probs, probs_without_tta = inference(\n        test_loader,\n        model,\n        device,\n        out_ch=cfg.out_ch,\n        tta=cfg.tta,\n        fp16=cfg.fp16,\n        dataloader2=test_loader2,\n    )\n    return probs, probs_without_tta\n\n\ndef inference_mean_cfgs(cfgs):\n    probs_all = None\n    probs_without_tta_all = None\n    for c in cfgs:\n        probs, probs_without_tta = inference_from_cfg(c)\n        if probs_all is None:\n            probs_all = probs\n            probs_without_tta_all = probs_without_tta\n        else:\n            probs_all = probs_all + probs\n            probs_without_tta_all = probs_without_tta_all + probs_without_tta\n    probs = probs_all / float(len(cfgs))\n    probs_without_tta = probs_without_tta_all / float(len(cfgs))\n    return probs, probs_without_tta\n\n\ndef main(on_kernel=False, debug=False):\n    cfgs = [Config(on_kernel=on_kernel, kfold=k, debug=debug) for k in [1, 4, 5]]\n    probs_raw, probs_without_tta_raw = inference_mean_cfgs(cfgs)\n\n    print(\"Round predicts\")\n    rounder = MyOptimizedRounder()\n    preds = rounder.predict_default_thres(probs_raw)\n    preds_without_tta = rounder.predict_default_thres(probs_without_tta_raw)\n\n    # Save or eval on local\n    if on_kernel:\n        df = cfgs[0].df[[\"image_id\"]].copy()\n        df[\"probs_raw\"] = probs_raw\n        df[\"isup_grade\"] = preds\n        df[\"isup_grade\"] = df[\"isup_grade\"].astype(int)\n        return df\n    else:\n        print(f\"\\n*** Normal preds results ***\")\n        print_eval_local(cfgs[0].df, preds_without_tta)\n        print(f\"\\n*** Mode TTA preds results ***\")\n        print_eval_local(cfgs[0].df, preds)\n\n        df = cfgs[0].df\n        df[\"probs_raw\"] = probs_raw\n        df[\"probs_without_tta_raw\"] = probs_without_tta_raw\n        df[\"preds\"] = preds\n        df[\"preds_without_tta\"] = preds_without_tta\n        df.to_csv(f\"local_preds_{cfgs[0].weight_name.split('.')[0]}.csv\", index=False)\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-07-21T19:39:43.814042Z","iopub.status.idle":"2021-07-21T19:39:43.814473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp = main(on_kernel=True, debug=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.815332Z","iopub.status.idle":"2021-07-21T19:39:43.815781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## part 2","metadata":{}},{"cell_type":"code","source":"import os\nimport sys\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.816796Z","iopub.status.idle":"2021-07-21T19:39:43.817232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sys.path = [\n    '../input/ttach-kaggle/ttach/',\n] + sys.path\nimport ttach as tta","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.818112Z","iopub.status.idle":"2021-07-21T19:39:43.818559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import skimage.io\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport torchvision\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nfrom efficientnet_pytorch import model as enet\n\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.819446Z","iopub.status.idle":"2021-07-21T19:39:43.8199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment'\ndf_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))\ndf_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))\n\nmodel_dir = '../input/panda-public-models'\nimage_folder = os.path.join(data_dir, 'test_images')\nis_test = os.path.exists(image_folder)  # IF test_images is not exists, we will use some train images.\nimage_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')\n\ndf = df_test if is_test else df_train.loc[:16]\n\ntile_size = 256\nimage_size = 256\nn_tiles = 36\nbatch_size = 4\nnum_workers = 4\n\ndevice = torch.device('cuda')\n\nprint(image_folder)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.821108Z","iopub.status.idle":"2021-07-21T19:39:43.82154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class enetv3(nn.Module):\n    def __init__(self, modelname, out_dim=5, freeze_bn=True):\n        super(enetv3, self).__init__()\n        if \"resnet\" in modelname or \"resnext\" in modelname:\n            basemodel = resnet.__dict__[modelname](\n                                    pretrained=True)\n        if modelname == \"rx50\":\n            basemodel = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext50_32x4d_ssl')\n                            \n        if \"eff\" in modelname:\n            from efficientnet_pytorch import EfficientNet\n            self.basemodel = EfficientNet.from_name(modelname) \n            self.myfc = nn.Linear(self.basemodel._fc.in_features, out_dim)\n            self.basemodel._fc = nn.Identity()     \n            self.basemodel._avg_pooling = nn.AdaptiveAvgPool2d(1) #GeM() \n        else:\n            fcs = basemodel.fc.state_dict()[\"weight\"].shape[1]\n            self.basemodel = nn.Sequential(*list(basemodel.children())[:-2])\n            if poolmethod==\"maxpool\":\n                self.pool = nn.AdaptiveMaxPool2d(1)\n            else:\n                self.pool = nn.AdaptiveAvgPool2d(1)\n            self.myfc = nn.Linear(fcs, out_dim)\n            \n    def extract(self, x):\n        return self.basemodel(x)\n\n    def forward(self, x):\n        x = self.basemodel(x)\n        if not \"eff\" in modelname:\n            x = self.pool(x).squeeze(2).squeeze(2)\n            x = self.myfc(x)\n        else:\n            x = self.myfc(x)\n        return x\n    \ndef load_models(model_files):\n    models = []\n    for model_f in model_files:\n        #model_f = os.path.join(\"../input/panda-models\", model_f)\n        model_f = os.path.join(\"../input/latesubspanda\", model_f)\n        if \"res\" in model_f:\n            model = enetv2(out_dim=5)\n        elif \"eff\" in model_f:\n            model = enetv3(modelname, out_dim=5)\n        model.load_state_dict(torch.load(model_f))\n        model.eval()\n        model.to(device)\n        models.append(model)\n        print(f'{model_f} loaded!')\n    return models\n\nmodelname=\"efficientnet-b0\"\nmodel_files = [\n    'efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold0.pth',\n]\n\nmodel_files2 = [\n    'efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold0.pth',\n    \"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold1.pth\",\n    \"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold2.pth\",\n    \"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold3.pth\",\n    \"efficientnet-b0famlabelsmodelsub_avgpool_tile36_imsize256_mixup_final_epoch20_fold4.pth\"\n]\n\nmodels2 = load_models(model_files2)\nmodels = load_models(model_files)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.822573Z","iopub.status.idle":"2021-07-21T19:39:43.823025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_tiles(img, mode=0):\n        result = []\n        h, w, c = img.shape\n        pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n        pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n\n        img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)\n        img3 = img2.reshape(\n            img2.shape[0] // tile_size,\n            tile_size,\n            img2.shape[1] // tile_size,\n            tile_size,\n            3\n        )\n\n        img3 = img3.transpose(0,2,1,3,4).reshape(-1, tile_size, tile_size,3)\n        n_tiles_with_info = (img3.reshape(img3.shape[0],-1).sum(1) < tile_size ** 2 * 3 * 255).sum()\n        if len(img) < n_tiles:\n            img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)\n        idxs = np.argsort(img3.reshape(img3.shape[0],-1).sum(-1))[:n_tiles]\n        img3 = img3[idxs]\n        for i in range(len(img3)):\n            result.append({'img':img3[i], 'idx':i})\n        return result, n_tiles_with_info >= n_tiles\n\n\nclass PANDADataset(Dataset):\n    def __init__(self,\n                 df,\n                 image_size,\n                 n_tiles=n_tiles,\n                 tile_mode=0,\n                 rand=False,\n                 sub_imgs=False\n                ):\n\n        self.df = df.reset_index(drop=True)\n        self.image_size = image_size\n        self.n_tiles = n_tiles\n        self.tile_mode = tile_mode\n        self.rand = rand\n        self.sub_imgs = sub_imgs\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        img_id = row.image_id\n        \n        tiff_file = os.path.join(image_folder, f'{img_id}.tiff')\n        image = skimage.io.MultiImage(tiff_file)[1][:,:,::-1]\n        tiles, OK = get_tiles(image, self.tile_mode)\n\n        if self.rand:\n            idxes = np.random.choice(list(range(self.n_tiles)), self.n_tiles, replace=False)\n        else:\n            idxes = list(range(self.n_tiles))\n        idxes = np.asarray(idxes) + self.n_tiles if self.sub_imgs else idxes\n\n        n_row_tiles = int(np.sqrt(self.n_tiles))\n        images = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n        for h in range(n_row_tiles):\n            for w in range(n_row_tiles):\n                i = h * n_row_tiles + w\n    \n                if len(tiles) > idxes[i]:\n                    this_img = tiles[idxes[i]]['img']\n                else:\n                    this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n                this_img = 255 - this_img\n                h1 = h * image_size\n                w1 = w * image_size\n                images[h1:h1+image_size, w1:w1+image_size] = this_img\n        #images = 255 - images\n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n\n        return torch.tensor(images)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.823947Z","iopub.status.idle":"2021-07-21T19:39:43.824366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_test:\n    dataset_show = PANDADataset(df, image_size, n_tiles, 4)\n    from pylab import rcParams\n    rcParams['figure.figsize'] = 20,10\n    for i in range(2):\n        f, axarr = plt.subplots(1,5)\n        for p in range(5):\n            idx = np.random.randint(0, len(dataset_show))\n            img = dataset_show[idx]\n            axarr[p].imshow(1. - img.transpose(0, 1).transpose(1,2).squeeze())\n            axarr[p].set_title(str(idx))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.825268Z","iopub.status.idle":"2021-07-21T19:39:43.825728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = PANDADataset(df, image_size, n_tiles, 0)  # mode == 0\nloader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)\n\ndataset2 = PANDADataset(df, image_size, n_tiles, 2)  # mode == 2\nloader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.826823Z","iopub.status.idle":"2021-07-21T19:39:43.827281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ttach as tta\nif len(dataset) > 50:\n    transforms = tta.Compose(\n        [\n            tta.HorizontalFlip(),\n            tta.VerticalFlip(),\n            tta.Rotate90(angles=[0, 90, 180]),\n\n        ]\n    )\nelse:\n    transforms = tta.Compose(\n        [\n            #tta.HorizontalFlip(),\n        ]\n    )\ntta_models = []\ntta_models2 = []\nfor model in models:\n    tta_models.append(tta.ClassificationTTAWrapper(model, transforms))\nfor model in models2:\n    tta_models2.append(tta.ClassificationTTAWrapper(model, transforms))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.828195Z","iopub.status.idle":"2021-07-21T19:39:43.828634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LOGITS = []\nLOGITS2 = []\nLOGITS12 = []\nLOGITS22 = []\n\nwith torch.no_grad():\n    for data in tqdm(loader):\n        data = data.to(device)\n        for tta_model in tta_models:\n            logits = tta_model(data)\n        LOGITS.append(logits)\n        \n        for i, tta_model in enumerate(tta_models2):\n            if i == 0:\n                logits = tta_model(data).sigmoid()\n            else:\n                logits = logits + tta_model(data).sigmoid()\n        LOGITS2.append(logits/5)\n        \n    for data in tqdm(loader2):\n        data = data.to(device)\n        for tta_model in tta_models:\n            logits = tta_model(data)\n        LOGITS12.append(logits)\n        \n        for i, tta_model in enumerate(tta_models2):\n            if i == 0:\n                logits = tta_model(data).sigmoid()\n            else:\n                logits = logits + tta_model(data).sigmoid()\n        LOGITS22.append(logits/5)\n\n\nLOGITS = (torch.cat(LOGITS).sigmoid().cpu() + torch.cat(LOGITS2).cpu()\n         +torch.cat(LOGITS12).sigmoid().cpu() + torch.cat(LOGITS22).cpu()) / 4","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.829475Z","iopub.status.idle":"2021-07-21T19:39:43.82994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PREDS = LOGITS.sum(1).numpy()\ndf['isup_grade'] = PREDS\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.831027Z","iopub.status.idle":"2021-07-21T19:39:43.831455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp.rename({'probs_raw': 'part_1_pred'}, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.832503Z","iopub.status.idle":"2021-07-21T19:39:43.832962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.merge(df_tmp[['image_id', 'part_1_pred']], on = 'image_id', how = 'left')","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.833925Z","iopub.status.idle":"2021-07-21T19:39:43.834346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aru fam, 0.5:0.5\np_w = 0.5\ndf['isup_grade'] = df['isup_grade'] * p_w +  df['part_1_pred'] * (1-p_w)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.8352Z","iopub.status.idle":"2021-07-21T19:39:43.835776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['isup_grade'] = df['isup_grade'].apply(lambda x: int(np.round(x)))","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.836694Z","iopub.status.idle":"2021-07-21T19:39:43.837162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)\nprint(df.head())\nprint()\nprint(df.isup_grade.value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.83813Z","iopub.status.idle":"2021-07-21T19:39:43.838562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head(20)","metadata":{"execution":{"iopub.status.busy":"2021-07-21T19:39:43.839568Z","iopub.status.idle":"2021-07-21T19:39:43.840037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}