{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Score model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"| type |  model  | public kappa | local all kappa  | karolinska kappa | radboud kappa |  fold num | image size | num tiles | epoch | TTA |\n|:--------:|:--------:| :--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|\n| reg | tiles-eb0-netvlad | 0.88 | 0.8952 | 0.8976 | 0.8704 | 0 | 256 | 36 | 28 | 8 |\n| reg | tiles-eb0-netvlad | 0.89 | 0.886 | 0.8979 | 0.8464 | 1 | 256 | 36 | 22 | 8 |\n| reg | tiles-eb4-netvlad | **0.90** | 0.8826 | 0.9047 | 0.8335 | 1 | 256 | 36 | 26 | 8 |\n| reg | tiles-eb0-netvlad with attention model to select tiles | 0.88 | 0.8833 | 0.9034 | 0.8367 | 1 | 256 | 16 | 27 | 8 TTA only for score model|\n| reg | tiles-eb4-netvlad with attention model to select tiles | 0.89 | 0.8766 | 0.9035 | 0.8246 | 1 | 256 | 16 | 27 | 8 TTA only for score model|\n| reg | newcv tiles-eb4-netvlad with attention model to select tiles | 0.89 | 0.8812 | 0.8958 | 0.8437 | 1 | 256 | 16 | 27 | 8 TTA only for score model|\n| ord reg | newcv tiles-eb0-netvlad with attention model to select tiles | 0.88 | 0.8958 | 0.8972 | 0.8732 | 1 | 256 | 16 | 27 | 8 TTA only for score model|\n| reg  | newcv stitch-regnety-800m with attention model to select tiles | 0.893 | 0.8935 | 0.8872 | 0.8757 | 1 | 256 | 16 | 27 | 8 TTA only for score model |\n| bins | oldcv tiles-eb0 with attention model to select tiles | 0.901 | 0.8911 | 0.9078 | 0.8550 | 0 | 256 | 16 | 26 | 4 TTA|\n| bins | oldcv tiles-eb0 with attention model to select tiles | 0.897 | 0.8957 | 0.9067 | 0.8653 | 0 | 256 | 16 | 27 | 4 TTA|\n| bins | oldcv tiles-eb0 with attention model to select tiles | 0.880 | 0.9046 | 0.9038 | 0.8828 | 0 | 256 | 25 | 29 | 4 TTA|\n| bins | newcv tiles-eb0 with attention model to select tiles | 0.896 | 0.8975 | 0.9014 | 0.8739 | 0 | 256 | 16 | 29 | 4 TTA|\n| bins | newcv tiles-eb0 with attention model to select tiles | 0.901 | 0.8982 | 0.9013 | 0.8751 | 1 | 256 | 16 | 27 | 4 TTA|\n| bins | oldcv tiles-eb0 with attention model to select tiles | ??? | 0.9014 | ??? | ??? | 1 | 256 | 9 | 23 | 4 TTA|\n| bins | oldcv tiles-eb0 with attention model to select tiles | ??? | 0.9034 | ??? | ??? | 3 | 256 | 9 | 25 | 4 TTA|\n| bins | oldcv tiles-eb0 with attention model to select tiles | ??? | 0.9021 | ??? | ??? | 1 | 256 | 16 | 33 | 4 TTA|","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL_DICT = {\n    'reg':{\n        'model1':{'ckpt':f'../input/pandachallengemodel/tiles_eb0_netvlad_reg_36_256_best_kappa_f0.pth',\n                  'backbone':'efficientnet-b0','threshold':[0.5433,1.5323,2.4829,3.484,4.2806],'num_tiles':36},\n#         'model2':{'ckpt':f'../input/pandachallengemodel/tiles_eb0_netvlad_reg_36_256_best_kappa_f1.pth',\n#                   'backbone':'efficientnet-b0','threshold':[0.544,1.4831,2.46,3.466,4.2866],'num_tiles':36},\n#         'model3':{'ckpt':f'../input/pandachallengemodel/tiles_eb4_netvlad_reg_36_256_best_kappa_f1.pth',\n#                   'backbone':'efficientnet-b4','threshold':[0.5352,1.5212,2.5706,3.3613,4.2941],'num_tiles':36}\n    },\n    'reg_with_attention':{\n#         'model1':{'atten_ckpt':'../input/pandachallengemodel/tiles_eb0_attention_cls_64_256_f1.pth','atten_output':6,\n#                   'ckpt':'../input/pandachallengemodel/tiles_eb0_netvlad_attention_reg_16_256_f1.pth',\n#                   'backbone':'efficientnet-b0','threshold':[0.5206,1.5448,2.4244,3.4923,4.3578],'num_tiles':16},\n#         'model2':{'atten_ckpt':'../input/pandachallengemodel/tiles_eb0_attention_cls_64_256_f1.pth','atten_output':6,\n#                   'ckpt':'../input/pandachallengemodel/tiles_eb4_netvlad_attention_reg_16_256_f1.pth',\n#                   'backbone':'efficientnet-b4','threshold':[0.5319,1.5226,2.4316,3.4599,4.346],'num_tiles':16},\n        'model3':{'atten_ckpt':'../input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,\n                  'ckpt':'../input/pandachallengemodel/newcv_tiles_eb4_netvlad_attention_reg_16_256_f1.pth',\n                  'backbone':'efficientnet-b4','threshold':[0.5445,1.4968,2.4163,3.4254,4.3853],'num_tiles':16}\n    },\n    'ordinal_reg':{\n        'model1':{'atten_ckpt':'../input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,\n                  'ckpt':'../input/pandachallengemodel/bag_eb0_attention_ordinal_reg_16_256_f1.pth',\n                  'backbone':'efficientnet-b0','num_tiles':16},\n    },\n    'stitch_reg':{\n        'model1':{'atten_ckpt':'../input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth','atten_output':1,\n                 'ckpt':'../input/pandachallengemodel/stitched_regnety_800m_attention_reg_16_256_f1.pth',\n                  'threshold':[0.5124,1.5636,2.4927,3.4076,4.4082],'num_tiles':16}}\n    \n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ATTENTION_MEMORIES = {'../input/pandachallengemodel/newcv_tiles_eb0_attention_reg_64_256_f1.pth':None, \n                      '../input/pandachallengemodel/tiles_eb0_attention_cls_64_256_f1.pth':None}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pytorch-lightning/pytorch_lightning-0.8.5-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path = [\n    '../input/efficientnet-package/EfficientNet-PyTorch/',\n    '../input/efficienet062020/EfficientNet-PyTorch-master/',\n    '../input/regnet/'\n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from tqdm import tqdm_notebook as tqdm\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport skimage.io\nimport numpy as np\nimport pandas as pd\nimport torch \nimport torch.nn as nn\nfrom torch.utils.data import Dataset,DataLoader\nimport torch.nn.functional as F\nfrom torch.nn.parameter import Parameter\nimport torchvision \nfrom efficientnet_pytorch import model as enet  # I think we dont use the same here\nfrom scipy.stats import mode\nimport regnet\n# Also reimport your efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA = '../input/prostate-cancer-grade-assessment/'\nSAMPLE = '../input/prostate-cancer-grade-assessment/sample_submission.csv'\ndf_train = pd.read_csv(os.path.join(DATA,'train.csv'))\ndf_test = pd.read_csv(os.path.join(DATA,'test.csv'))\n\nimage_folder = os.path.join(DATA,'test_images')\nis_test = os.path.exists(image_folder)\nimage_folder = image_folder if is_test else os.path.join(DATA,'train_images')\ndf = df_test if is_test else df_train.loc[:5]\n\ntile_size = 256\nbatch_size = 2\nnum_tiles = 16\nnum_attention_tiles = 64\nnworkers = 4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Score Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.nn.init as init\nimport math\nfrom torch.autograd import Variable\n\n\nclass NetVLAD(nn.Module):\n    def __init__(self, feature_size, max_frames,cluster_size, add_bn=False, truncate=True):\n        super(NetVLAD, self).__init__()\n        self.feature_size = feature_size / 2 if truncate else feature_size\n        self.max_frames = max_frames\n        self.cluster_size = cluster_size\n        self.batch_norm = nn.BatchNorm1d(cluster_size, eps=1e-3, momentum=0.01)\n        self.linear = nn.Linear(self.feature_size, self.cluster_size)\n        self.softmax = nn.Softmax(dim=1)\n        self.cluster_weights2 = nn.Parameter(torch.FloatTensor(1, self.feature_size,\n                                                               self.cluster_size))\n        self.add_bn = add_bn\n        self.truncate = truncate\n        self.first = True\n        self.init_parameters()\n\n    def init_parameters(self):\n        init.normal(self.cluster_weights2, std=1 / math.sqrt(self.feature_size))\n\n    def forward(self, reshaped_input):\n        random_idx = torch.bernoulli(torch.Tensor([0.5]))\n        if self.truncate:\n            if self.training == True:\n                reshaped_input = reshaped_input[:, :self.feature_size].contiguous() if random_idx[0]==0 else reshaped_input[:, self.feature_size:].contiguous()\n            else:\n                if self.first == True:\n                    reshaped_input = reshaped_input[:, :self.feature_size].contiguous()\n                else:\n                    reshaped_input = reshaped_input[:, self.feature_size:].contiguous()\n        activation = self.linear(reshaped_input)\n        if self.add_bn:\n            activation = self.batch_norm(activation)\n        activation = self.softmax(activation).view([-1, self.max_frames, self.cluster_size])\n        a_sum = activation.sum(-2).unsqueeze(1)\n        a = torch.mul(a_sum, self.cluster_weights2)\n        activation = activation.permute(0, 2, 1).contiguous()\n        reshaped_input = reshaped_input.view([-1, self.max_frames, self.feature_size])\n        vlad = torch.matmul(activation, reshaped_input).permute(0, 2, 1).contiguous()\n        vlad = vlad.sub(a).view([-1, self.cluster_size * self.feature_size])\n        if self.training == False:\n            self.first = 1 - self.first\n        return vlad\n    \n\nclass Model(nn.Module):\n    def __init__(self, num_clusters,num_tiles,num_classes=1,arch='efficientnet-b0'):\n        super().__init__()\n        self.base = enet.EfficientNet.from_name(arch)\n        self.nc = self.base._fc.in_features\n        self.tile = nn.Sequential(\n            nn.BatchNorm2d(self.nc,eps=0.001,momentum=0.01),\n            nn.AdaptiveAvgPool2d(output_size=1),\n            nn.Flatten(),\n            nn.Dropout(p=0.2),\n        )    \n        self.netvlad = NetVLAD(cluster_size=num_clusters,max_frames=num_tiles,\n                    feature_size=self.nc,truncate=False)\n        self.fc = nn.Linear(num_clusters*self.nc,num_classes)\n    \n    def forward(self, x):\n        \"\"\"\n        Args:\n            x (batch,N,3,h,w):\n        \"\"\"\n        batch = x.shape[0]\n        shape = x[0].shape\n        n = shape[0]\n        x = x.reshape(-1,shape[1],shape[2],shape[3]) #x: bs*num_tiles x 3 x H x W\n        x = self.base.extract_features(x) #x: bs*num_tiles x nc\n        x = self.tile(x)\n        x = x.view(batch,n,self.nc)\n        x = self.netvlad(x)\n        x = self.fc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Attention Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdaptiveConcatPool2d(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))\n\n    def forward(self, x):\n        avg_x = self.avg(x)\n        max_x = self.max(x)\n        return torch.cat([avg_x, max_x], dim=1).squeeze(2).squeeze(2)\n\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.shape[0], -1)\n\n\nclass EfficientModel(nn.Module):\n\n    def __init__(self, c_out=6, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):\n        super().__init__()\n\n        from efficientnet_pytorch import EfficientNet\n        m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)\n        c_feature = m._fc.in_features\n        m._fc = nn.Identity()\n        self.feature_extractor = m\n        self.n_tiles = n_tiles\n        self.tile_size = tile_size\n        \n        if strategy == 'stitched':\n            if head == 'basic':\n                self.head = nn.Linear(c_feature, c_out)\n            elif head == 'concat':\n                m._avg_pooling = AdaptiveConcatPool2d()\n                self.head = nn.Linear(c_feature * 2, c_out)\n            elif head == 'gem':\n                m._avg_pooling = GeM()\n                self.head = nn.Linear(c_feature, c_out)\n        elif strategy == 'bag':\n            if head == 'basic':\n                self.head = BasicHead(c_feature, c_out, n_tiles)\n            elif head == 'attention':\n                self.head = AttentionHead(c_feature, c_out, n_tiles)\n                \n        self.strategy = strategy\n\n    def forward(self, x):\n        if self.strategy == 'bag':\n            x = x.view(-1, 3, self.tile_size, self.tile_size)\n        h = self.feature_extractor(x)\n        h = self.head(h)\n        return h\n\n\nclass BasicHead(nn.Module):\n\n    def __init__(self, c_in, c_out, n_tiles):\n        self.n_tiles = n_tiles\n        super().__init__()\n        self.fc = nn.Sequential(AdaptiveConcatPool2d(),\n                                nn.Linear(c_in * 2, c_out))\n\n    def forward(self, x):\n\n        bn, c = x.shape\n        h = x.view(-1, self.n_tiles, c, 1, 1).permute(0, 2, 1, 3, 4) \\\n            .contiguous().view(-1, c, 1 * self.n_tiles, 1)\n        h = self.fc(h)\n        return h\n\n\nclass AttentionHead(nn.Module):\n\n    def __init__(self, c_in, c_out, n_tiles):\n        self.n_tiles = n_tiles\n        super().__init__()\n        self.attention_pool = AttentionPool(c_in, c_in//2)\n        self.fc = nn.Linear(c_in, c_out)\n\n    def forward(self, x):\n\n        bn, c = x.shape\n        h = x.view(-1, self.n_tiles, c)\n        h = self.attention_pool(h)\n        h = self.fc(h)\n        return h\n\n\nclass AttentionPool(nn.Module):\n\n    def __init__(self, c_in, d):\n        super().__init__()\n        self.lin_V = nn.Linear(c_in, d)\n        self.lin_w = nn.Linear(d, 1)\n\n    def compute_weights(self, x):\n        key = self.lin_V(x)  # b, n, d\n        weights = self.lin_w(torch.tanh(key))  # b, n, 1\n        weights = torch.softmax(weights, dim=1)\n        return weights\n\n    def forward(self, x):\n        weights = self.compute_weights(x)\n        pooled = torch.matmul(x.transpose(1, 2), weights).squeeze(2)   # b, c, n x b, n, 1 => b, c, 1\n        return pooled","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### threshold rounder model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import scipy\nclass OptimizedRounder():\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n\n        ll = quadratic_weighted_kappa(y, X_p)\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]\n        self.coef_ = scipy.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']\n\noptimized_rounder = OptimizedRounder()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataLoader","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(img,sz,num_tiles):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:num_tiles]\n    img = img[idxs]\n    return img\n\n\n\nclass PandaDataset(Dataset):\n    def __init__(self,path,test_df,num_tiles,attention_model=None):\n        self.path = path \n        self.names = list(test_df.image_id)\n        self.num_tiles = num_tiles\n        self.attention_model = attention_model\n        self.attention_memory = {}\n    \n    def __len__(self):\n        return len(self.names)\n    \n    def __getitem__(self,idx):\n        name = self.names[idx]\n        image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff'))[1]\n        tiles = preprocess(image,tile_size,num_attention_tiles)\n        if self.attention_model:\n            images = np.zeros((num_attention_tiles,3,tile_size,tile_size),np.float32)\n        else:\n            tiles = self.get_sum_tiles(tiles)\n            images = np.zeros((self.num_tiles,3,tile_size,tile_size),np.float32)\n        for i,tile in enumerate(tiles):\n            tile = tile.astype(np.float32)\n            tile /=255. \n            tile = tile.transpose(2,0,1)\n            images[i,:,:,:] = tile \n        if self.attention_model:\n            images = self.get_best_tiles(images, name)\n        return torch.tensor(images).float(),name\n        \n    def get_sum_tiles(self,tiles):\n        idxs = np.argsort(tiles.reshape(tiles.shape[0],-1).sum(-1))[:self.num_tiles]\n        tiles = tiles[idxs]\n        return tiles \n    \n    def get_best_tiles(self,tiles,name):\n        \n        if name in self.attention_memory:\n            weights = self.attention_memory[name]\n        else:\n            tile_bags = torch.tensor(tiles.copy())\n        \n            with torch.no_grad():\n                features = self.attention_model.feature_extractor(tile_bags.cuda())\n                features = features.view(-1, num_attention_tiles, features.shape[-1])\n                weights = self.attention_model.head.attention_pool.compute_weights(features)\n            weights = weights.reshape(-1).argsort(0).cpu().numpy()[::-1]\n            self.attention_memory[name] = weights\n            \n        best_tiles = [tiles[w] for w in weights[:num_tiles]] \n        return best_tiles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PREDS = []","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Regression Model Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load regression models\nMODELS = []\nTHRES = []\nfor k in MODEL_DICT['reg']:\n    ckpt = MODEL_DICT['reg'][k]['ckpt']\n    n_tiles = MODEL_DICT['reg'][k]['num_tiles']\n    backbone = MODEL_DICT['reg'][k]['backbone']\n    threshold = MODEL_DICT['reg'][k]['threshold']\n    state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)\n    model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)\n    model.load_state_dict(state_dict,strict=True)\n    model.float()\n    model.eval()\n    model.cuda()\n    del state_dict\n    MODELS.append(model)\n    THRES.append(threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(SAMPLE)\n\nif os.path.exists(DATA):\n    ds = PandaDataset(image_folder,df,36,attention_model=None)\n    for model,threshold in zip(MODELS,THRES):\n        preds = []\n        names = []\n        with torch.no_grad():\n            for x,y in tqdm(ds):\n                x = x.cuda()\n                #dihedral TTA\n                x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n                  x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n                  x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n    #             x = x.view(-1,num_tiles,3,tile_size,tile_size)\n                p = model(x)\n                p = p.view(-1).mean().cpu()\n                names.append(y)\n                preds.append(p)\n        final_preds = torch.tensor(preds).numpy()\n        final_preds = optimized_rounder.predict(final_preds,threshold)\n        final_preds = final_preds.astype(int)\n        \n        PREDS.append(final_preds)\n        NAMES = names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model in MODELS:\n    del model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Regression with Attention","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load Attention Models\nATTENTION_MODELS = []\nATTENTION_MODELS_NAMES = []\nfor k in MODEL_DICT['reg_with_attention']:\n    atten_ckpt = MODEL_DICT['reg_with_attention'][k]['atten_ckpt']\n    atten_output = MODEL_DICT['reg_with_attention'][k]['atten_output']\n    state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)\n    attention_model = EfficientModel(c_out=atten_output,\n                                     n_tiles=num_attention_tiles, \n                                     tile_size=tile_size, \n                                     name='efficientnet-b0', \n                                     strategy='bag', \n                                     head='attention')\n    attention_model.load_state_dict(state_dict,strict=True)\n    attention_model.float()\n    attention_model.eval()\n    attention_model.cuda()\n    del state_dict\n    ATTENTION_MODELS.append(attention_model)\n    ATTENTION_MODELS_NAMES.append(atten_ckpt)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load regression models\nMODELS = []\nTHRES = []\nfor k in MODEL_DICT['reg_with_attention']:\n    ckpt = MODEL_DICT['reg_with_attention'][k]['ckpt']\n    n_tiles = MODEL_DICT['reg_with_attention'][k]['num_tiles']\n    backbone = MODEL_DICT['reg_with_attention'][k]['backbone']\n    threshold = MODEL_DICT['reg_with_attention'][k]['threshold']\n    state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)\n    model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=1,arch=backbone)\n    model.load_state_dict(state_dict,strict=True)\n    model.float()\n    model.eval()\n    model.cuda()\n    del state_dict\n    MODELS.append(model)\n    THRES.append(threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(SAMPLE)\n\nif os.path.exists(DATA): \n    for model, threshold, atten_model, atten_model_name in zip(MODELS, THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):\n        ds = PandaDataset(image_folder,df,16,attention_model=atten_model)\n        if ATTENTION_MEMORIES[atten_model_name] is not None:\n            ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]\n            \n        preds = []\n        names = []\n        with torch.no_grad():\n            for x,y in tqdm(ds):\n                x = x.cuda()\n                #dihedral TTA\n                x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n                  x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n                  x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n    #             x = x.view(-1,num_tiles,3,tile_size,tile_size)\n                p = model(x)\n                p = p.view(-1).mean().cpu()\n                names.append(y)\n                preds.append(p)\n        final_preds = torch.tensor(preds).numpy()\n        final_preds = optimized_rounder.predict(final_preds,threshold)\n        final_preds = final_preds.astype(int)\n        \n        PREDS.append(final_preds)\n        NAMES = names\n        \n        ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model in ATTENTION_MODELS:\n    del model\nfor model in MODELS:\n    del model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Oridnal Regression Inference","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"atten_ckpt = MODEL_DICT['ordinal_reg']['model1']['atten_ckpt']\natten_output = MODEL_DICT['ordinal_reg']['model1']['atten_output']\nstate_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)\nattention_model = EfficientModel(c_out=atten_output,\n                                 n_tiles=num_attention_tiles, \n                                 tile_size=tile_size, \n                                 name='efficientnet-b0', \n                                 strategy='bag', \n                                 head='attention')\nattention_model.load_state_dict(state_dict,strict=True)\nattention_model.float()\nattention_model.eval()\nattention_model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load regression models\nMODELS = []\nfor k in MODEL_DICT['ordinal_reg']:\n    ckpt = MODEL_DICT['ordinal_reg'][k]['ckpt']\n    n_tiles = MODEL_DICT['ordinal_reg'][k]['num_tiles']\n    backbone = MODEL_DICT['ordinal_reg'][k]['backbone']\n    state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)\n    model = Model(num_clusters=6,num_tiles=n_tiles,num_classes=5,arch=backbone)\n    model.load_state_dict(state_dict,strict=True)\n    model.float()\n    model.eval()\n    model.cuda()\n    del state_dict\n    MODELS.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(SAMPLE)\natten_model_name = atten_ckpt\nif os.path.exists(DATA):\n    ds = PandaDataset(image_folder,df,16,attention_model)\n    \n    if ATTENTION_MEMORIES[atten_model_name] is not None:\n        ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]\n    \n    for model in MODELS:\n        names,preds = [],[]\n        with torch.no_grad():\n            for x,y in tqdm(ds):\n                x = x.cuda()\n                #dihedral TTA\n                x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n                  x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n                  x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n    #             x = x.view(-1,num_tiles,3,tile_size,tile_size)\n                p = model(x)\n                p = torch.sigmoid(p)\n                p = p.view(8,5).mean(0).sum().round().cpu()\n                names.append(y)\n                preds.append(p)\n        final_preds = torch.tensor(preds).numpy()\n        final_preds = final_preds.astype(int)\n        \n        PREDS.append(final_preds)\n        NAMES = names\n    \n    ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stitch model , regession with attention","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load Attention Models\nATTENTION_MODELS = []\nATTENTION_MODELS_NAMES = []\nfor k in MODEL_DICT['stitch_reg']:\n    atten_ckpt = MODEL_DICT['stitch_reg'][k]['atten_ckpt']\n    atten_output = MODEL_DICT['stitch_reg'][k]['atten_output']\n    state_dict = torch.load(atten_ckpt,map_location=lambda storage, loc: storage)\n    attention_model = EfficientModel(c_out=atten_output,\n                                     n_tiles=num_attention_tiles, \n                                     tile_size=tile_size, \n                                     name='efficientnet-b0', \n                                     strategy='bag', \n                                     head='attention')\n    attention_model.load_state_dict(state_dict,strict=True)\n    attention_model.float()\n    attention_model.eval()\n    attention_model.cuda()\n    del state_dict\n    ATTENTION_MODELS.append(attention_model)\n    ATTENTION_MODELS_NAMES.append(atten_ckpt)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load regression models\nMODELS = []\nfor k in MODEL_DICT['stitch_reg']:\n    ckpt = MODEL_DICT['stitch_reg'][k]['ckpt']\n    n_tiles = MODEL_DICT['stitch_reg'][k]['num_tiles']\n    state_dict = torch.load(ckpt,map_location=lambda storage, loc: storage)\n    model = regnet.Regnet(1)\n    model.load_state_dict(state_dict,strict=True)\n    model.float()\n    model.eval()\n    model.cuda()\n    del state_dict\n    MODELS.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(img,sz,num_tiles):\n    shape = img.shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < num_tiles:\n        img = np.pad(img,[[0,num_tiles-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:num_tiles]\n    img = img[idxs]\n    return img\n\n\n\nclass StitchPandaDataset(Dataset):\n    def __init__(self,path,test_df,num_tiles,attention_model):\n        self.path = path \n        self.names = list(test_df.image_id)\n        self.attention_model = attention_model\n        self.num_tiles = num_tiles\n    \n    def __len__(self):\n        return len(self.names)\n    \n    def __getitem__(self,idx):\n        name = self.names[idx]\n        image = skimage.io.MultiImage(os.path.join(self.path,name+'.tiff'))[1]\n            \n        best_tiles = self.get_best_tiles(image)\n        images = self.make_square(best_tiles)\n        return torch.tensor(images).float(),name\n    \n    def make_square(self, tiles):\n        \n        idxes = list(range(self.num_tiles))\n        image_size = tiles[0].shape[0]\n\n        n_rows = int(self.num_tiles**0.5)\n        images = np.zeros((image_size * n_rows, image_size * n_rows, 3))\n        for h in range(n_rows):\n            for w in range(n_rows):\n                i = h * n_rows + w\n\n                this_img = tiles[idxes[i]]\n\n                this_img = 255 - this_img\n                \n                h1 = h * image_size\n                w1 = w * image_size\n                images[h1:h1 + image_size, w1:w1 + image_size] = this_img\n        \n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n        images = torch.tensor(images)\n        \n        return images\n            \n    def get_best_tiles(self,imgs):\n        tiles = preprocess(imgs,tile_size,num_attention_tiles)\n        tiles_bag = self.make_bag(tiles)\n        with torch.no_grad():\n            features = self.attention_model.feature_extractor(tiles_bag.cuda())\n            features = features.view(-1, num_attention_tiles, features.shape[-1])\n            weights = self.attention_model.head.attention_pool.compute_weights(features)\n        weights = weights.reshape(-1).argsort(0).cpu().numpy()[::-1]\n        best_tiles = [tiles[w] for w in weights[:num_tiles]] \n        return best_tiles\n    \n    def make_bag(self,tiles):\n        images = []\n        for tile in tiles:\n            tile = tile.astype(np.float32)\n            images.append(tile)\n        \n        images = np.stack(images,axis=0)\n        images = images.astype(np.float32)\n        images = images.transpose(0,3,1,2)\n        images /= 255 \n        images = torch.tensor(images)\n        return images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_df = pd.read_csv(SAMPLE)\n\nif os.path.exists(DATA):\n    for model, threshold, atten_model, atten_model_name in zip(MODELS,THRES, ATTENTION_MODELS, ATTENTION_MODELS_NAMES):\n        ds = StitchPandaDataset(image_folder,df,16,attention_model=atten_model)\n        \n        if ATTENTION_MEMORIES[atten_model_name] is not None:\n            ds.attention_memory = ATTENTION_MEMORIES[atten_model_name]\n        \n        preds = []\n        names = []\n        with torch.no_grad():\n            for x,y in tqdm(ds):\n                x = x.cuda()\n                #dihedral TTA\n                x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),\n                  x.transpose(-1,-2),x.transpose(-1,-2).flip(-1),\n                  x.transpose(-1,-2).flip(-2),x.transpose(-1,-2).flip(-1,-2)],0)\n    #             x = x.view(-1,num_tiles,3,tile_size,tile_size)\n                p = model(x)\n                p = p.view(-1).mean().cpu()\n                names.append(y)\n                preds.append(p)\n        final_preds = torch.tensor(preds).numpy()\n        final_preds = optimized_rounder.predict(final_preds,threshold)\n        final_preds = final_preds.astype(int)\n        \n        PREDS.append(final_preds)\n        NAMES = names\n        \n        ATTENTION_MEMORIES[atten_model_name] = ds.attention_memory.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Arnaud models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport cv2\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.utils.data as tdata\nimport os\nimport pickle\nfrom hubconf import *\nfrom mish_activation import Mish\nimport skimage.io\nfrom scipy.stats import mode\nfrom efficientnet_pytorch.model import EfficientNet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ATTENTION_MODELS = [f'../input/panda-localtrain/attention_gcp/fold_1/epoch28-kappa0.8734.ckpt']\n\nMODELS = [f'../input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_1/epoch23-kappa0.9014.ckpt',\n          f'../input/panda-localtrain/efficient0-20200618-064502/efficient0-20200618-064502/fold_4/epoch26-kappa0.9034.ckpt',\n          f'../input/panda-localtrain/efficient0-20200709-212941/efficient0-20200709-212941/fold_1/epoch33-kappa0.9021.ckpt',\n          \n          f'../input/panda-localtrain/efficient0-20200628-213437/efficient0-20200628-213437/fold_1/epoch27-kappa0.8957.ckpt',\n          f'../input/panda-localtrain/efficient0-20200616-200111/efficient0-20200616-200111/fold_1/epoch26-kappa0.8911.ckpt',\n          f'../input/panda-localtrain/25tiles-gcp/25tiles-gcp/fold_1/epoch29-kappa0.9046.ckpt',\n          f'../input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch29-kappa0.8975.ckpt',\n          f'../input/panda-localtrain/efficient0-20200708-205659/efficient0-20200708-205659/fold_1/epoch27-kappa0.8982.ckpt']\n\nattention_tiles_num = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AdaptiveConcatPool2d(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.avg = nn.AdaptiveAvgPool2d(output_size=(1, 1))\n        self.max = nn.AdaptiveMaxPool2d(output_size=(1, 1))\n\n    def forward(self, x):\n        avg_x = self.avg(x)\n        max_x = self.max(x)\n        return torch.cat([avg_x, max_x], dim=1).squeeze(2).squeeze(2)\n\n\nclass Flatten(nn.Module):\n    def forward(self, x):\n        return x.view(x.shape[0], -1)\n\n\nclass EfficientModel(nn.Module):\n\n    def __init__(self, c_out=5, n_tiles=36, tile_size=224, name='efficientnet-b0', strategy='stitched', head='basic'):\n        super().__init__()\n\n        from efficientnet_pytorch import EfficientNet\n        m = EfficientNet.from_name(name, num_classes=c_out, in_channels=3)\n        c_feature = m._fc.in_features\n        m._fc = nn.Identity()\n        self.feature_extractor = m\n        self.n_tiles = n_tiles\n        self.tile_size = tile_size\n        \n        if strategy == 'stitched':\n            if head == 'basic':\n                self.head = nn.Linear(c_feature, c_out)\n            elif head == 'concat':\n                m._avg_pooling = AdaptiveConcatPool2d()\n                self.head = nn.Linear(c_feature * 2, c_out)\n            elif head == 'gem':\n                m._avg_pooling = GeM()\n                self.head = nn.Linear(c_feature, c_out)\n        elif strategy == 'bag':\n            if head == 'basic':\n                self.head = BasicHead(c_feature, c_out, n_tiles)\n            elif head == 'attention':\n                self.head = AttentionHead(c_feature, c_out, n_tiles)\n                \n        self.strategy = strategy\n\n    def forward(self, x):\n        if self.strategy == 'bag':\n            x = x.view(-1, 3, self.tile_size, self.tile_size)\n        h = self.feature_extractor(x)\n        h = self.head(h)\n        return h\n\n\nclass BasicHead(nn.Module):\n\n    def __init__(self, c_in, c_out, n_tiles):\n        self.n_tiles = n_tiles\n        super().__init__()\n        self.fc = nn.Sequential(AdaptiveConcatPool2d(),\n                                nn.Linear(c_in * 2, c_out))\n\n    def forward(self, x):\n\n        bn, c = x.shape\n        h = x.view(-1, self.n_tiles, c, 1, 1).permute(0, 2, 1, 3, 4) \\\n            .contiguous().view(-1, c, 1 * self.n_tiles, 1)\n        h = self.fc(h)\n        return h\n\n\nclass AttentionHead(nn.Module):\n\n    def __init__(self, c_in, c_out, n_tiles):\n        self.n_tiles = n_tiles\n        super().__init__()\n        self.attention_pool = AttentionPool(c_in, c_in//2)\n        self.fc = nn.Linear(c_in, c_out)\n\n    def forward(self, x):\n\n        bn, c = x.shape\n        h = x.view(-1, self.n_tiles, c)\n        h = self.attention_pool(h)\n        h = self.fc(h)\n        return h\n\n\nclass AttentionPool(nn.Module):\n\n    def __init__(self, c_in, d):\n        super().__init__()\n        self.lin_V = nn.Linear(c_in, d)\n        self.lin_w = nn.Linear(d, 1)\n\n    def compute_weights(self, x):\n        key = self.lin_V(x)  # b, n, d\n        weights = self.lin_w(torch.tanh(key))  # b, n, 1\n        weights = torch.softmax(weights, dim=1)\n        return weights\n\n    def forward(self, x):\n        weights = self.compute_weights(x)\n        pooled = torch.matmul(x.transpose(1, 2), weights).squeeze(2)   # b, c, n x b, n, 1 => b, c, 1\n        return pooled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TileMaker:\n    def __init__(self, size, number, scale):\n        self.size = size\n        self.number = number\n        self.scale = scale\n\n    def __pad(self, image, constant_values=255):\n        h, w, c = image.shape\n        horizontal_pad = 0 if (w % self.size) == 0 else self.size - (w % self.size)\n        vertical_pad = 0 if (h % self.size) == 0 else self.size - (h % self.size)\n\n        image = np.pad(image, pad_width=((vertical_pad // 2, vertical_pad - vertical_pad // 2),\n                                         (horizontal_pad // 2, horizontal_pad - horizontal_pad // 2),\n                                         (0, 0)),\n                       mode='constant', constant_values=constant_values)  # Empty is white in this data\n        return image\n\n    def __call__(self, image, constant_values=255):\n\n        image = self.__pad(image, constant_values=constant_values)\n        h, w, c = image.shape\n        image = image.reshape(h // self.size, self.size, w // self.size, self.size, c)\n        image = image.swapaxes(1, 2).reshape(-1, self.size, self.size, c)\n\n        if image.shape[0] < self.number:\n            image = np.pad(image, pad_width=((0, self.number - image.shape[0]), (0, 0), (0, 0), (0, 0)),\n                           mode='constant', constant_values=constant_values)\n\n        sorted_tiles = np.argsort(np.sum(image, axis=(1, 2, 3)))\n        sorted_tiles = sorted_tiles[:self.number]\n\n        image = image[sorted_tiles]\n\n        return image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import argparse\ndef dict_to_args(d):\n\n    args = argparse.Namespace()\n\n    def dict_to_args_recursive(args, d, prefix=''):\n        for k, v in d.items():\n            if type(v) == dict:\n                dict_to_args_recursive(args, v, prefix=k)\n            elif type(v) in [tuple, list]:\n                continue\n            else:\n                if prefix:\n                    args.__setattr__(prefix + '_' + k, v)\n                else:\n                    args.__setattr__(k, v)\n\n    dict_to_args_recursive(args, d)\n    return args","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, model):\n        super().__init__()\n        self.model = model\n    def forward(self, x):\n        return model(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"attention_models = []\n\nfor attention_model_path in ATTENTION_MODELS:\n    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n    ckpt = torch.load(attention_model_path)\n    hparams = dict_to_args(ckpt['hparams'])\n\n    attention_model = EfficientModel(c_out=5,\n                            n_tiles=attention_tiles_num,\n                            tile_size=hparams.tile_size,\n                            name=hparams.backbone,\n                           strategy=hparams.strategy,\n                           head=hparams.head\n                            )\n    \n    attention_model = Model(attention_model)\n    attention_model.load_state_dict(ckpt['state_dict'])\n    attention_model = attention_model.model.to(device)\n    attention_model.eval();\n    attention_models.append(attention_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"models = []\nhparams_list = []\nfor model_path in MODELS:\n    ckpt = torch.load(model_path)\n    try:\n        hparams = dict_to_args(ckpt['hyper_parameters'])\n    except:\n        hparams = dict_to_args(ckpt['hparams'])\n    if hparams.loss == 'mse':\n        c_out = 1\n    elif hparams.loss == 'bce':\n        c_out = 5\n    model = EfficientModel(c_out=c_out,\n                        n_tiles=hparams.n_tiles,\n                        tile_size=hparams.tile_size,\n                        name=hparams.backbone,\n                       strategy=hparams.strategy,\n                       head=hparams.head\n                        )\n    model = Model(model)\n    model.load_state_dict(ckpt['state_dict'])\n    model = model.model.to(device)\n    model.eval();\n    models.append(model)\n    hparams_list.append(hparams)\n    print(hparams)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tilemaker = TileMaker(hparams.tile_size, attention_tiles_num, hparams.scale)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(tile):\n    tile = transforms.ToTensor()(tile)\n    tile = 1 - tile\n    return tile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InferenceDataset(tdata.Dataset):\n    \n    def __init__(self, attention_model, img_path, df, level, tile_size, num_tiles, scale, transform=None):\n        self.img_path = img_path\n        self.tile_maker = TileMaker(tile_size, attention_tiles_num, scale)\n        self.imgs_names = list(df.image_id)\n        self.df = df\n        self.level = level\n        self.num_tiles = num_tiles\n        self.transform = transform\n        self.attention_model = attention_model\n        self.attention_memory = {}\n    \n    def __len__(self):\n        return len(self.attention_dataset.imgs_names)\n    \n    def __getitem__(self, idx):\n        name = self.imgs_names[idx]\n        img = skimage.io.MultiImage(os.path.join(self.img_path, name + '.tiff'))[-self.level]\n\n        best_tiles = self.get_best_tiles(img, name, transform_id=0)\n        images = self.make_square(best_tiles)\n        best_tiles = self.get_best_tiles(img[128:-128], name, transform_id=1)\n        images2 = self.make_square(best_tiles)\n        best_tiles = self.get_best_tiles(img[:, 128:-128], name, transform_id=2)\n        images3 = self.make_square(best_tiles)\n        best_tiles = self.get_best_tiles(img[128:-128, 128:-128], name, transform_id=3)\n        images4 = self.make_square(best_tiles)\n        \n        return images, images2, images3, images4, name\n\n    def __len__(self):\n        return len(self.imgs_names)\n    \n    def make_square(self, tiles):\n        \n        idxes = list(range(self.num_tiles))\n        image_size = tiles[0].shape[0]\n\n        n_rows = int(self.num_tiles**0.5)\n        images = np.zeros((image_size * n_rows, image_size * n_rows, 3))\n        for h in range(n_rows):\n            for w in range(n_rows):\n                i = h * n_rows + w\n\n                this_img = tiles[idxes[i]]\n\n                this_img = 255 - this_img\n                if self.transform is not None:\n                    this_img = self.transform(image=this_img)['image']  # albumentations\n\n                h1 = h * image_size\n                w1 = w * image_size\n                images[h1:h1 + image_size, w1:w1 + image_size] = this_img\n        \n        if self.transform is not None:\n            images = self.transform(image=images)['image']  # albumentations\n\n        images = images.astype(np.float32)\n        images /= 255\n        images = images.transpose(2, 0, 1)\n        images = torch.tensor(images)\n        \n        return images\n    \n    def get_best_tiles(self, img, name, transform_id):\n        tiles = self.tile_maker(img)\n        tiles_bag = self.make_bag(tiles)\n        \n        memory_name = name + '_' + str(transform_id)\n        \n        if memory_name in self.attention_memory:\n            weights = self.attention_memory[memory_name]\n        else:\n            weights = 0\n            for attention_model in self.attention_model:\n                with torch.no_grad():\n                    features = attention_model.feature_extractor(tiles_bag.to(device))\n                    features = features.view(-1, attention_tiles_num, features.shape[-1])\n                    weights += attention_model.head.attention_pool.compute_weights(features)\n\n            weights /= len(self.attention_model)\n            weights = weights.reshape(-1).argsort(0).cpu().numpy()[::-1]\n            \n            #save in memory\n            self.attention_memory[memory_name] = weights.astype(np.int32)\n        \n        best_tiles = [tiles[w] for w in weights[:self.num_tiles]]\n        \n        return best_tiles\n        \n    def make_bag(self, tiles):\n        images = []\n        for tile in tiles:\n            tile = 255 - tile\n            if self.transform is not None:\n                tile = self.transform(image=tile)['image']\n            tile = tile.astype(np.float32)\n            images.append(tile)\n            \n        images = np.stack(images, axis=0)\n        images = images.astype(np.float32)\n        images = images.transpose(0, 3, 1, 2)\n        images /= 255\n        images = torch.tensor(images)\n        return images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfs_preds = []\nprevious_memory = None\nfor model, hparams in zip(models, hparams_list):\n\n    dataset = InferenceDataset(attention_models, image_folder, df, hparams.level, hparams.tile_size, hparams.n_tiles, hparams.scale)\n    if previous_memory is not None:\n        dataset.attention_memory = previous_memory\n\n    ids = []\n    predictions = []\n    with torch.no_grad():\n        for idx in range(len(dataset)):\n            imgs, imgs2, imgs3, imgs4, name = dataset[idx]\n            imgs, imgs2, imgs3, imgs4 = imgs.unsqueeze(0), imgs2.unsqueeze(0), imgs3.unsqueeze(0), imgs4.unsqueeze(0)\n            logits = model(imgs.to(device))\n            logits = logits + model(imgs2.to(device))\n            logits = logits + model(imgs3.to(device))\n            logits = logits + model(imgs4.to(device))\n            logits = logits/4\n            if hparams.loss == 'bce':\n                pred = logits.sigmoid().sum(1).round()\n            elif hparams.loss == 'mse':\n                pred = (logits.sigmoid().squeeze(1) * 5).round()\n\n            pred = pred.detach().cpu().numpy()\n            ids.append(name)\n            predictions.append(pred)\n\n    ids = np.array(ids)\n    predictions = np.concatenate(predictions, axis=0)\n    pred_df = pd.DataFrame({'id': ids, 'preds': predictions})\n    dfs_preds.append(pred_df)\n    \n    previous_memory = dataset.attention_memory.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(len(dfs_preds)):\n    PREDS.append(dfs_preds[i]['preds'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.stats import mode","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"arr = np.stack(PREDS, axis=1)\nmajority_vote = arr.mean(axis=1).round().astype(int)  # Either Mean (since ordinal) or Mode for majority vote\n#majority_vote = mode(arr, axis=1)[0][:, 0].astype(int)  # Either Mean (since ordinal) or Mode for majority vote\nsub_df = pd.DataFrame({'image_id': ids, 'isup_grade': majority_vote})\nsub_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}