{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"_____\n**Credit:** \n- Parts of this notebook are based on the [great](https://www.kaggle.com/code/realneuralnetwork/coatnet-strip-ai-inference) [notebooks](https://www.kaggle.com/code/realneuralnetwork/cnn-strip-ai-inference) by [Kabir Ivan\n](https://www.kaggle.com/realneuralnetwork)\n_____\n","metadata":{}},{"cell_type":"markdown","source":"# [Train + Infer] CoAtNet + EfficientNet\n\nThis notebook contains steps and code to train CoAtNet and EfficientNet-B4 for the Mayo Clinic - STRIP AI competition.\nIn this scenario we train CoAtNet + EfficinetNet-B4, then use these models for inference","metadata":{}},{"cell_type":"markdown","source":"#### Setup","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/einops')\nsys.path.append('../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master')\nfrom efficientnet_pytorch import EfficientNet\n# !pip install --upgrade efficientnet-pytorch","metadata":{"execution":{"iopub.status.busy":"2022-07-20T11:24:09.524422Z","iopub.execute_input":"2022-07-20T11:24:09.524765Z","iopub.status.idle":"2022-07-20T11:24:09.570926Z","shell.execute_reply.started":"2022-07-20T11:24:09.524735Z","shell.execute_reply":"2022-07-20T11:24:09.569908Z"},"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CoAtNet: Marrying Convolution and Attention for All Data Sizes\n\n[Paper](https://arxiv.org/pdf/2106.04803.pdf)\n\nThe CoAtNet paper attempts to effectively combine the strengths from both convolutional and transformers architectures, they present CoAtNets(pronounced \"coat\" nets), a family of hybrid models built from two key insights: \n- Depthwise Convolution and self-Attention can be naturally unified via simple relative attention\n- Vertically stacking convolution layers and attention layers in a principled way is surprisingly effective in improving generalization, capacity and efficiency.\n\n![](https://i.ibb.co/Sd6wj7D/Selection-998.png)","metadata":{}},{"cell_type":"markdown","source":"### Imports, Seed, Data loading","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport torch\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch.nn as nn\nimport seaborn as sns\nfrom random import randint\nfrom einops import rearrange\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom torch.optim import lr_scheduler\nfrom einops.layers.torch import Rearrange\nfrom efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nimport warnings; warnings.filterwarnings(\"ignore\")\ngc.enable()\n\n\ndef seed_everything(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    os.environ['PYTHONHASHSEED'] = str(seed_value)    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = True\n\nseed = 42\nseed_everything(seed)\n\n\ndebug = False\ngenerate_new = False\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 1000)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":3.063995,"end_time":"2022-07-08T14:24:41.045696","exception":false,"start_time":"2022-07-08T14:24:37.981701","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-20T11:24:11.858965Z","iopub.execute_input":"2022-07-20T11:24:11.859875Z","iopub.status.idle":"2022-07-20T11:24:11.931208Z","shell.execute_reply.started":"2022-07-20T11:24:11.859835Z","shell.execute_reply":"2022-07-20T11:24:11.930321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_count = max(train_df.label.value_counts())\nfor label in train_df.label.unique():\n    df = train_df.loc[train_df.label == label]\n    while(train_df.label.value_counts()[label] < max_count):\n        train_df = pd.concat([train_df, df.head(max_count - train_df.label.value_counts()[label])], axis = 0)\n        \nif(generate_new):\n    os.mkdir(\"./train/\")\n    os.mkdir(\"./test/\")\n    for i in tqdm(range(test_df.shape[0])):\n        img_id = test_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./test/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n    for i in tqdm(range(train_df.shape[0])):\n        img_id = train_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[0] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./train/{img_id}.jpg\", img)\n        del img\n        gc.collect()","metadata":{"papermill":{"duration":69.477711,"end_time":"2022-07-08T14:25:50.554416","exception":false,"start_time":"2022-07-08T14:24:41.076705","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-20T04:12:45.16538Z","iopub.execute_input":"2022-07-20T04:12:45.166211Z","iopub.status.idle":"2022-07-20T04:12:45.177478Z","shell.execute_reply.started":"2022-07-20T04:12:45.16617Z","shell.execute_reply":"2022-07-20T04:12:45.176114Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Class","metadata":{}},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns    \n    def __len__(self): return len(self.df)    \n    def __getitem__(self, index):\n        if(generate_new): paths = [\"./test/\", \"./train/\"]\n        else: paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        label = None\n        if(self.train): label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        return image, label","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-20T04:12:45.183967Z","iopub.execute_input":"2022-07-20T04:12:45.184253Z","iopub.status.idle":"2022-07-20T04:12:45.198305Z","shell.execute_reply.started":"2022-07-20T04:12:45.184226Z","shell.execute_reply":"2022-07-20T04:12:45.196711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoAtNet Training","metadata":{}},{"cell_type":"code","source":"def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs):\n    best_acc = 0.0\n    for epoch in range(num_epochs):\n        model.cuda()       \n        for phase in ['train', 'val']:\n            if phase == 'train': model.train()\n            else: model.eval()\n               \n            epoch_loss = 0.0\n            epoch_acc = 0\n            \n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().long()\n                optimizer.zero_grad()                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)                    \n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')    \n        if epoch_acc > best_acc:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n            traced.save('model.pth')\n            best_acc = epoch_acc","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-20T04:12:45.200221Z","iopub.execute_input":"2022-07-20T04:12:45.200791Z","iopub.status.idle":"2022-07-20T04:12:45.217917Z","shell.execute_reply.started":"2022-07-20T04:12:45.200746Z","shell.execute_reply":"2022-07-20T04:12:45.216122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### CoAtNet Architecture","metadata":{}},{"cell_type":"code","source":"def conv_3x3_bn(inp, oup, image_size, downsample=False):\n    stride = 1 if downsample == False else 2\n    return nn.Sequential(\n        nn.Conv2d(inp, oup, 3, stride, 1, bias=False),\n        nn.BatchNorm2d(oup),\n        nn.GELU()\n    )\n\nclass PreNorm(nn.Module):\n    def __init__(self, dim, fn, norm):\n        super().__init__()\n        self.norm = norm(dim)\n        self.fn = fn\n    def forward(self, x, **kwargs):\n        return self.fn(self.norm(x), **kwargs)\n\nclass SE(nn.Module):\n    def __init__(self, inp, oup, expansion=0.25):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(oup, int(inp * expansion), bias=False),\n            nn.GELU(),\n            nn.Linear(int(inp * expansion), oup, bias=False),\n            nn.Sigmoid())\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y\n\nclass FeedForward(nn.Module):\n    def __init__(self, dim, hidden_dim, dropout=0.):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(dim, hidden_dim),\n            nn.GELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, dim),\n            nn.Dropout(dropout))\n    def forward(self, x):\n        return self.net(x)\n\n\nclass MBConv(nn.Module):\n    def __init__(self, inp, oup, image_size, downsample=False, expansion=4):\n        super().__init__()\n        self.downsample = downsample\n        stride = 1 if self.downsample == False else 2\n        hidden_dim = int(inp * expansion)\n\n        if self.downsample:\n            self.pool = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n\n        if expansion == 1:\n            self.conv = nn.Sequential(\n                nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup))\n        else:\n            self.conv = nn.Sequential(\n                nn.Conv2d(inp, hidden_dim, 1, stride, 0, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),                \n                nn.Conv2d(hidden_dim, hidden_dim, 3, 1, 1, groups=hidden_dim, bias=False),\n                nn.BatchNorm2d(hidden_dim),\n                nn.GELU(),\n                SE(inp, hidden_dim),\n                nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False),\n                nn.BatchNorm2d(oup))\n        \n        self.conv = PreNorm(inp, self.conv, nn.BatchNorm2d)\n\n    def forward(self, x):\n        if self.downsample: return self.proj(self.pool(x)) + self.conv(x)\n        else: return x + self.conv(x)\n\nclass Attention(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, dropout=0.):\n        super().__init__()\n        inner_dim = dim_head * heads\n        project_out = not (heads == 1 and dim_head == inp)\n        self.ih, self.iw = image_size\n        self.heads = heads\n        self.scale = dim_head ** -0.5       \n        self.relative_bias_table = nn.Parameter(\n            torch.zeros((2 * self.ih - 1) * (2 * self.iw - 1), heads))\n        coords = torch.meshgrid((torch.arange(self.ih), torch.arange(self.iw)))\n        coords = torch.flatten(torch.stack(coords), 1)\n        relative_coords = coords[:, :, None] - coords[:, None, :]\n        relative_coords[0] += self.ih - 1\n        relative_coords[1] += self.iw - 1\n        relative_coords[0] *= 2 * self.iw - 1\n        relative_coords = rearrange(relative_coords, 'c h w -> h w c')\n        relative_index = relative_coords.sum(-1).flatten().unsqueeze(1)\n        self.register_buffer(\"relative_index\", relative_index)\n        self.attend = nn.Softmax(dim=-1)\n        self.to_qkv = nn.Linear(inp, inner_dim * 3, bias=False)\n        self.to_out = nn.Sequential(\n            nn.Linear(inner_dim, oup),\n            nn.Dropout(dropout)\n        ) if project_out else nn.Identity()\n\n    def forward(self, x):\n        qkv = self.to_qkv(x).chunk(3, dim=-1)\n        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=self.heads), qkv)\n        dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n        relative_bias = self.relative_bias_table.gather(0, self.relative_index.repeat(1, self.heads))\n        relative_bias = rearrange(relative_bias, '(h w) c -> 1 c h w', h=self.ih*self.iw, w=self.ih*self.iw)\n        dots = dots + relative_bias\n        attn = self.attend(dots)\n        out = torch.matmul(attn, v)\n        out = rearrange(out, 'b h n d -> b n (h d)')\n        out = self.to_out(out)\n        return out\n\nclass Transformer(nn.Module):\n    def __init__(self, inp, oup, image_size, heads=8, dim_head=32, downsample=False, dropout=0.):\n        super().__init__()\n        hidden_dim = int(inp * 4)\n        self.ih, self.iw = image_size\n        self.downsample = downsample\n        if self.downsample:\n            self.pool1 = nn.MaxPool2d(3, 2, 1)\n            self.pool2 = nn.MaxPool2d(3, 2, 1)\n            self.proj = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)\n        self.attn = Attention(inp, oup, image_size, heads, dim_head, dropout)\n        self.ff = FeedForward(oup, hidden_dim, dropout)\n        self.attn = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(inp, self.attn, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw))\n        self.ff = nn.Sequential(\n            Rearrange('b c ih iw -> b (ih iw) c'),\n            PreNorm(oup, self.ff, nn.LayerNorm),\n            Rearrange('b (ih iw) c -> b c ih iw', ih=self.ih, iw=self.iw))\n\n    def forward(self, x):\n        if self.downsample: x = self.proj(self.pool1(x)) + self.attn(self.pool2(x))\n        else: x = x + self.attn(x)\n        x = x + self.ff(x)\n        return x\n\n\nclass CoAtNet(nn.Module):\n    def __init__(self, image_size, in_channels, num_blocks, channels, num_classes=1000, block_types=['C', 'C', 'T', 'T']):\n        super().__init__()\n        ih, iw = image_size\n        block = {'C': MBConv, 'T': Transformer}\n\n        self.s0 = self._make_layer(\n            conv_3x3_bn, in_channels, channels[0], num_blocks[0], (ih // 2, iw // 2))\n        self.s1 = self._make_layer(\n            block[block_types[0]], channels[0], channels[1], num_blocks[1], (ih // 4, iw // 4))\n        self.s2 = self._make_layer(\n            block[block_types[1]], channels[1], channels[2], num_blocks[2], (ih // 8, iw // 8))\n        self.s3 = self._make_layer(\n            block[block_types[2]], channels[2], channels[3], num_blocks[3], (ih // 16, iw // 16))\n        self.s4 = self._make_layer(\n            block[block_types[3]], channels[3], channels[4], num_blocks[4], (ih // 32, iw // 32))\n\n        self.pool = nn.AvgPool2d(ih // 32, 1)\n        self.fc = nn.Linear(channels[-1], num_classes, bias=False)\n\n    def forward(self, x):\n        x = self.s0(x)\n        x = self.s1(x)\n        x = self.s2(x)\n        x = self.s3(x)\n        x = self.s4(x)\n\n        x = self.pool(x).view(-1, x.shape[1])\n        x = self.fc(x)\n        return x\n\n    def _make_layer(self, block, inp, oup, depth, image_size):\n        layers = nn.ModuleList([])\n        for i in range(depth):\n            if i == 0:\n                layers.append(block(inp, oup, image_size, downsample=True))\n            else:\n                layers.append(block(oup, oup, image_size))\n        return nn.Sequential(*layers)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T04:12:45.220335Z","iopub.execute_input":"2022-07-20T04:12:45.220815Z","iopub.status.idle":"2022-07-20T04:12:45.279328Z","shell.execute_reply.started":"2022-07-20T04:12:45.22077Z","shell.execute_reply":"2022-07-20T04:12:45.278181Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Architecture Definition & Running the training","metadata":{}},{"cell_type":"code","source":"num_blocks = [2, 2, 12, 28, 2]\nchannels = [64, 64, 128, 256, 512]\nmodel = CoAtNet((512, 512), 3, num_blocks, channels, num_classes=2)\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\nbatch_size = 1\ntrain_loader = DataLoader(ImgDataset(train), batch_size=batch_size, shuffle=False, num_workers=1)\nval_loader = DataLoader(ImgDataset(val), batch_size=batch_size, shuffle=False, num_workers=1)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\ncriterion = nn.CrossEntropyLoss()\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-20T04:12:45.283156Z","iopub.execute_input":"2022-07-20T04:12:45.284399Z","iopub.status.idle":"2022-07-20T04:12:46.083151Z","shell.execute_reply.started":"2022-07-20T04:12:45.284358Z","shell.execute_reply":"2022-07-20T04:12:46.081866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CoAtNet Inference","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport torch\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\n\nimport warnings; warnings.filterwarnings(\"ignore\")\ngc.enable()\n\ndebug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\nif(test_df.shape[0] == 4): test_df = pd.concat([test_df for i in range(25)])\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df\n\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()      \n    \nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id       \n        \ndef predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids       \n    \nmodel = torch.jit.load('model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\n\nanss, ids = predict(model, test_loader)\nprob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\nsubmission.to_csv(\"submission_coatnet.csv\", index = False)","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"_____\n\n# EfficientNet B4\n\nEfficientNet is one of the most solid baselines known today. It is a family of convolutional neural networks that have achieved state-of-the-art accuracy on ImageNet while also being smaller and faster than other models.\nThe main idea of EfficientNet is scaling up CNNs in a principled way. It uses a scalable architecture, named compound scaling, which balances network depth, width, and resolution to achieve superior performance.\nThe image below shows the scaling method in more detail.\n\n![](https://i.ibb.co/Y86KGDg/image4-1.png)","metadata":{}},{"cell_type":"markdown","source":"#### EfficientNet B4 - Training","metadata":{}},{"cell_type":"markdown","source":"**Setups**","metadata":{}},{"cell_type":"code","source":"!mkdir /root/.cache/torch\n!mkdir /root/.cache/torch/hub\n!mkdir /root/.cache/torch/hub/checkpoints\n!cp -r ../input/torchhub-efficientnet-b4/nvidia_efficientnet-b4_210412.pth /root/.cache/torch/hub/checkpoints/","metadata":{"execution":{"iopub.status.busy":"2022-07-20T11:08:48.996095Z","iopub.execute_input":"2022-07-20T11:08:48.996455Z","iopub.status.idle":"2022-07-20T11:08:53.673901Z","shell.execute_reply.started":"2022-07-20T11:08:48.996422Z","shell.execute_reply":"2022-07-20T11:08:53.672603Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport torch\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nfrom torch import nn\nimport seaborn as sns\nimport efficientnet_pytorch\nfrom torchvision import models\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom torch.optim import lr_scheduler\nfrom efficientnet_pytorch import EfficientNet\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nimport warnings; warnings.filterwarnings(\"ignore\")\n\ndebug = False\ngenerate_new = False\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\").head(10 if debug else 1000)\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\n\nif(generate_new):\n    os.mkdir(\"./train/\")\n    os.mkdir(\"./test/\")\n    for i in tqdm(range(test_df.shape[0])):\n        img_id = test_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./test/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n    for i in tqdm(range(train_df.shape[0])):\n        img_id = train_df.iloc[i].image_id\n        img = cv2.resize(tifffile.imread(dirs[0] + img_id + \".tif\"), (512, 512))\n        cv2.imwrite(f\"./train/{img_id}.jpg\", img)\n        del img\n        gc.collect()\n        \nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"./test/\", \"./train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        if len(image.shape) == 5:\n            image = image.squeeze().transpose(1, 2, 0)\n        image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        label = None\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        return image, label\n    \ndef train_model(model, dataloaders_dict, criterion, optimizer, num_epochs):\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        model.cuda()\n        \n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()\n            else:\n                model.eval()\n                \n            epoch_loss = 0.0\n            epoch_acc = 0\n            \n            dataloader = dataloaders_dict[phase]\n            for item in tqdm(dataloader, leave=False):\n                images = item[0].cuda().float()\n                classes = item[1].cuda().long()\n\n                optimizer.zero_grad()\n                \n                with torch.set_grad_enabled(phase == 'train'):\n                    output = model(images)\n                    loss = criterion(output, classes)\n                    _, preds = torch.max(output, 1)\n\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                    epoch_loss += loss.item() * len(output)\n                    epoch_acc += torch.sum(preds == classes.data)\n                    \n\n            data_size = len(dataloader.dataset)\n            epoch_loss = epoch_loss / data_size\n            epoch_acc = epoch_acc.double() / data_size\n\n            print(f'Epoch {epoch + 1}/{num_epochs} | {phase:^5} | Loss: {epoch_loss:.4f} | Acc: {epoch_acc:.4f}')\n        \n        if epoch_acc > best_acc:\n            traced = torch.jit.trace(model.cpu(), torch.rand(1, 3, 512, 512))\n            traced.save('efficientnet_model.pth')\n            best_acc = epoch_acc","metadata":{"execution":{"iopub.status.busy":"2022-07-20T11:27:41.135449Z","iopub.execute_input":"2022-07-20T11:27:41.135775Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Running the training","metadata":{}},{"cell_type":"code","source":"model = efficientnet_pytorch.EfficientNet.from_name(\"efficientnet-b4\")\ncheckpoint = torch.load('../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth')\nmodel.load_state_dict(checkpoint)\n\ntrain, val = train_test_split(train_df, test_size=0.2, random_state=42, stratify = train_df.label)\nbatch_size = 1\ntrain_loader = DataLoader(ImgDataset(train), batch_size=batch_size, shuffle=False, num_workers=1)\nval_loader = DataLoader(ImgDataset(val), batch_size=batch_size, shuffle=False, num_workers=1)\ndataloaders_dict = {\"train\": train_loader, \"val\": val_loader}\ncriterion = nn.CrossEntropyLoss()\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\ntrain_model(model, dataloaders_dict, criterion, optimizer, 1)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T11:27:41.135449Z","iopub.execute_input":"2022-07-20T11:27:41.135775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Efficientnet Inference","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport torch\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nfrom torch import nn\nimport seaborn as sns\nimport efficientnet_pytorch\nfrom tqdm.notebook import tqdm\nfrom torchvision import models\nimport matplotlib.pyplot as plt\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nimport warnings; warnings.filterwarnings(\"ignore\")\n\ngc.enable()\ndebug = False\ngenerate_new = True\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\ntest_df\n\ntry:\n    os.mkdir(\"../test/\")\nexcept:\n    pass\nfor i in tqdm(range(test_df.shape[0])):\n    img_id = test_df.iloc[i].image_id\n    try:\n        sz = os.path.getsize(dirs[1] + img_id + \".tif\")\n    except:\n        sz = 1000000000\n    if(sz > 8e8):\n        img = np.zeros((512,512,3), np.uint8)\n    else:\n        try:\n            img = cv2.resize(tifffile.imread(dirs[1] + img_id + \".tif\"), (512, 512))\n        except:\n            img = np.zeros((512,512,3), np.uint8)\n    cv2.imwrite(f\"../test/{img_id}.jpg\", img)\n    del img\n    gc.collect()\n    \n    \nclass ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        if(generate_new):\n            paths = [\"../test/\", \"../train/\"]\n        else:\n            paths = [\"../input/jpg-images-strip-ai/test/\", \"../input/jpg-images-strip-ai/train/\"]\n        try:\n            image = cv2.imread(paths[self.train] + self.df.iloc[index].image_id + \".jpg\")\n        except:\n            image = np.zeros((512,512,3), np.uint8)\n        label = 0\n        try:\n            if len(image.shape) == 5:\n                image = image.squeeze().transpose(1, 2, 0)\n            image = cv2.resize(image, (512, 512)).transpose(2, 0, 1)\n        except:\n            image = np.zeros((3, 512, 512))\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        patient_id = self.df.iloc[index].patient_id\n        return image, label, patient_id\ndef predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2][0]\n        try:\n            images = item[0].cuda().float()\n            ids.append(patient_id)\n            output = model(images)\n            outputs.append(s(output.cpu()[:,:2])[0].detach().numpy())\n        except:\n            ids.append(patient_id)\n            outputs.append(s(torch.tensor([[1, 1]]).float())[0].detach().numpy())\n    return np.array(outputs), ids\n\n\nmodel = torch.jit.load('efficientnet_model.pth')\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(test_df), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)\nanss, ids = predict(model, test_loader)\n\nprob = pd.DataFrame({\"CE\" : anss[:,0], \"LAA\" : anss[:,1], \"id\" : ids}).groupby(\"id\").mean()\nsubmission = pd.read_csv(\"../input/mayo-clinic-strip-ai/sample_submission.csv\")\nsubmission.CE = prob.CE.to_list()\nsubmission.LAA = prob.LAA.to_list()\nsubmission.to_csv(\"submission_efficientnet.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-20T11:27:41.135449Z","iopub.execute_input":"2022-07-20T11:27:41.135775Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble\n\nNow that we got both our models ready, we can simply combine the predictions of both and use them as an ensemble!\nWe do this by running the predictions and averaging them. This yields a more powerful model.","metadata":{}},{"cell_type":"code","source":"submission_coatnet = pd.read_csv(\"submission_coatnet.csv\")\nsubmission_efficientnet = pd.read_csv(\"submission_efficientnet.csv\")\nsub_df = pd.read_csv('../input/mayo-clinic-strip-ai/sample_submission.csv')\n\nsub_df['CE'] = (submission_efficientnet['CE'].values + submission_coatnet['CE'].values) / 2.0\nsub_df['LAA'] = (submission_efficientnet['LAA'].values + submission_coatnet['LAA'].values) / 2.0\n\n\nsub_df.to_csv('submission.csv', index = False)","metadata":{},"execution_count":null,"outputs":[]}]}