{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CLAM\n\nNOTE: Some of the descriptions or images are cited from: https://github.com/mahmoodlab/CLAM\n\n![img](https://github.com/mahmoodlab/CLAM/raw/master/docs/CLAM2.jpg)\n\n## TL;DR:\n\n+ CLAM is a high-throughput and interpretable method for data efficient whole slide image (WSI) classification using slide-level labels without any ROI extraction or patch-level annotations, and is capable of handling multi-class subtyping problems. Tested on three different WSI datasets, trained models adapt to independent test cohorts of WSI resections and biopsies as well as smartphone microscopy images (photomicrographs).\n+ paper: https://arxiv.org/abs/2004.09666\n\n## How to apply CLAM on the STRIP AI dataset ?\n\n+ I prepared four notebooks for pre-process, train and inference:\n\n### pre-process\n\n+ (1) image generation: https://www.kaggle.com/code/fx6300/clam-strip-ai-image-generation\n+ <b>&gt; THIS NOTEBOOK &lt;</b> (2) feature extraction: https://www.kaggle.com/code/fx6300/clam-strip-ai-feature-extraction\n\n### train\n\n+ (3) train: https://www.kaggle.com/code/fx6300/clam-strip-ai-train\n\n### inference\n\n+ (4) inference: https://www.kaggle.com/code/fx6300/clam-strip-ai-inference\n\n## How to visualize the attention generated by CLAM ?\n\n+ I prepared an example:\n  + https://www.kaggle.com/fx6300/clam-strip-ai-attention-heatmap\n\n## NOTE\n\n+ The source code from CLAM (https://github.com/mahmoodlab/CLAM) is licensed under GPLv3 and available for non-commercial academic purposes.","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport time\nimport random\nimport string\nimport joblib\nimport numpy as np \nimport pandas as pd \nimport torch\nfrom torch import nn\nimport seaborn as sns\nfrom torchvision import models\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.notebook import tqdm\nfrom tqdm import trange\nimport warnings\nimport torch.nn.functional as F\nimport h5py\nimport torch.utils.model_zoo as model_zoo\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":3.221842,"end_time":"2022-07-08T22:23:18.429026","exception":false,"start_time":"2022-07-08T22:23:15.207184","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-27T01:51:18.055747Z","iopub.execute_input":"2022-09-27T01:51:18.057028Z","iopub.status.idle":"2022-09-27T01:51:20.77982Z","shell.execute_reply.started":"2022-09-27T01:51:18.056899Z","shell.execute_reply":"2022-09-27T01:51:20.778723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debug = False\ngenerate_new = True\ntrain_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\")\ntest_df = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndirs = [\"../input/mayo-clinic-strip-ai/train/\", \"../input/mayo-clinic-strip-ai/test/\"]\nIMG_HEIGHT = 512 * 8\nIMG_WIDTH = 512 * 8","metadata":{"papermill":{"duration":0.026694,"end_time":"2022-07-08T22:23:18.458885","exception":false,"start_time":"2022-07-08T22:23:18.432191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-09-27T01:51:20.782237Z","iopub.execute_input":"2022-09-27T01:51:20.7832Z","iopub.status.idle":"2022-09-27T01:51:20.806891Z","shell.execute_reply.started":"2022-09-27T01:51:20.783161Z","shell.execute_reply":"2022-09-27T01:51:20.805915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',\n           'resnet152']\n\nmodel_urls = {\n    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',\n    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',\n    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',\n    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',\n    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',\n}\n\nclass Bottleneck_Baseline(nn.Module):\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super(Bottleneck_Baseline, self).__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n\nclass ResNet_Baseline(nn.Module):\n\n    def __init__(self, block, layers):\n        self.inplanes = 64\n        super(ResNet_Baseline, self).__init__()\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n                               bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d(1) \n\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n            elif isinstance(m, nn.BatchNorm2d):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample))\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n\n        return x\n\ndef resnet50_baseline(pretrained=False):\n    \"\"\"Constructs a Modified ResNet-50 model.\n    Args:\n        pretrained (bool): If True, returns a model pre-trained on ImageNet\n    \"\"\"\n    model = ResNet_Baseline(Bottleneck_Baseline, [3, 4, 6, 3])\n    if pretrained:\n        model = load_pretrained_weights(model, 'resnet50')\n    return model\n\ndef load_pretrained_weights(model, name):\n    pretrained_dict = model_zoo.load_url(model_urls[name])\n    model.load_state_dict(pretrained_dict, strict=False)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-09-27T01:51:20.810067Z","iopub.execute_input":"2022-09-27T01:51:20.810765Z","iopub.status.idle":"2022-09-27T01:51:20.856024Z","shell.execute_reply.started":"2022-09-27T01:51:20.810728Z","shell.execute_reply":"2022-09-27T01:51:20.854965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df):\n        self.df = df \n        self.train = 'label' in df.columns\n    def __len__(self):\n        return len(self.df) * 64\n    \n    def __getitem__(self, _index):\n        paths = [\"../input/4096-tiles-v4/test-4096-tiles-v4/4096-tiles-v4/\", \"../input/4096-tiles-v4/train-4096-tiles-v4/4096-tiles-v4/\"]\n        index = _index // 64\n        pos = _index % 64\n        image_id = self.df.iloc[index].image_id\n        image = cv2.imread(paths[self.train] + image_id + f\"_{pos}\" + \".jpg\").transpose(2, 0, 1)\n        label = None\n        if(self.train):\n            label = {\"CE\" : 0, \"LAA\": 1}[self.df.iloc[index].label]\n        return image, label, image_id, pos","metadata":{"execution":{"iopub.status.busy":"2022-09-27T01:51:20.864292Z","iopub.execute_input":"2022-09-27T01:51:20.866977Z","iopub.status.idle":"2022-09-27T01:51:20.878293Z","shell.execute_reply.started":"2022-09-27T01:51:20.866935Z","shell.execute_reply":"2022-09-27T01:51:20.877327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_model(model, train_loader, output_dir):\n    s = nn.Softmax(dim=1)\n    model.cuda()\n    model.eval()\n    for item in tqdm(train_loader, leave=False):\n        images = item[0].cuda().float()\n        classes = item[1].cuda().long()\n        image_ids = item[2]\n        image_poses = item[3]\n        with torch.no_grad():\n            output = model(images)\n        for i in range(output.shape[0]):\n            f = h5py.File(f\"{output_dir}/{image_ids[i]}.h5\", 'a')\n            f.create_dataset(f\"{image_poses[i]}\", data=output[i].cpu(), dtype=np.float32)\n            f.close()\n        del images, classes, image_ids, image_poses\n        gc.collect()\n        torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T01:51:20.88287Z","iopub.execute_input":"2022-09-27T01:51:20.885299Z","iopub.status.idle":"2022-09-27T01:51:20.895542Z","shell.execute_reply.started":"2022-09-27T01:51:20.88526Z","shell.execute_reply":"2022-09-27T01:51:20.894598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, train_idx in enumerate(train_df.index):   \n    model = resnet50_baseline(pretrained=True)\n    train = train_df.iloc[[train_idx]]\n    batch_size = 32\n    train_loader = DataLoader(\n        ImgDataset(train), \n        batch_size=batch_size, \n        shuffle=False, \n        num_workers=1\n    )\n    output_dir = \"./my_features-v4\"\n    if not os.path.exists(output_dir):\n        os.mkdir(output_dir)\n    apply_model(model, train_loader, output_dir)\n    del model, train, train_loader\n    gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T01:51:20.899784Z","iopub.execute_input":"2022-09-27T01:51:20.902218Z","iopub.status.idle":"2022-09-27T02:16:18.053531Z","shell.execute_reply.started":"2022-09-27T01:51:20.902181Z","shell.execute_reply":"2022-09-27T02:16:18.052369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}