{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install ../input/environment-pyvips/*.tar.bz2 ","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:36:56.337803Z","iopub.execute_input":"2022-08-30T21:36:56.339256Z","iopub.status.idle":"2022-08-30T21:37:04.436402Z","shell.execute_reply.started":"2022-08-30T21:36:56.339186Z","shell.execute_reply":"2022-08-30T21:37:04.435258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"USE_PYVIPS = True","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:04.43916Z","iopub.execute_input":"2022-08-30T21:37:04.439564Z","iopub.status.idle":"2022-08-30T21:37:04.44708Z","shell.execute_reply.started":"2022-08-30T21:37:04.439524Z","shell.execute_reply":"2022-08-30T21:37:04.446086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport string\nimport joblib\nimport tifffile\nimport numpy as np \nimport pandas as pd \nimport torch\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport glob\nimport openslide\nimport psutil\n\nfrom torch import nn\nfrom torchvision import models\nfrom scipy.stats import mode \nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom tqdm.notebook import tqdm\nfrom torch.optim import lr_scheduler\nimport torchvision.transforms as transforms\nimport warnings\nimport sys\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\n\nif USE_PYVIPS:\n    import pyvips\n    pyvips.cache_set_max(0)\nCOMPARE_PYVIPS_AND_OPENSLIDE = USE_PYVIPS and False","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:04.448929Z","iopub.execute_input":"2022-08-30T21:37:04.449802Z","iopub.status.idle":"2022-08-30T21:37:11.048868Z","shell.execute_reply.started":"2022-08-30T21:37:04.449766Z","shell.execute_reply":"2022-08-30T21:37:11.047714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"process = psutil.Process(os.getpid())\ntest_images_path = \"/kaggle/input/mayo-clinic-strip-ai/test/\"\ntest_images_names = os.listdir(test_images_path)\ntest_output_path = \"./test_cropped/\"\nif not os.path.exists(test_output_path):\n    os.mkdir(test_output_path)\nSIZE = 512","metadata":{"papermill":{"duration":0.02504,"end_time":"2022-07-08T14:24:41.073811","exception":false,"start_time":"2022-07-08T14:24:41.048771","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T21:37:11.052413Z","iopub.execute_input":"2022-08-30T21:37:11.053386Z","iopub.status.idle":"2022-08-30T21:37:11.062099Z","shell.execute_reply.started":"2022-08-30T21:37:11.053343Z","shell.execute_reply":"2022-08-30T21:37:11.061135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_csv = pd.read_csv(\"../input/mayo-clinic-strip-ai/test.csv\")\ndf_test_csv","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.063855Z","iopub.execute_input":"2022-08-30T21:37:11.064287Z","iopub.status.idle":"2022-08-30T21:37:11.091173Z","shell.execute_reply.started":"2022-08-30T21:37:11.064216Z","shell.execute_reply":"2022-08-30T21:37:11.09021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick comparison *OpenSlide* dimensions and *pyvips* get","metadata":{}},{"cell_type":"code","source":"if COMPARE_PYVIPS_AND_OPENSLIDE:\n    image = pyvips.Image.new_from_file(test_images_path + test_images_names[0], access='sequential')\n    print(\"Fields to get:\", *image.get_fields())\n    print(\"width:\", image.get(\"width\"))\n    print(\"height: \", image.get(\"height\"))\n    image.invalidate()\n    del image\n    gc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.092537Z","iopub.execute_input":"2022-08-30T21:37:11.092873Z","iopub.status.idle":"2022-08-30T21:37:11.099382Z","shell.execute_reply.started":"2022-08-30T21:37:11.09284Z","shell.execute_reply":"2022-08-30T21:37:11.098144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if COMPARE_PYVIPS_AND_OPENSLIDE:\n    image = openslide.OpenSlide(test_images_path + test_images_names[0])\n    print(\"dim 0: \", image.dimensions[0])\n    print(\"dim 1: \", image.dimensions[1])\n    del image\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.100975Z","iopub.execute_input":"2022-08-30T21:37:11.101589Z","iopub.status.idle":"2022-08-30T21:37:11.107279Z","shell.execute_reply.started":"2022-08-30T21:37:11.101555Z","shell.execute_reply":"2022-08-30T21:37:11.106304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating patches","metadata":{}},{"cell_type":"markdown","source":"#### Memory before processing","metadata":{}},{"cell_type":"code","source":"mb = process.memory_info().rss / (1024 * 1024)\nprint(f\"Current memory: {mb} MB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.10902Z","iopub.execute_input":"2022-08-30T21:37:11.109446Z","iopub.status.idle":"2022-08-30T21:37:11.116041Z","shell.execute_reply.started":"2022-08-30T21:37:11.109413Z","shell.execute_reply":"2022-08-30T21:37:11.114968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### preprocessing the images","metadata":{}},{"cell_type":"code","source":"def get_background_pixel(image, x_size, y_size):\n    pixels = []\n    for i in range(20):\n        x_random = np.random.randint(0, x_size)\n        y_random = np.random.randint(0, y_size)\n        if USE_PYVIPS:\n            pixel = image.crop(y_random,x_random,1,1).numpy().copy().squeeze()\n        else:\n            pixel = np.array(image.read_region((y_random,x_random),0,(1,1)).convert(\"RGB\")).squeeze()\n        pixels.append(pixel)    \n    background_pixel = mode(pixels, axis=0)\n    return background_pixel[0][0]\n\ndef accept_image(cropped_image, background_pixel, threshold ):\n    image = cropped_image\n    n_pixels = image.shape[0]*image.shape[1]\n    n_background_pixels = (image == background_pixel).all(axis=2).sum()\n    return (n_background_pixels / n_pixels) < (1-threshold)\n\ndef patch_image(image, size, image_name, threshold):\n    stride = size\n    counter = 0\n    if USE_PYVIPS:\n        x_size = image.get(\"height\") # shape[1]\n        y_size = image.get(\"width\") # shape[0]\n    else:\n        x_size = image.dimensions[1]\n        y_size = image.dimensions[0]\n    \n    background_pixel = get_background_pixel(image, x_size, y_size)\n    for y in tqdm(range(0, y_size - stride,stride)):\n        for x in range(0, x_size - stride, stride):\n            if USE_PYVIPS:\n                pyvips_im = image.crop(y,x,SIZE,SIZE)\n                cropped_im = pyvips_im.numpy().copy()\n            else:\n                cropped_im = np.array(image.read_region((y,x),0,(SIZE,SIZE)).convert(\"RGB\"))\n            if (accept_image(cropped_im, background_pixel, threshold)):\n                if cropped_im.shape == (512,512,3):\n                    img_to_save = cv2.resize(cropped_im, (96,96))\n                    cv2.imwrite(test_output_path + f\"{image_name}_{counter}.jpg\", img=img_to_save)\n                    counter +=1\n            if USE_PYVIPS:\n                pyvips_im.invalidate()\n    print(f\"Image {image_name} was patched into {counter} images\")\n    return counter","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.117829Z","iopub.execute_input":"2022-08-30T21:37:11.118721Z","iopub.status.idle":"2022-08-30T21:37:11.134465Z","shell.execute_reply.started":"2022-08-30T21:37:11.118686Z","shell.execute_reply":"2022-08-30T21:37:11.133478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, test_image_name in enumerate(test_images_names):\n    print(f\"Image {i}: {test_image_name}\")\n    retry = True\n    threshold = 0.95\n    while retry and threshold > 0.1:\n        try:\n            if USE_PYVIPS:\n                image = pyvips.Image.new_from_file(test_images_path + test_image_name, access=\"sequential\")\n            else:\n                image = openslide.OpenSlide(test_images_path + test_image_name)\n        except:\n            image = np.zeros((96,96,3), np.uint8)\n        try:\n            counter = patch_image(image, size=SIZE, image_name = test_image_name, threshold=threshold)\n        except Exception as e:\n            print(\"Error: \", e)\n            counter = 1\n        if USE_PYVIPS:\n            image.invalidate()\n        del image\n        gc.collect()\n        if counter==0:\n            threshold = threshold - 0.05\n        else:\n            retry = False\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:37:11.140165Z","iopub.execute_input":"2022-08-30T21:37:11.140475Z","iopub.status.idle":"2022-08-30T21:40:26.55089Z","shell.execute_reply.started":"2022-08-30T21:37:11.140442Z","shell.execute_reply":"2022-08-30T21:40:26.549782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### memory after preprocessing","metadata":{}},{"cell_type":"code","source":"mb = process.memory_info().rss / (1024 * 1024)\nprint(f\"Current memory: {mb} MB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.552302Z","iopub.execute_input":"2022-08-30T21:40:26.552748Z","iopub.status.idle":"2022-08-30T21:40:26.559479Z","shell.execute_reply.started":"2022-08-30T21:40:26.552711Z","shell.execute_reply":"2022-08-30T21:40:26.558294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating test dataframe","metadata":{}},{"cell_type":"code","source":"patched_images_paths = []\npatched_image_ids = []\npaths = glob.glob(f\"{test_output_path}/*.jpg\")\npatched_images_paths.extend(paths)\npatched_image_ids.extend([os.path.basename(file_name)[:8] for file_name in paths])","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.561099Z","iopub.execute_input":"2022-08-30T21:40:26.562555Z","iopub.status.idle":"2022-08-30T21:40:26.579991Z","shell.execute_reply.started":"2022-08-30T21:40:26.562514Z","shell.execute_reply":"2022-08-30T21:40:26.579146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patched_images_paths = np.array(patched_images_paths)\npatched_image_ids = np.array(patched_image_ids)\ndf_test_csv[\"patched_paths\"] = df_test_csv[\"image_id\"].map(lambda x: patched_images_paths[x==patched_image_ids])\ndel patched_image_ids\ndel patched_images_paths\ngc.collect()\ndf_test_csv","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.581361Z","iopub.execute_input":"2022-08-30T21:40:26.581698Z","iopub.status.idle":"2022-08-30T21:40:26.770485Z","shell.execute_reply.started":"2022-08-30T21:40:26.581665Z","shell.execute_reply":"2022-08-30T21:40:26.76951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corresponging_image_paths = []\ncorresponding_image_ids = []\nfor i, row in tqdm(df_test_csv.iterrows()):\n    N = len(row.patched_paths)\n    corresponding_image_ids.extend([str(row.image_id)]*N)\n    corresponging_image_paths.extend(row.patched_paths)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.771933Z","iopub.execute_input":"2022-08-30T21:40:26.772552Z","iopub.status.idle":"2022-08-30T21:40:26.816595Z","shell.execute_reply.started":"2022-08-30T21:40:26.772514Z","shell.execute_reply":"2022-08-30T21:40:26.815702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_dict = {\"image_id\": corresponding_image_ids, \"path\": corresponging_image_paths}\ndf_test = pd.DataFrame(df_test_dict)\ndf_test","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.817955Z","iopub.execute_input":"2022-08-30T21:40:26.818531Z","iopub.status.idle":"2022-08-30T21:40:26.833108Z","shell.execute_reply.started":"2022-08-30T21:40:26.818496Z","shell.execute_reply":"2022-08-30T21:40:26.832161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del corresponding_image_ids\ndel corresponging_image_paths\ndel df_test_csv\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:26.834591Z","iopub.execute_input":"2022-08-30T21:40:26.834922Z","iopub.status.idle":"2022-08-30T21:40:27.008249Z","shell.execute_reply.started":"2022-08-30T21:40:26.834888Z","shell.execute_reply":"2022-08-30T21:40:27.006722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class ImgDataset(Dataset):\n    def __init__(self, df, transform):\n        self.df = df \n        self.transform = transform\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        path = self.df.iloc[index].path\n        try:\n            image = cv2.imread(path)\n        except:\n            image = np.zeros((96,96,3), np.uint8)\n        label = 0\n        if self.transform:\n            try:\n                image = self.transform(image)\n            except:\n                try:\n                    image = image.transpose(2, 0, 1)\n                except:\n                    image = np.zeros((3, 96, 96))\n        else:\n            image = image.transpose(2, 0, 1)\n\n        patient_id = str(self.df.iloc[index].image_id)[:6]\n        return image, label, patient_id","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T21:40:27.009779Z","iopub.execute_input":"2022-08-30T21:40:27.010429Z","iopub.status.idle":"2022-08-30T21:40:27.020212Z","shell.execute_reply.started":"2022-08-30T21:40:27.010385Z","shell.execute_reply":"2022-08-30T21:40:27.019285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose(\n    [\n    transforms.ToTensor(),\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:27.023393Z","iopub.execute_input":"2022-08-30T21:40:27.023655Z","iopub.status.idle":"2022-08-30T21:40:27.032327Z","shell.execute_reply.started":"2022-08-30T21:40:27.023632Z","shell.execute_reply":"2022-08-30T21:40:27.030995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model prediction","metadata":{}},{"cell_type":"markdown","source":"#### memory before predicting","metadata":{}},{"cell_type":"code","source":"mb = process.memory_info().rss / (1024 * 1024)\nprint(f\"Current memory: {mb} MB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:27.033518Z","iopub.execute_input":"2022-08-30T21:40:27.034454Z","iopub.status.idle":"2022-08-30T21:40:27.043764Z","shell.execute_reply.started":"2022-08-30T21:40:27.034419Z","shell.execute_reply":"2022-08-30T21:40:27.042725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### predicting","metadata":{}},{"cell_type":"code","source":"def predict(model, dataloader):\n    model.cuda()\n    model.eval()\n    dataloader = dataloader\n    outputs = []\n    s = nn.Softmax(dim=1)\n    ids = []\n    for item in tqdm(dataloader, leave=False):\n        patient_id = item[2]\n        try:\n            images = item[0].cuda().float()\n            ids.extend(patient_id)\n            output = model(images)\n            softm = s(output.cpu()[:,:2])\n            outputs.extend(softm.detach().numpy())\n        except Exception as e:\n            print(e)\n            ids.extend(patient_id)\n            outputs.extend(s(torch.ones(batch_size, 2).float()).detach().numpy())\n    return np.array(outputs), ids","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T21:40:27.045444Z","iopub.execute_input":"2022-08-30T21:40:27.045815Z","iopub.status.idle":"2022-08-30T21:40:27.054441Z","shell.execute_reply.started":"2022-08-30T21:40:27.045782Z","shell.execute_reply":"2022-08-30T21:40:27.053382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# --------------------------------------------------------\n# Swin Transformer V2\n# Copyright (c) 2022 Microsoft\n# Licensed under The MIT License [see LICENSE for details]\n# Written by Ze Liu\n# --------------------------------------------------------\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.checkpoint as checkpoint\nfrom timm.models.layers import DropPath, to_2tuple, trunc_normal_\nimport numpy as np\n\n\nclass Mlp(nn.Module):\n    def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.):\n        super().__init__()\n        out_features = out_features or in_features\n        hidden_features = hidden_features or in_features\n        self.fc1 = nn.Linear(in_features, hidden_features)\n        self.act = act_layer()\n        self.fc2 = nn.Linear(hidden_features, out_features)\n        self.drop = nn.Dropout(drop)\n\n    def forward(self, x):\n        x = self.fc1(x)\n        x = self.act(x)\n        x = self.drop(x)\n        x = self.fc2(x)\n        x = self.drop(x)\n        return x\n\n\ndef window_partition(x, window_size):\n    \"\"\"\n    Args:\n        x: (B, H, W, C)\n        window_size (int): window size\n\n    Returns:\n        windows: (num_windows*B, window_size, window_size, C)\n    \"\"\"\n    B, H, W, C = x.shape\n    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)\n    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)\n    return windows\n\n\ndef window_reverse(windows, window_size, H, W):\n    \"\"\"\n    Args:\n        windows: (num_windows*B, window_size, window_size, C)\n        window_size (int): Window size\n        H (int): Height of image\n        W (int): Width of image\n\n    Returns:\n        x: (B, H, W, C)\n    \"\"\"\n    B = int(windows.shape[0] / (H * W / window_size / window_size))\n    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)\n    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)\n    return x\n\n\nclass WindowAttention(nn.Module):\n    r\"\"\" Window based multi-head self attention (W-MSA) module with relative position bias.\n    It supports both of shifted and non-shifted window.\n\n    Args:\n        dim (int): Number of input channels.\n        window_size (tuple[int]): The height and width of the window.\n        num_heads (int): Number of attention heads.\n        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True\n        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0\n        proj_drop (float, optional): Dropout ratio of output. Default: 0.0\n        pretrained_window_size (tuple[int]): The height and width of the window in pre-training.\n    \"\"\"\n\n    def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,\n                 pretrained_window_size=[0, 0]):\n\n        super().__init__()\n        self.dim = dim\n        self.window_size = window_size  # Wh, Ww\n        self.pretrained_window_size = pretrained_window_size\n        self.num_heads = num_heads\n\n        self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))), requires_grad=True)\n\n        # mlp to generate continuous relative position bias\n        self.cpb_mlp = nn.Sequential(nn.Linear(2, 512, bias=True),\n                                     nn.ReLU(inplace=True),\n                                     nn.Linear(512, num_heads, bias=False))\n\n        # get relative_coords_table\n        relative_coords_h = torch.arange(-(self.window_size[0] - 1), self.window_size[0], dtype=torch.float32)\n        relative_coords_w = torch.arange(-(self.window_size[1] - 1), self.window_size[1], dtype=torch.float32)\n        relative_coords_table = torch.stack(\n            torch.meshgrid([relative_coords_h,\n                            relative_coords_w])).permute(1, 2, 0).contiguous().unsqueeze(0)  # 1, 2*Wh-1, 2*Ww-1, 2\n        if pretrained_window_size[0] > 0:\n            relative_coords_table[:, :, :, 0] /= (pretrained_window_size[0] - 1)\n            relative_coords_table[:, :, :, 1] /= (pretrained_window_size[1] - 1)\n        else:\n            relative_coords_table[:, :, :, 0] /= (self.window_size[0] - 1)\n            relative_coords_table[:, :, :, 1] /= (self.window_size[1] - 1)\n        relative_coords_table *= 8  # normalize to -8, 8\n        relative_coords_table = torch.sign(relative_coords_table) * torch.log2(\n            torch.abs(relative_coords_table) + 1.0) / np.log2(8)\n\n        self.register_buffer(\"relative_coords_table\", relative_coords_table)\n\n        # get pair-wise relative position index for each token inside the window\n        coords_h = torch.arange(self.window_size[0])\n        coords_w = torch.arange(self.window_size[1])\n        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww\n        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww\n        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww\n        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2\n        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0\n        relative_coords[:, :, 1] += self.window_size[1] - 1\n        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1\n        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww\n        self.register_buffer(\"relative_position_index\", relative_position_index)\n\n        self.qkv = nn.Linear(dim, dim * 3, bias=False)\n        if qkv_bias:\n            self.q_bias = nn.Parameter(torch.zeros(dim))\n            self.v_bias = nn.Parameter(torch.zeros(dim))\n        else:\n            self.q_bias = None\n            self.v_bias = None\n        self.attn_drop = nn.Dropout(attn_drop)\n        self.proj = nn.Linear(dim, dim)\n        self.proj_drop = nn.Dropout(proj_drop)\n        self.softmax = nn.Softmax(dim=-1)\n\n    def forward(self, x, mask=None):\n        \"\"\"\n        Args:\n            x: input features with shape of (num_windows*B, N, C)\n            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None\n        \"\"\"\n        B_, N, C = x.shape\n        qkv_bias = None\n        if self.q_bias is not None:\n            qkv_bias = torch.cat((self.q_bias, torch.zeros_like(self.v_bias, requires_grad=False), self.v_bias))\n        qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias)\n        qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)\n        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)\n\n        # cosine attention\n        attn = (F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1))\n        logit_scale = torch.clamp(self.logit_scale, max=torch.log(torch.tensor(1. / 0.01))).exp()\n        attn = attn * logit_scale\n\n        relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view(-1, self.num_heads)\n        relative_position_bias = relative_position_bias_table[self.relative_position_index.view(-1)].view(\n            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH\n        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww\n        relative_position_bias = 16 * torch.sigmoid(relative_position_bias)\n        attn = attn + relative_position_bias.unsqueeze(0)\n\n        if mask is not None:\n            nW = mask.shape[0]\n            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)\n            attn = attn.view(-1, self.num_heads, N, N)\n            attn = self.softmax(attn)\n        else:\n            attn = self.softmax(attn)\n\n        attn = self.attn_drop(attn)\n\n        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)\n        x = self.proj(x)\n        x = self.proj_drop(x)\n        return x\n\n    def extra_repr(self) -> str:\n        return f'dim={self.dim}, window_size={self.window_size}, ' \\\n               f'pretrained_window_size={self.pretrained_window_size}, num_heads={self.num_heads}'\n\n    def flops(self, N):\n        # calculate flops for 1 window with token length of N\n        flops = 0\n        # qkv = self.qkv(x)\n        flops += N * self.dim * 3 * self.dim\n        # attn = (q @ k.transpose(-2, -1))\n        flops += self.num_heads * N * (self.dim // self.num_heads) * N\n        #  x = (attn @ v)\n        flops += self.num_heads * N * N * (self.dim // self.num_heads)\n        # x = self.proj(x)\n        flops += N * self.dim * self.dim\n        return flops\n\n\nclass SwinTransformerBlock(nn.Module):\n    r\"\"\" Swin Transformer Block.\n\n    Args:\n        dim (int): Number of input channels.\n        input_resolution (tuple[int]): Input resulotion.\n        num_heads (int): Number of attention heads.\n        window_size (int): Window size.\n        shift_size (int): Shift size for SW-MSA.\n        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.\n        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True\n        drop (float, optional): Dropout rate. Default: 0.0\n        attn_drop (float, optional): Attention dropout rate. Default: 0.0\n        drop_path (float, optional): Stochastic depth rate. Default: 0.0\n        act_layer (nn.Module, optional): Activation layer. Default: nn.GELU\n        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm\n        pretrained_window_size (int): Window size in pre-training.\n    \"\"\"\n\n    def __init__(self, dim, input_resolution, num_heads, window_size=7, shift_size=0,\n                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0., drop_path=0.,\n                 act_layer=nn.GELU, norm_layer=nn.LayerNorm, pretrained_window_size=0):\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.num_heads = num_heads\n        self.window_size = window_size\n        self.shift_size = shift_size\n        self.mlp_ratio = mlp_ratio\n        if min(self.input_resolution) <= self.window_size:\n            # if window size is larger than input resolution, we don't partition windows\n            self.shift_size = 0\n            self.window_size = min(self.input_resolution)\n        assert 0 <= self.shift_size < self.window_size, \"shift_size must in 0-window_size\"\n\n        self.norm1 = norm_layer(dim)\n        self.attn = WindowAttention(\n            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,\n            qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop,\n            pretrained_window_size=to_2tuple(pretrained_window_size))\n\n        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()\n        self.norm2 = norm_layer(dim)\n        mlp_hidden_dim = int(dim * mlp_ratio)\n        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop)\n\n        if self.shift_size > 0:\n            # calculate attention mask for SW-MSA\n            H, W = self.input_resolution\n            img_mask = torch.zeros((1, H, W, 1))  # 1 H W 1\n            h_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            w_slices = (slice(0, -self.window_size),\n                        slice(-self.window_size, -self.shift_size),\n                        slice(-self.shift_size, None))\n            cnt = 0\n            for h in h_slices:\n                for w in w_slices:\n                    img_mask[:, h, w, :] = cnt\n                    cnt += 1\n\n            mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1\n            mask_windows = mask_windows.view(-1, self.window_size * self.window_size)\n            attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)\n            attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))\n        else:\n            attn_mask = None\n\n        self.register_buffer(\"attn_mask\", attn_mask)\n\n    def forward(self, x):\n        H, W = self.input_resolution\n        B, L, C = x.shape\n        assert L == H * W, \"input feature has wrong size\"\n\n        shortcut = x\n        x = x.view(B, H, W, C)\n\n        # cyclic shift\n        if self.shift_size > 0:\n            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))\n        else:\n            shifted_x = x\n\n        # partition windows\n        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C\n        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C\n\n        # W-MSA/SW-MSA\n        attn_windows = self.attn(x_windows, mask=self.attn_mask)  # nW*B, window_size*window_size, C\n\n        # merge windows\n        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)\n        shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C\n\n        # reverse cyclic shift\n        if self.shift_size > 0:\n            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))\n        else:\n            x = shifted_x\n        x = x.view(B, H * W, C)\n        x = shortcut + self.drop_path(self.norm1(x))\n\n        # FFN\n        x = x + self.drop_path(self.norm2(self.mlp(x)))\n\n        return x\n\n    def extra_repr(self) -> str:\n        return f\"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, \" \\\n               f\"window_size={self.window_size}, shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}\"\n\n    def flops(self):\n        flops = 0\n        H, W = self.input_resolution\n        # norm1\n        flops += self.dim * H * W\n        # W-MSA/SW-MSA\n        nW = H * W / self.window_size / self.window_size\n        flops += nW * self.attn.flops(self.window_size * self.window_size)\n        # mlp\n        flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio\n        # norm2\n        flops += self.dim * H * W\n        return flops\n\n\nclass PatchMerging(nn.Module):\n    r\"\"\" Patch Merging Layer.\n\n    Args:\n        input_resolution (tuple[int]): Resolution of input feature.\n        dim (int): Number of input channels.\n        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm\n    \"\"\"\n\n    def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm):\n        super().__init__()\n        self.input_resolution = input_resolution\n        self.dim = dim\n        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)\n        self.norm = norm_layer(2 * dim)\n\n    def forward(self, x):\n        \"\"\"\n        x: B, H*W, C\n        \"\"\"\n        H, W = self.input_resolution\n        B, L, C = x.shape\n        assert L == H * W, \"input feature has wrong size\"\n        assert H % 2 == 0 and W % 2 == 0, f\"x size ({H}*{W}) are not even.\"\n\n        x = x.view(B, H, W, C)\n\n        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C\n        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C\n        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C\n        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C\n        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C\n        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C\n\n        x = self.reduction(x)\n        x = self.norm(x)\n\n        return x\n\n    def extra_repr(self) -> str:\n        return f\"input_resolution={self.input_resolution}, dim={self.dim}\"\n\n    def flops(self):\n        H, W = self.input_resolution\n        flops = (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim\n        flops += H * W * self.dim // 2\n        return flops\n\n\nclass BasicLayer(nn.Module):\n    \"\"\" A basic Swin Transformer layer for one stage.\n\n    Args:\n        dim (int): Number of input channels.\n        input_resolution (tuple[int]): Input resolution.\n        depth (int): Number of blocks.\n        num_heads (int): Number of attention heads.\n        window_size (int): Local window size.\n        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.\n        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True\n        drop (float, optional): Dropout rate. Default: 0.0\n        attn_drop (float, optional): Attention dropout rate. Default: 0.0\n        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0\n        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm\n        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None\n        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.\n        pretrained_window_size (int): Local window size in pre-training.\n    \"\"\"\n\n    def __init__(self, dim, input_resolution, depth, num_heads, window_size,\n                 mlp_ratio=4., qkv_bias=True, drop=0., attn_drop=0.,\n                 drop_path=0., norm_layer=nn.LayerNorm, downsample=None, use_checkpoint=False,\n                 pretrained_window_size=0):\n\n        super().__init__()\n        self.dim = dim\n        self.input_resolution = input_resolution\n        self.depth = depth\n        self.use_checkpoint = use_checkpoint\n\n        # build blocks\n        self.blocks = nn.ModuleList([\n            SwinTransformerBlock(dim=dim, input_resolution=input_resolution,\n                                 num_heads=num_heads, window_size=window_size,\n                                 shift_size=0 if (i % 2 == 0) else window_size // 2,\n                                 mlp_ratio=mlp_ratio,\n                                 qkv_bias=qkv_bias,\n                                 drop=drop, attn_drop=attn_drop,\n                                 drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,\n                                 norm_layer=norm_layer,\n                                 pretrained_window_size=pretrained_window_size)\n            for i in range(depth)])\n\n        # patch merging layer\n        if downsample is not None:\n            self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer)\n        else:\n            self.downsample = None\n\n    def forward(self, x):\n        for blk in self.blocks:\n            if self.use_checkpoint:\n                x = checkpoint.checkpoint(blk, x)\n            else:\n                x = blk(x)\n        if self.downsample is not None:\n            x = self.downsample(x)\n        return x\n\n    def extra_repr(self) -> str:\n        return f\"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}\"\n\n    def flops(self):\n        flops = 0\n        for blk in self.blocks:\n            flops += blk.flops()\n        if self.downsample is not None:\n            flops += self.downsample.flops()\n        return flops\n\n    def _init_respostnorm(self):\n        for blk in self.blocks:\n            nn.init.constant_(blk.norm1.bias, 0)\n            nn.init.constant_(blk.norm1.weight, 0)\n            nn.init.constant_(blk.norm2.bias, 0)\n            nn.init.constant_(blk.norm2.weight, 0)\n\n\nclass PatchEmbed(nn.Module):\n    r\"\"\" Image to Patch Embedding\n\n    Args:\n        img_size (int): Image size.  Default: 224.\n        patch_size (int): Patch token size. Default: 4.\n        in_chans (int): Number of input image channels. Default: 3.\n        embed_dim (int): Number of linear projection output channels. Default: 96.\n        norm_layer (nn.Module, optional): Normalization layer. Default: None\n    \"\"\"\n\n    def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):\n        super().__init__()\n        img_size = to_2tuple(img_size)\n        patch_size = to_2tuple(patch_size)\n        patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]]\n        self.img_size = img_size\n        self.patch_size = patch_size\n        self.patches_resolution = patches_resolution\n        self.num_patches = patches_resolution[0] * patches_resolution[1]\n\n        self.in_chans = in_chans\n        self.embed_dim = embed_dim\n\n        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)\n        if norm_layer is not None:\n            self.norm = norm_layer(embed_dim)\n        else:\n            self.norm = None\n\n    def forward(self, x):\n        B, C, H, W = x.shape\n        # FIXME look at relaxing size constraints\n        assert H == self.img_size[0] and W == self.img_size[1], \\\n            f\"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]}).\"\n        x = self.proj(x).flatten(2).transpose(1, 2)  # B Ph*Pw C\n        if self.norm is not None:\n            x = self.norm(x)\n        return x\n\n    def flops(self):\n        Ho, Wo = self.patches_resolution\n        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])\n        if self.norm is not None:\n            flops += Ho * Wo * self.embed_dim\n        return flops\n\n\nclass SwinTransformerV2(nn.Module):\n    r\"\"\" Swin Transformer\n        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -\n          https://arxiv.org/pdf/2103.14030\n\n    Args:\n        img_size (int | tuple(int)): Input image size. Default 224\n        patch_size (int | tuple(int)): Patch size. Default: 4\n        in_chans (int): Number of input image channels. Default: 3\n        num_classes (int): Number of classes for classification head. Default: 1000\n        embed_dim (int): Patch embedding dimension. Default: 96\n        depths (tuple(int)): Depth of each Swin Transformer layer.\n        num_heads (tuple(int)): Number of attention heads in different layers.\n        window_size (int): Window size. Default: 7\n        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4\n        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True\n        drop_rate (float): Dropout rate. Default: 0\n        attn_drop_rate (float): Attention dropout rate. Default: 0\n        drop_path_rate (float): Stochastic depth rate. Default: 0.1\n        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.\n        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False\n        patch_norm (bool): If True, add normalization after patch embedding. Default: True\n        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False\n        pretrained_window_sizes (tuple(int)): Pretrained window sizes of each layer.\n    \"\"\"\n\n    def __init__(self, img_size=224, patch_size=4, in_chans=3, num_classes=1000,\n                 embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24],\n                 window_size=7, mlp_ratio=4., qkv_bias=True,\n                 drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,\n                 norm_layer=nn.LayerNorm, ape=False, patch_norm=True,\n                 use_checkpoint=False, pretrained_window_sizes=[0, 0, 0, 0], **kwargs):\n        super().__init__()\n\n        self.num_classes = num_classes\n        self.num_layers = len(depths)\n        self.embed_dim = embed_dim\n        self.ape = ape\n        self.patch_norm = patch_norm\n        self.num_features = int(embed_dim * 2 ** (self.num_layers - 1))\n        self.mlp_ratio = mlp_ratio\n\n        # split image into non-overlapping patches\n        self.patch_embed = PatchEmbed(\n            img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,\n            norm_layer=norm_layer if self.patch_norm else None)\n        num_patches = self.patch_embed.num_patches\n        patches_resolution = self.patch_embed.patches_resolution\n        self.patches_resolution = patches_resolution\n\n        # absolute position embedding\n        if self.ape:\n            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim))\n            trunc_normal_(self.absolute_pos_embed, std=.02)\n\n        self.pos_drop = nn.Dropout(p=drop_rate)\n\n        # stochastic depth\n        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule\n\n        # build layers\n        self.layers = nn.ModuleList()\n        for i_layer in range(self.num_layers):\n            layer = BasicLayer(dim=int(embed_dim * 2 ** i_layer),\n                               input_resolution=(patches_resolution[0] // (2 ** i_layer),\n                                                 patches_resolution[1] // (2 ** i_layer)),\n                               depth=depths[i_layer],\n                               num_heads=num_heads[i_layer],\n                               window_size=window_size,\n                               mlp_ratio=self.mlp_ratio,\n                               qkv_bias=qkv_bias,\n                               drop=drop_rate, attn_drop=attn_drop_rate,\n                               drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],\n                               norm_layer=norm_layer,\n                               downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,\n                               use_checkpoint=use_checkpoint,\n                               pretrained_window_size=pretrained_window_sizes[i_layer])\n            self.layers.append(layer)\n\n        self.norm = norm_layer(self.num_features)\n        self.avgpool = nn.AdaptiveAvgPool1d(1)\n        self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity()\n\n        self.apply(self._init_weights)\n        for bly in self.layers:\n            bly._init_respostnorm()\n\n    def _init_weights(self, m):\n        if isinstance(m, nn.Linear):\n            trunc_normal_(m.weight, std=.02)\n            if isinstance(m, nn.Linear) and m.bias is not None:\n                nn.init.constant_(m.bias, 0)\n        elif isinstance(m, nn.LayerNorm):\n            nn.init.constant_(m.bias, 0)\n            nn.init.constant_(m.weight, 1.0)\n\n    @torch.jit.ignore\n    def no_weight_decay(self):\n        return {'absolute_pos_embed'}\n\n    @torch.jit.ignore\n    def no_weight_decay_keywords(self):\n        return {\"cpb_mlp\", \"logit_scale\", 'relative_position_bias_table'}\n\n    def forward_features(self, x):\n        x = self.patch_embed(x)\n        if self.ape:\n            x = x + self.absolute_pos_embed\n        x = self.pos_drop(x)\n\n        for layer in self.layers:\n            x = layer(x)\n\n        x = self.norm(x)  # B L C\n        x = self.avgpool(x.transpose(1, 2))  # B C 1\n        x = torch.flatten(x, 1)\n        return x\n\n    def forward(self, x):\n        x = self.forward_features(x)\n        x = self.head(x)\n        return x\n\n    def flops(self):\n        flops = 0\n        flops += self.patch_embed.flops()\n        for i, layer in enumerate(self.layers):\n            flops += layer.flops()\n        flops += self.num_features * self.patches_resolution[0] * self.patches_resolution[1] // (2 ** self.num_layers)\n        flops += self.num_features * self.num_classes\n        return flops","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:40:27.056368Z","iopub.execute_input":"2022-08-30T21:40:27.056915Z","iopub.status.idle":"2022-08-30T21:40:27.148576Z","shell.execute_reply.started":"2022-08-30T21:40:27.056849Z","shell.execute_reply":"2022-08-30T21:40:27.14766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = torch.jit.load('../input/mayo-clinic-swin-v2-training/swin_new_0.pth')\nmodel = SwinTransformerV2(img_size=96, num_classes=2, window_size=6, patch_size=4, embed_dim = 96 )\nmodel.load_state_dict(torch.load(\"../input/mayoclinic-swinv2-patched-96-models/swin_model_1_25.pt\"))\nbatch_size = 1\ntest_loader = DataLoader(\n    ImgDataset(df_test, transform=transform), \n    batch_size=batch_size, \n    shuffle=False, \n    num_workers=1\n)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T21:40:27.152062Z","iopub.execute_input":"2022-08-30T21:40:27.152374Z","iopub.status.idle":"2022-08-30T21:40:33.710148Z","shell.execute_reply.started":"2022-08-30T21:40:27.152349Z","shell.execute_reply":"2022-08-30T21:40:33.709128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"anss, ids = predict(model, test_loader)","metadata":{"papermill":{"duration":null,"end_time":null,"exception":null,"start_time":null,"status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-30T21:40:33.711719Z","iopub.execute_input":"2022-08-30T21:40:33.712081Z","iopub.status.idle":"2022-08-30T21:41:26.174015Z","shell.execute_reply.started":"2022-08-30T21:40:33.712042Z","shell.execute_reply":"2022-08-30T21:41:26.173018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### memory after predicting","metadata":{}},{"cell_type":"code","source":"mb = process.memory_info().rss / (1024 * 1024)\nprint(f\"Current memory: {mb} MB\")","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:41:26.17663Z","iopub.execute_input":"2022-08-30T21:41:26.177403Z","iopub.status.idle":"2022-08-30T21:41:26.184413Z","shell.execute_reply.started":"2022-08-30T21:41:26.17734Z","shell.execute_reply":"2022-08-30T21:41:26.183195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\"patient_id\": ids, \"CE\":anss[:,0], \"LAA\":anss[:,1]})\nsubmission = submission.groupby(by = [\"patient_id\"], as_index = False).mean()\nsubmission","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:41:26.186491Z","iopub.execute_input":"2022-08-30T21:41:26.186922Z","iopub.status.idle":"2022-08-30T21:41:26.214414Z","shell.execute_reply.started":"2022-08-30T21:41:26.186886Z","shell.execute_reply":"2022-08-30T21:41:26.213547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-30T21:41:26.215758Z","iopub.execute_input":"2022-08-30T21:41:26.216072Z","iopub.status.idle":"2022-08-30T21:41:26.224874Z","shell.execute_reply.started":"2022-08-30T21:41:26.216041Z","shell.execute_reply":"2022-08-30T21:41:26.223806Z"},"trusted":true},"execution_count":null,"outputs":[]}]}