{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Setup","metadata":{}},{"cell_type":"code","source":"import sys\nsys.path.append('../input/mayo-clinic-strip-ai-dataset/packages/pretrained-models.pytorch-master')\nsys.path.append('../input/mayo-clinic-strip-ai-dataset/packages/EfficientNet-PyTorch-master')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:42:35.016088Z","iopub.execute_input":"2022-10-05T04:42:35.016893Z","iopub.status.idle":"2022-10-05T04:42:35.022361Z","shell.execute_reply.started":"2022-10-05T04:42:35.016855Z","shell.execute_reply":"2022-10-05T04:42:35.021188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!conda install --offline ../input/how-to-use-pyvips-offline/*.tar.bz2 \n!pip install ../input/mayo-clinic-strip-ai-dataset/packages/timm-0.6.7-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:41:23.484809Z","iopub.execute_input":"2022-10-05T04:41:23.485159Z","iopub.status.idle":"2022-10-05T04:42:35.012574Z","shell.execute_reply.started":"2022-10-05T04:41:23.485124Z","shell.execute_reply":"2022-10-05T04:42:35.011352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nfrom glob import glob\nfrom tqdm import tqdm\nimport yaml\nimport json\nimport pyvips\n\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport tifffile\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom sklearn.metrics import accuracy_score, roc_auc_score, log_loss\nfrom sklearn.linear_model import LinearRegression\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport timm\n\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-05T04:42:57.07018Z","iopub.execute_input":"2022-10-05T04:42:57.070558Z","iopub.status.idle":"2022-10-05T04:43:01.357165Z","shell.execute_reply.started":"2022-10-05T04:42:57.070523Z","shell.execute_reply":"2022-10-05T04:43:01.356078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INTERNAL_DATASET = Path('../input/mayo-clinic-strip-ai')\nEXTERNAL_DATASET = Path('../input/mayo-clinic-strip-ai-dataset')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:43:01.35963Z","iopub.execute_input":"2022-10-05T04:43:01.360525Z","iopub.status.idle":"2022-10-05T04:43:01.368043Z","shell.execute_reply.started":"2022-10-05T04:43:01.360476Z","shell.execute_reply":"2022-10-05T04:43:01.366967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(INTERNAL_DATASET / 'train.csv')\ndf_train = df_train.merge(pd.read_csv(EXTERNAL_DATASET / 'folds.csv'), on='image_id', how='left')\ndf_test = pd.read_csv(INTERNAL_DATASET / 'test.csv')\n\ntrain_images = INTERNAL_DATASET / 'train'\ntest_images = INTERNAL_DATASET / 'test'\nother_images = INTERNAL_DATASET / 'other'\n\nprint(f'Training Set Shape: {df_train.shape} - Memory Usage: {df_train.memory_usage().sum() / 1024 ** 2:.2f} MB')\nprint(f'Test Set Shape: {df_test.shape} - Memory Usage: {df_test.memory_usage().sum() / 1024 ** 2:.2f} MB')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:43:01.369369Z","iopub.execute_input":"2022-10-05T04:43:01.369788Z","iopub.status.idle":"2022-10-05T04:43:01.432071Z","shell.execute_reply.started":"2022-10-05T04:43:01.369751Z","shell.execute_reply":"2022-10-05T04:43:01.430996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Extract aspect ratio and disk space of images\n* Create binary label","metadata":{}},{"cell_type":"code","source":"for idx, row in tqdm(df_train.iterrows(), total=df_train.shape[0]):\n    \n    image_path = f'{train_images}/{row[\"image_id\"]}.tif'\n    image_thumbnail = pyvips.Image.thumbnail(image_path, 1000)\n    df_train.loc[idx, 'aspect_ratio'] = image_thumbnail.width / image_thumbnail.height\n    df_train.loc[idx, 'image_filesize_megabytes'] = int(os.path.getsize(image_path)) / 1024 ** 2\n    \nfor idx, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):\n\n    image_path = f'{test_images}/{row[\"image_id\"]}.tif'\n    image_thumbnail = pyvips.Image.thumbnail(image_path, 1000)\n    df_test.loc[idx, 'aspect_ratio'] = image_thumbnail.width / image_thumbnail.height\n    df_test.loc[idx, 'image_filesize_megabytes'] = int(os.path.getsize(image_path)) / 1024 ** 2\n\ndf_train['image_count'] = df_train.groupby('patient_id')['image_num'].transform('max')\ndf_test['image_count'] = df_test.groupby('patient_id')['image_num'].transform('max')\n\ndf_train['binary_encoded_label'] = df_train['label'].map({'CE': 0, 'LAA': 1}).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:33.025452Z","iopub.execute_input":"2022-10-05T04:44:33.025915Z","iopub.status.idle":"2022-10-05T04:44:58.192094Z","shell.execute_reply.started":"2022-10-05T04:44:33.025875Z","shell.execute_reply":"2022-10-05T04:44:58.191023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Image Utilities","metadata":{}},{"cell_type":"code","source":"def resize_with_aspect_ratio(image, longest_edge):\n\n    \"\"\"\n    Resize image while preserving aspect ratio\n\n    Parameters\n    ----------\n    image (numpy.ndarray of shape (height, width, 3)): Image array\n    longest_edge (int): Number of pixels on the longest edge\n\n    Returns\n    -------\n    image (numpy.ndarray of shape (resized_height, resized_width, 3)): Resized image array\n    \"\"\"\n\n    height, width = image.shape[:2]\n    scale = longest_edge / max(height, width)\n    image = cv2.resize(image, dsize=(int(width * scale), int(height * scale)), interpolation=cv2.INTER_NEAREST)\n\n    return image\n\n\ndef tile_image(image, tile_size=256, n_tiles=4):\n\n    \"\"\"\n    Resize image while preserving aspect ratio\n\n    Parameters\n    ----------\n    image (numpy.ndarray of shape (height, width, channel)): Image array\n    tile_size (int): Number of pixels on the edges of tiles\n    n_tiles (int): Number of tiles\n\n    Returns\n    -------\n    image (numpy.ndarray of shape (n_tiles, tile_size, tile_size, channel)): Resized image array\n    \"\"\"\n\n    height, width, channel = image.shape\n    pad_height, pad_width = (tile_size - height % tile_size) % tile_size, (tile_size - width % tile_size) % tile_size\n    padding = [[pad_height // 2, pad_height - pad_height // 2], [pad_width // 2, pad_width - pad_width // 2], [0, 0]]\n    image = np.pad(image, padding, mode='constant', constant_values=255)\n    image = image.reshape(image.shape[0] // tile_size, tile_size, image.shape[1] // tile_size, tile_size, channel)\n    image = image.transpose(0, 2, 1, 3, 4).reshape(-1, tile_size, tile_size, channel)\n\n    if len(image) < n_tiles:\n        padding = [[0, n_tiles - len(image)], [0, 0], [0, 0], [0, 0]]\n        image = np.pad(image, padding, mode='constant', constant_values=255)\n\n    # Sort tiles by their sums and retrieve top n tiles with the highest sums\n    sorting_idx = np.argsort(image.reshape(image.shape[0], -1).sum(-1))[:n_tiles]\n    image = image[sorting_idx]\n\n    return image\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:08.313381Z","iopub.execute_input":"2022-10-05T04:44:08.314016Z","iopub.status.idle":"2022-10-05T04:44:08.331618Z","shell.execute_reply.started":"2022-10-05T04:44:08.313981Z","shell.execute_reply":"2022-10-05T04:44:08.330482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Metrics","metadata":{}},{"cell_type":"code","source":"def soft_predictions_to_labels(x, threshold):\n\n    \"\"\"\n    Convert soft predictions into hard labels in given array\n\n    Parameters\n    ----------\n    x (array-like of any shape): Soft predictions array\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n\n    Returns\n    -------\n    x (array-like of any shape): Hard labels array\n    \"\"\"\n\n    if isinstance(x, torch.Tensor):\n        x = x.numpy()\n    else:\n        x = np.array(x)\n\n    x = np.uint8(x >= threshold)\n\n    return x\n\n\ndef binary_weighted_log_loss1(y_true, y_pred):\n\n    \"\"\"\n    Calculate positive, negative and weighted log loss on predictions and ground-truth\n\n    Parameters\n    ----------\n    y_true (array-like of shape (n_samples)): Ground-truth\n    y_pred (array-like of shape (n_samples)): Predictions\n\n    Returns\n    -------\n    log_loss_positive (float): Positive log loss score calculated on predictions and ground-truth\n    log_loss_negative (float): Negative log loss score calculated on predictions and ground-truth\n    log_loss_weighted (float): Weighted log loss score calculated on predictions and ground-truth\n    \"\"\"\n\n    log_loss_positive = log_loss(y_true, y_pred)\n    log_loss_negative = log_loss(y_true, 1 - y_pred)\n    log_loss_weighted = 0.5 * log_loss_positive + 0.5 * log_loss_negative\n\n    return log_loss_positive, log_loss_negative, log_loss_weighted\n\n\ndef binary_weighted_log_loss2(y_true, y_pred):\n\n    \"\"\"\n    Calculate positive, negative and weighted log loss on predictions and ground-truth\n\n    Parameters\n    ----------\n    y_true (array-like of shape (n_samples)): Ground-truth\n    y_pred (array-like of shape (n_samples)): Predictions\n\n    Returns\n    -------\n    log_loss_positive (float): Positive log loss score calculated on predictions and ground-truth\n    log_loss_negative (float): Negative log loss score calculated on predictions and ground-truth\n    log_loss_weighted (float): Weighted log loss score calculated on predictions and ground-truth\n    \"\"\"\n\n    y_true_positive = y_true == 1\n    y_true_negative = y_true == 0\n\n    log_loss_positive = log_loss(y_true[y_true_positive], y_pred[y_true_positive], labels=[0, 1])\n    log_loss_negative = log_loss(y_true[y_true_negative], y_pred[y_true_negative], labels=[0, 1])\n    log_loss_weighted = 0.5 * log_loss_positive + 0.5 * log_loss_negative\n\n    return log_loss_positive, log_loss_negative, log_loss_weighted\n\n\ndef binary_classification_scores(y_true, y_pred, threshold):\n\n    \"\"\"\n    Calculate binary classification metrics on predictions and ground-truth\n\n    Parameters\n    ----------\n    y_true (array-like of shape (n_samples)): Ground-truth\n    y_pred (array-like of shape (n_samples)): Predictions\n    threshold (float): Threshold for converting soft predictions into hard labels (0 <= threshold <= 1)\n\n    Returns\n    -------\n    scores (dict): Dictionary of scores\n    \"\"\"\n\n    accuracy = accuracy_score(y_true, soft_predictions_to_labels(y_pred, threshold=threshold))\n    try:\n        roc_auc = roc_auc_score(y_true, y_pred)\n    except ValueError:\n        roc_auc = 0.5\n\n    log_loss_positive1, log_loss_negative1, log_loss_weighted1 = binary_weighted_log_loss1(y_true, y_pred)\n    log_loss_positive2, log_loss_negative2, log_loss_weighted2 = binary_weighted_log_loss2(y_true, y_pred)\n\n    scores = {\n        'accuracy': accuracy,\n        'roc_auc': roc_auc,\n        'log_loss_positive1': log_loss_positive1,\n        'log_loss_negative1': log_loss_negative1,\n        'log_loss_weighted1': log_loss_weighted1,\n        'log_loss_positive2': log_loss_positive2,\n        'log_loss_negative2': log_loss_negative2,\n        'log_loss_weighted2': log_loss_weighted2,\n    }\n\n    return scores\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:14.890785Z","iopub.execute_input":"2022-10-05T04:44:14.891308Z","iopub.status.idle":"2022-10-05T04:44:14.966856Z","shell.execute_reply.started":"2022-10-05T04:44:14.891245Z","shell.execute_reply":"2022-10-05T04:44:14.960364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Transforms","metadata":{}},{"cell_type":"code","source":"def get_classification_transforms(**transform_parameters):\n\n    \"\"\"\n    Get transforms for classification dataset\n\n    Parameters\n    ----------\n    transform_parameters (dict): Dictionary of transform parameters\n\n    Returns\n    -------\n    transforms (dict): Transforms for training, validation and test sets\n    \"\"\"\n\n    train_transforms = A.Compose([\n        A.Resize(\n            height=transform_parameters['resize_height'],\n            width=transform_parameters['resize_width'],\n            interpolation=cv2.INTER_NEAREST,\n            always_apply=True\n        ),\n        A.HorizontalFlip(p=transform_parameters['horizontal_flip_probability']),\n        A.VerticalFlip(p=transform_parameters['vertical_flip_probability']),\n        A.RandomRotate90(p=transform_parameters['random_rotate_90_probability']),\n        A.HueSaturationValue(\n            hue_shift_limit=transform_parameters['hue_shift_limit'],\n            sat_shift_limit=transform_parameters['saturation_shift_limit'],\n            val_shift_limit=transform_parameters['value_shift_limit'],\n            p=transform_parameters['hue_saturation_value_probability']\n        ),\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        A.OneOf([\n            A.CoarseDropout(\n                max_holes=transform_parameters['coarse_dropout_max_holes'],\n                max_height=transform_parameters['coarse_dropout_max_height'],\n                max_width=transform_parameters['coarse_dropout_max_width'],\n                min_holes=transform_parameters['coarse_dropout_min_holes'],\n                min_height=transform_parameters['coarse_dropout_min_height'],\n                min_width=transform_parameters['coarse_dropout_min_width'],\n                fill_value=transform_parameters['coarse_dropout_fill_value'],\n                p=transform_parameters['coarse_dropout_probability']\n            ),\n            A.PixelDropout(\n                dropout_prob=transform_parameters['pixel_dropout_dropout_probability'],\n                per_channel=transform_parameters['pixel_dropout_per_channel'],\n                drop_value=transform_parameters['pixel_dropout_drop_value'],\n                p=transform_parameters['pixel_dropout_probability']\n            )\n        ], p=transform_parameters['dropout_probability']),\n        ToTensorV2(always_apply=True)\n    ])\n\n    val_transforms = A.Compose([\n        A.Resize(\n            height=transform_parameters['resize_height'],\n            width=transform_parameters['resize_width'],\n            interpolation=cv2.INTER_NEAREST,\n            always_apply=True\n        ),\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        ToTensorV2(always_apply=True)\n    ])\n\n    test_transforms = A.Compose([\n        A.Resize(\n            height=transform_parameters['resize_height'],\n            width=transform_parameters['resize_width'],\n            interpolation=cv2.INTER_NEAREST,\n            always_apply=True\n        ),\n        A.Normalize(\n            mean=transform_parameters['normalize_mean'],\n            std=transform_parameters['normalize_std'],\n            max_pixel_value=transform_parameters['normalize_max_pixel_value'],\n            always_apply=True\n        ),\n        ToTensorV2(always_apply=True)\n    ])\n\n    transforms = {'train': train_transforms, 'val': val_transforms, 'test': test_transforms}\n    return transforms\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:16.122903Z","iopub.execute_input":"2022-10-05T04:44:16.123409Z","iopub.status.idle":"2022-10-05T04:44:16.137301Z","shell.execute_reply.started":"2022-10-05T04:44:16.123368Z","shell.execute_reply":"2022-10-05T04:44:16.136161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Models","metadata":{}},{"cell_type":"code","source":"def init_weights(module,\n                 linear_weight_init_type, linear_weight_init_args, linear_bias_init_type, linear_bias_init_args,\n                 batch_normalization_weight_init_type=None, batch_normalization_weight_init_args=None,\n                 batch_normalization_bias_init_type=None, batch_normalization_bias_init_args=None):\n\n    \"\"\"\n    Initialize weights and biases of given layers with specified configurations\n\n    Parameters\n    ----------\n    module (torch.nn.Module): Layer\n    linear_weight_init_type (str): Weight initialization method of the linear layer\n    linear_weight_init_args (dict): Weight initialization arguments of the linear layer\n    linear_bias_init_type (str): Bias initialization method of the linear layer\n    linear_bias_init_args (dict): Bias initialization arguments of the linear layer\n    batch_normalization_weight_init_type (str): Weight initialization method of the batch normalization layer\n    batch_normalization_weight_init_args (dict): Weight initialization arguments of the batch normalization layer\n    batch_normalization_bias_init_type (str): Bias initialization method of the batch normalization layer\n    batch_normalization_bias_init_args (dict): Bias initialization arguments of the batch normalization layer\n    \"\"\"\n\n    if isinstance(module, nn.Linear):\n        # Initialize weights of linear layer\n        if linear_weight_init_type == 'uniform':\n            nn.init.uniform_(\n                module.weight,\n                a=linear_weight_init_args['a'],\n                b=linear_weight_init_args['b']\n            )\n        elif linear_weight_init_type == 'normal':\n            nn.init.normal_(\n                module.weight,\n                mean=linear_weight_init_args['mean'],\n                std=linear_weight_init_args['std']\n            )\n        elif linear_weight_init_type == 'xavier_uniform':\n            nn.init.xavier_uniform_(\n                module.weight,\n                gain=nn.init.calculate_gain(\n                    nonlinearity=linear_weight_init_args['nonlinearity'],\n                    param=linear_weight_init_args['nonlinearity_param']\n                )\n            )\n        elif linear_weight_init_type == 'xavier_normal':\n            nn.init.xavier_normal_(\n                module.weight,\n                gain=nn.init.calculate_gain(\n                    nonlinearity=linear_weight_init_args['nonlinearity'],\n                    param=linear_weight_init_args['nonlinearity_param']\n                )\n            )\n        elif linear_weight_init_type == 'kaiming_uniform':\n            nn.init.kaiming_uniform_(\n                module.weight,\n                a=linear_weight_init_args['nonlinearity_param'],\n                mode=linear_weight_init_args['mode'],\n                nonlinearity=linear_weight_init_args['nonlinearity']\n            )\n        elif linear_weight_init_type == 'kaiming_normal':\n            nn.init.kaiming_normal_(\n                module.weight,\n                a=linear_weight_init_args['nonlinearity_param'],\n                mode=linear_weight_init_args['mode'],\n                nonlinearity=linear_weight_init_args['nonlinearity']\n            )\n        elif linear_weight_init_type == 'orthogonal':\n            nn.init.orthogonal_(\n                module.weight,\n                gain=nn.init.calculate_gain(\n                    nonlinearity=linear_weight_init_args['nonlinearity'],\n                    param=linear_weight_init_args['nonlinearity_param']\n                )\n            )\n        elif linear_weight_init_type == 'sparse':\n            nn.init.sparse_(\n                module.weight,\n                sparsity=linear_weight_init_args['sparsity'],\n                std=linear_weight_init_args['std']\n            )\n        # Initialize biases of Linear layer\n        if module.bias is not None:\n            if linear_bias_init_type == 'uniform':\n                nn.init.uniform_(\n                    module.bias,\n                    a=linear_bias_init_args['a'],\n                    b=linear_bias_init_args['b']\n                )\n            elif linear_bias_init_type == 'normal':\n                nn.init.normal_(\n                    module.bias,\n                    mean=linear_bias_init_args['mean'],\n                    std=linear_bias_init_args['std']\n                )\n\n    elif isinstance(module, nn.BatchNorm1d):\n        # Initialize weights of batch normalization layer\n        if batch_normalization_weight_init_type is not None:\n            if batch_normalization_weight_init_type == 'uniform':\n                nn.init.uniform_(\n                    module.weight,\n                    a=batch_normalization_weight_init_args['a'],\n                    b=batch_normalization_weight_init_args['b']\n                )\n            elif batch_normalization_weight_init_type == 'normal':\n                nn.init.normal_(\n                    module.weight,\n                    mean=batch_normalization_weight_init_args['mean'],\n                    std=batch_normalization_weight_init_args['std']\n                )\n            elif batch_normalization_weight_init_type == 'constant':\n                nn.init.constant_(\n                    module.weight,\n                    val=batch_normalization_weight_init_args['val'],\n                )\n        # Initialize biases of batch normalization layer\n        if batch_normalization_bias_init_type is not None:\n            if batch_normalization_bias_init_type == 'uniform':\n                nn.init.uniform_(\n                    module.bias,\n                    a=batch_normalization_bias_init_args['a'],\n                    b=batch_normalization_bias_init_args['b']\n                )\n            elif batch_normalization_bias_init_type == 'normal':\n                nn.init.normal_(\n                    module.bias,\n                    mean=batch_normalization_bias_init_args['mean'],\n                    std=batch_normalization_bias_init_args['std']\n                )\n            elif batch_normalization_bias_init_type == 'constant':\n                nn.init.constant_(\n                    module.bias,\n                    val=batch_normalization_bias_init_args['val'],\n                )","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:17.232273Z","iopub.execute_input":"2022-10-05T04:44:17.232734Z","iopub.status.idle":"2022-10-05T04:44:17.251063Z","shell.execute_reply.started":"2022-10-05T04:44:17.2327Z","shell.execute_reply":"2022-10-05T04:44:17.250077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvolutionalClassificationHead(nn.Module):\n\n    def __init__(self,\n                 input_features, intermediate_features, n_classes, pooling_type,\n                 activation, activation_args, dropout_probability=0., batch_normalization=False,\n                 initialization_args=None):\n\n        super(ConvolutionalClassificationHead, self).__init__()\n\n        self.pooling_type = pooling_type\n        self.classifier = nn.Sequential(\n            nn.Linear(input_features * 2 if pooling_type == 'concat' else input_features, intermediate_features, bias=True),\n            getattr(nn, activation)(**activation_args),\n            nn.BatchNorm1d(num_features=intermediate_features) if batch_normalization else nn.Identity(),\n            nn.Dropout(p=dropout_probability) if dropout_probability >= 0. else nn.Identity(),\n            nn.Linear(intermediate_features, n_classes, bias=True),\n            nn.Softmax(dim=-1) if n_classes > 1 else nn.Identity()\n        )\n\n        if initialization_args is not None:\n            for module in self.classifier:\n                if isinstance(module, nn.Linear) or isinstance(module, nn.BatchNorm1d):\n                    initialize_weights(module=module, **initialization_args)\n\n    def forward(self, x):\n\n        if self.pooling_type == 'avg':\n            x = F.adaptive_avg_pool2d(x, output_size=(1, 1)).view(x.size(0), -1)\n        elif self.pooling_type == 'max':\n            x = F.adaptive_max_pool2d(x, output_size=(1, 1)).view(x.size(0), -1)\n        elif self.pooling_type == 'concat':\n            x = torch.cat([\n                F.adaptive_avg_pool2d(x, output_size=(1, 1)).view(x.size(0), -1),\n                F.adaptive_max_pool2d(x, output_size=(1, 1)).view(x.size(0), -1)\n            ], dim=-1)\n\n        output = self.classifier(x)\n\n        return output\n\n\nclass ConvolutionalMultiInstanceLearningModel(nn.Module):\n\n    def __init__(self, n_instances, model_name, pretrained, freeze_parameters, head_class, head_args):\n\n        super(ConvolutionalMultiInstanceLearningModel, self).__init__()\n\n        self.backbone = timm.create_model(\n            model_name=model_name,\n            pretrained=pretrained,\n            num_classes=head_args['n_classes']\n        )\n\n        if freeze_parameters is not None:\n            # Freeze all parameters in backbone\n            if freeze_parameters == 'all':\n                for parameter in self.backbone.parameters():\n                    parameter.requires_grad = False\n            else:\n                # Freeze specified parameters in backbone\n                for group in freeze_parameters:\n                    if isinstance(self.backbone, timm.models.DenseNet):\n                        for parameter in self.backbone.features[group].parameters():\n                            parameter.requires_grad = False\n                    elif isinstance(self.backbone, timm.models.EfficientNet):\n                        for parameter in self.backbone.blocks[group].parameters():\n                            parameter.requires_grad = False\n\n        n_classifier_features = self.backbone.get_classifier().in_features\n        self.classification_head = eval(head_class)(input_features=n_classifier_features * n_instances, **head_args)\n\n    def forward(self, x):\n\n        # Stack instances on batch dimension before passing input to feature extractor\n        input_batch_size, input_instance, input_channel, input_height, input_width = x.shape\n        x = x.view(input_batch_size * input_instance, input_channel, input_height, input_width)\n        x = self.backbone.forward_features(x)\n\n        # Stack feature maps on channel dimension before passing feature maps to classification head\n        feature_batch_size, feature_channel, feature_height, feature_width = x.shape\n        x = x.contiguous().view(input_batch_size, feature_channel * input_instance, feature_height, feature_width)\n        output = self.classification_head(x)\n\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:17.986847Z","iopub.execute_input":"2022-10-05T04:44:17.988608Z","iopub.status.idle":"2022-10-05T04:44:18.00699Z","shell.execute_reply.started":"2022-10-05T04:44:17.98856Z","shell.execute_reply":"2022-10-05T04:44:18.005849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformerClassificationHead(nn.Module):\n\n    def __init__(self,\n                 input_features, intermediate_features, n_classes,\n                 activation, activation_args, dropout_probability=0., batch_normalization=False,\n                 initialization_args=None):\n\n        super(TransformerClassificationHead, self).__init__()\n\n        self.classifier = nn.Sequential(\n            nn.Linear(input_features, intermediate_features, bias=True),\n            getattr(nn, activation)(**activation_args),\n            nn.BatchNorm1d(num_features=intermediate_features) if batch_normalization else nn.Identity(),\n            nn.Dropout(p=dropout_probability) if dropout_probability >= 0. else nn.Identity(),\n            nn.Linear(intermediate_features, n_classes, bias=True),\n            nn.Softmax(dim=-1) if n_classes > 1 else nn.Identity()\n        )\n\n        if initialization_args is not None:\n            for module in self.classifier:\n                if isinstance(module, nn.Linear) or isinstance(module, nn.BatchNorm1d):\n                    initialize_weights(module=module, **initialization_args)\n\n    def forward(self, x):\n\n        output = self.classifier(x)\n\n        return output\n    \n\nclass TransformerMultiInstanceLearningModel(nn.Module):\n\n    def __init__(self, n_instances, model_name, pretrained, freeze_parameters, head_class, head_args):\n\n        super(TransformerMultiInstanceLearningModel, self).__init__()\n\n        self.backbone = timm.create_model(\n            model_name=model_name,\n            pretrained=pretrained,\n            num_classes=head_args['n_classes']\n        )\n\n        if freeze_parameters is not None:\n            # Freeze all parameters in backbone\n            if freeze_parameters == 'all':\n                for parameter in self.backbone.parameters():\n                    parameter.requires_grad = False\n\n        n_classifier_features = self.backbone.get_classifier().in_features\n        self.backbone.head = nn.Identity()\n        self.classification_head = eval(head_class)(input_features=n_classifier_features * n_instances, **head_args)\n\n    def forward(self, x):\n\n        # Stack instances on batch dimension before passing input to feature extractor\n        input_batch_size, input_instance, input_channel, input_height, input_width = x.shape\n        x = x.view(input_batch_size * input_instance, input_channel, input_height, input_width)\n        x = self.backbone(x)\n\n        # Stack feature maps on channel dimension before passing feature maps to classification head\n        feature_batch_size, feature_count = x.shape\n        x = x.contiguous().view(input_batch_size, feature_count * input_instance)\n        output = self.classification_head(x)\n\n        return output\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:18.842052Z","iopub.execute_input":"2022-10-05T04:44:18.842421Z","iopub.status.idle":"2022-10-05T04:44:18.855244Z","shell.execute_reply.started":"2022-10-05T04:44:18.842388Z","shell.execute_reply":"2022-10-05T04:44:18.854173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model_directory_path, folds=(1, 2, 3, 4, 5), verbose=False):\n    \n    \"\"\"\n    Load models and config file from the given directory path\n\n    Parameters\n    ----------\n    model_directory_path (str): Path of the model directory\n    verbose (bool): Verbosity flag\n    \n    Returns\n    -------\n    config (dict): Dictionary of model configurations\n    models (list): List of trained models\n    \"\"\"\n    \n    train_predictions = pd.read_csv(f'{model_directory_path}/train_predictions.csv')\n    config = yaml.load(open(f'{model_directory_path}/config.yaml', 'r'), Loader=yaml.FullLoader)\n    config['model_parameters']['model_args']['pretrained'] = False\n    model_paths = sorted(glob(f'{model_directory_path}/*.pt'))\n    models = {}\n    \n    oof_scores = binary_classification_scores(df_train['binary_encoded_label'], train_predictions['predictions'], threshold=0.5)\n    \n    print(f'\\n{model_directory_path.split(\"/\")[-1]} - OOF Scores {json.dumps(oof_scores, indent=2)}')\n    \n    for fold, model_path in enumerate(model_paths, 1):\n        \n        if fold not in folds:\n            continue\n        else:\n            model = eval(config['model_parameters']['model_class'])(**config['model_parameters']['model_args'])\n            model.load_state_dict(torch.load(model_path))\n            model = model.to(config['training_parameters']['device'])\n            model.eval()\n            models[fold] = model\n            print(f'Loaded pretrained weights from {model_path}')\n            \n    if verbose:\n        print(json.dumps(config, indent=2))\n        \n    return config, train_predictions, models\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:44:19.794959Z","iopub.execute_input":"2022-10-05T04:44:19.795542Z","iopub.status.idle":"2022-10-05T04:44:19.805706Z","shell.execute_reply.started":"2022-10-05T04:44:19.795506Z","shell.execute_reply":"2022-10-05T04:44:19.804624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mil_densenet121_16_256_config, mil_densenet121_16_256_train_predictions, mil_densenet121_16_256_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_densenet121_16_256',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_densenetblur121d_16_256_config, mil_densenetblur121d_16_256_train_predictions, mil_densenetblur121d_16_256_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_densenetblur121d_16_256',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_densenet169_16_256_config, mil_densenet169_16_256_train_predictions, mil_densenet169_16_256_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_densenet169_16_256',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_efficientnetb2_16_256_config, mil_efficientnetb2_16_256_train_predictions, mil_efficientnetb2_16_256_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_efficientnetb2_16_256',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_efficientnetv2rwt_16_256_config, mil_efficientnetv2rwt_16_256_train_predictions, mil_efficientnetv2rwt_16_256_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_efficientnetv2rwt_16_256',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_coatlitemini_16_224_config, mil_coatlitemini_16_224_train_predictions, mil_coatlitemini_16_224_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_coatlitemini_16_224',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_poolformer24_16_224_config, mil_poolformer24_16_224_train_predictions, mil_poolformer24_16_224_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_poolformer24_16_224',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)\n\nmil_swintinypatch4window7_16_224_config, mil_swintinypatch4window7_16_224_train_predictions, mil_swintinypatch4window7_16_224_models = load_model(\n    model_directory_path=f'{EXTERNAL_DATASET}/mil_swintinypatch4window7_16_224',\n    folds=(1, 2, 3, 4, 5),\n    verbose=False\n)","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:45:01.079511Z","iopub.execute_input":"2022-10-05T04:45:01.079871Z","iopub.status.idle":"2022-10-05T04:45:57.913216Z","shell.execute_reply.started":"2022-10-05T04:45:01.079841Z","shell.execute_reply":"2022-10-05T04:45:57.912167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"configs = [\n    mil_densenet121_16_256_config,\n    mil_densenetblur121d_16_256_config,\n    mil_densenet169_16_256_config,\n    mil_efficientnetb2_16_256_config,\n    mil_efficientnetv2rwt_16_256_config,\n    mil_coatlitemini_16_224_config,\n    mil_poolformer24_16_224_config,\n    mil_swintinypatch4window7_16_224_config\n]\n\nmodels = [\n    mil_densenet121_16_256_models,\n    mil_densenetblur121d_16_256_models,\n    mil_densenet169_16_256_models,\n    mil_efficientnetb2_16_256_models,\n    mil_efficientnetv2rwt_16_256_models,\n    mil_coatlitemini_16_224_models,\n    mil_poolformer24_16_224_models,\n    mil_swintinypatch4window7_16_224_models\n]\n\ntrain_predictions = [\n    mil_densenet121_16_256_train_predictions,\n    mil_densenetblur121d_16_256_train_predictions,\n    mil_densenet169_16_256_train_predictions,\n    mil_efficientnetb2_16_256_train_predictions,\n    mil_efficientnetv2rwt_16_256_train_predictions,\n    mil_coatlitemini_16_224_train_predictions,\n    mil_poolformer24_16_224_train_predictions,\n    mil_swintinypatch4window7_16_224_train_predictions\n]\n\nmodel_names = [\n    'mil_densenet121_16_256',\n    'mil_densenetblur121d_16_256',\n    'mil_densenet169_16_256',\n    'mil_efficientnetb2_16_256',\n    'mil_efficientnetv2rwt_16_256',\n    'mil_coatlitemini_16_224',\n    'mil_poolformer24_16_224',\n    'mil_swintinypatch4window7_16_224'\n]\n\nfor model_name, train_prediction in zip(model_names, train_predictions):\n    df_train[f'{model_name}_predictions'] = train_prediction['predictions']\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:47:50.630137Z","iopub.execute_input":"2022-10-05T04:47:50.63121Z","iopub.status.idle":"2022-10-05T04:47:50.644537Z","shell.execute_reply.started":"2022-10-05T04:47:50.631158Z","shell.execute_reply":"2022-10-05T04:47:50.643602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_predictions(df, prediction_column):\n\n    fig, ax = plt.subplots(figsize=(32, 8))\n    sns.kdeplot(df[prediction_column].values, label=prediction_column, fill=True, ax=ax)\n    ax.set_xlabel('')\n    ax.set_ylabel('')\n    ax.tick_params(axis='x', labelsize=12.5, pad=10)\n    ax.tick_params(axis='y', labelsize=12.5, pad=10)\n    ax.legend(prop={'size': 17.5})\n    \n    scores = binary_classification_scores(df['binary_encoded_label'], df[prediction_column], threshold=0.5)\n    title = f'''\n    {prediction_column}\n    Mean {df[prediction_column].mean():.4f} Std {df[prediction_column].std():.4f} Min {df[prediction_column].min():.4f} Max {df[prediction_column].max():.4f}\n    Accuracy {scores[\"accuracy\"]:.4f} ROC AUC {scores[\"roc_auc\"]:.4f}\n    1 - Log Loss Positive {scores[\"log_loss_positive1\"]:.4f} Negative {scores[\"log_loss_negative1\"]:.4f} Weighted {scores[\"log_loss_weighted1\"]:.4f}\n    2 - Log Loss Positive {scores[\"log_loss_positive2\"]:.4f} Negative {scores[\"log_loss_negative2\"]:.4f} Weighted {scores[\"log_loss_weighted2\"]:.4f}\n    '''\n    ax.set_title(title, size=20, pad=15)\n\n    plt.show()\n\n    \ndef visualize_predictions_correlations(df, prediction_columns):\n    \n    fig, ax = plt.subplots(figsize=(16, 16), dpi=100)\n    sns.heatmap(\n        df_train[prediction_columns + ['binary_encoded_label']].corr(),\n        annot=True,\n        square=True,\n        cmap='coolwarm',\n        annot_kws={'size': 12},\n        fmt='.4f'\n    )\n\n    plt.tick_params(axis='x', labelsize=10, rotation=90)\n    plt.tick_params(axis='y', labelsize=10, rotation=0)\n    plt.title('Prediction Correlations', size=20, pad=20)\n\n    plt.show()\n\n\nprediction_columns = [f'{model_name}_predictions' for model_name in model_names]\n\nfor prediction_column in prediction_columns:\n    visualize_predictions(df_train, prediction_column)\n\nvisualize_predictions_correlations(df_train, prediction_columns)","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:47:52.973388Z","iopub.execute_input":"2022-10-05T04:47:52.973764Z","iopub.status.idle":"2022-10-05T04:47:57.186352Z","shell.execute_reply.started":"2022-10-05T04:47:52.973732Z","shell.execute_reply":"2022-10-05T04:47:57.185429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Inference","metadata":{}},{"cell_type":"code","source":"# Resize longest to IMAGE_MAX_SIZE before loading images with pyvips\nIMAGE_MAX_SIZE = 20000\nDTYPE_MAPPING = {\n   'uchar': np.uint8,\n   'char': np.int8,\n   'ushort': np.uint16,\n   'short': np.int16,\n   'uint': np.uint32,\n   'int': np.int32,\n   'float': np.float32,\n   'double': np.float64,\n   'complex': np.complex64,\n   'dpcomplex': np.complex128,\n}\n\ndef vips_to_numpy(image_thumbnail):\n\n    return np.ndarray(\n        buffer=image_thumbnail.write_to_memory(),\n        dtype=DTYPE_MAPPING[image_thumbnail.format],\n        shape=[image_thumbnail.height, image_thumbnail.width, image_thumbnail.bands]\n    )\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:48:18.320679Z","iopub.execute_input":"2022-10-05T04:48:18.321043Z","iopub.status.idle":"2022-10-05T04:48:18.328619Z","shell.execute_reply.started":"2022-10-05T04:48:18.321011Z","shell.execute_reply":"2022-10-05T04:48:18.327233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for idx, row in tqdm(df_test.iterrows(), total=df_test.shape[0]):\n    \n    image_path = f'{test_images}/{row[\"image_id\"]}.tif'\n\n    if row['image_filesize_megabytes'] >= 250:\n        image_thumbnail = pyvips.Image.thumbnail(image_path, IMAGE_MAX_SIZE)\n        image = vips_to_numpy(image_thumbnail=image_thumbnail)\n        del image_thumbnail\n    else:\n        image = tifffile.imread(image_path)\n        image = resize_with_aspect_ratio(image, IMAGE_MAX_SIZE)\n\n    tiles = tile_image(\n        image=image,\n        tile_size=1024,\n        n_tiles=16\n    )\n    del image\n\n    for config, models_, model_name in zip(configs, models, model_names):\n        \n        test_transforms = get_classification_transforms(**config['transform_parameters'])['test']\n        \n        # Apply transforms to tiles and stack them along the batch dimension\n        inputs = [test_transforms(image=tile)['image'].float() for tile in tiles]\n        inputs = torch.stack(inputs, dim=0)\n        inputs = torch.unsqueeze(inputs, dim=0)\n        inputs = inputs.to('cuda')\n        \n        model_predictions = 0\n        \n        for model in models_.values():\n            with torch.no_grad():\n                outputs = model(inputs)\n\n            fold_predictions = outputs.detach().cpu()\n            fold_predictions = torch.sigmoid(torch.squeeze(fold_predictions, dim=1)).numpy().astype(np.float32)\n            model_predictions += (fold_predictions / len(models_))\n            \n        df_test.loc[idx, f'{model_name}_predictions'] = model_predictions\n","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:48:20.133482Z","iopub.execute_input":"2022-10-05T04:48:20.133843Z","iopub.status.idle":"2022-10-05T04:50:14.814359Z","shell.execute_reply.started":"2022-10-05T04:48:20.133811Z","shell.execute_reply":"2022-10-05T04:50:14.81278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7. Blending","metadata":{}},{"cell_type":"code","source":"blend_weights = {\n    'mil_densenet121_16_256': 0.10,\n    'mil_densenet169_16_256': 0.10,\n    'mil_densenetblur121d_16_256': 0.25,\n    'mil_efficientnetb2_16_256': 0.125,\n    'mil_efficientnetv2rwt_16_256': 0.125,\n    'mil_coatlitemini_16_224': 0.10,\n    'mil_poolformer24_16_224': 0.10,\n    'mil_swintinypatch4window7_16_224': 0.10\n}\n\ndf_train['blend_predictions'] = 0\ndf_test['blend_predictions'] = 0\n\nfor model_name, weight in blend_weights.items():\n    df_train['blend_predictions'] += (df_train[f'{model_name}_predictions'] * weight)\n    df_test['blend_predictions'] += (df_test[f'{model_name}_predictions'] * weight)\n    \nvisualize_predictions(df_train, 'blend_predictions')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:50:46.25898Z","iopub.execute_input":"2022-10-05T04:50:46.259479Z","iopub.status.idle":"2022-10-05T04:50:46.703226Z","shell.execute_reply.started":"2022-10-05T04:50:46.259434Z","shell.execute_reply":"2022-10-05T04:50:46.702304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8. Stacking","metadata":{}},{"cell_type":"code","source":"prediction_columns = [f'{model_name}_predictions' for model_name in model_names]\nlinear_model = LinearRegression()\nlinear_model.fit(df_train[prediction_columns], df_train['binary_encoded_label'])\ndf_train['stack_predictions'] = np.clip(linear_model.predict(df_train[prediction_columns]), a_min=0, a_max=1)\ndf_test['stack_predictions'] = np.clip(linear_model.predict(df_test[prediction_columns]), a_min=0, a_max=1)\n\nvisualize_predictions(df_train, 'stack_predictions')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:50:54.820616Z","iopub.execute_input":"2022-10-05T04:50:54.821004Z","iopub.status.idle":"2022-10-05T04:50:55.158397Z","shell.execute_reply.started":"2022-10-05T04:50:54.820968Z","shell.execute_reply":"2022-10-05T04:50:55.157469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Submission","metadata":{}},{"cell_type":"code","source":"submission_predictions = 'stack_predictions'\ndf_train['adjusted_predictions'] = df_train[submission_predictions] + 0.21\ndf_train_aggregated = df_train.groupby('patient_id')[['adjusted_predictions', 'binary_encoded_label']].mean()\n\nvisualize_predictions(df_train_aggregated, 'adjusted_predictions')","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:51:07.152681Z","iopub.execute_input":"2022-10-05T04:51:07.153043Z","iopub.status.idle":"2022-10-05T04:51:07.556641Z","shell.execute_reply.started":"2022-10-05T04:51:07.153012Z","shell.execute_reply":"2022-10-05T04:51:07.555685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['adjusted_predictions'] = df_test[submission_predictions] + 0.21\ndf_test['LAA'] = df_test['adjusted_predictions']\ndf_test['CE'] = 1 - df_test['LAA']\ndf_submission = df_test[['patient_id', 'CE', 'LAA']].groupby('patient_id').mean().reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:51:19.544073Z","iopub.execute_input":"2022-10-05T04:51:19.544448Z","iopub.status.idle":"2022-10-05T04:51:19.557814Z","shell.execute_reply.started":"2022-10-05T04:51:19.544412Z","shell.execute_reply":"2022-10-05T04:51:19.556674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)\ndf_submission","metadata":{"execution":{"iopub.status.busy":"2022-10-05T04:51:19.877493Z","iopub.execute_input":"2022-10-05T04:51:19.877797Z","iopub.status.idle":"2022-10-05T04:51:19.896622Z","shell.execute_reply.started":"2022-10-05T04:51:19.877769Z","shell.execute_reply":"2022-10-05T04:51:19.895576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}