{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nInference Pipeline\n\"\"\"\n\n# {{{ Module Imports for all models\n\nimport sys\n#sys.path.append('/kaggle/input/pretrainedmodels/pretrainedmodels-0.7.4')\n#sys.path.append('/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch-master')\n#sys.path.append('/kaggle/input/timm-pytorch-image-models/pytorch-image-models-master')\n#sys.path.append('/kaggle/input/segmentation-models-pytorch/segmentation_models.pytorch-master')\n#sys.path.append('/kaggle/input/segmentation-models-pytorch-v2')\n#sys.path.append('/kaggle/input/resnet3d')\n\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"../input/efficientnet-pytorch\")\nsys.path.append(\"/kaggle/input/smp-github/segmentation_models.pytorch-master\")\nsys.path.append(\"/kaggle/input/timm-pretrained-resnest/resnest/\")\n\n# Generic Imports\nimport pickle\nimport warnings\nimport pandas as pd\nimport os\nimport gc\nimport sys\nimport math\nimport time\nimport random\nimport datetime\nimport importlib\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nfrom functools import partial\nimport hashlib\nfrom typing import Optional, List\n\n# Computer Vision\nimport cv2\nimport PIL.Image as Image\nimport imageio\n\n# ML modules\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score, accuracy_score, f1_score, log_loss, fbeta_score\n\n# Torch\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.cuda.amp import autocast, GradScaler\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\nimport torchvision.transforms as T\nfrom torchmetrics.functional import dice\n\n# Pytorch Lightning\nimport pytorch_lightning as pl\n\n# TIMM\nimport timm\n#from timm.models.resnet import resnet34d\nfrom timm import create_model\n\n# Plotting\nimport matplotlib.pyplot as plt\n\n# Monitoring\nfrom tqdm.auto import tqdm\n\n# Data Augmentation\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\n# Logging\nfrom logging import getLogger, INFO, FileHandler, Formatter, StreamHandler\n\n# Model Imports\nimport segmentation_models_pytorch as smp\nfrom segmentation_models_pytorch.decoders.unet.decoder import UnetDecoder\nfrom segmentation_models_pytorch.encoders import get_encoder\n#from resnet3d import generate_model \n\n# For importing pre-trained model. See https://github.com/pytorch/pytorch/issues/33288\nimport ssl\nssl._create_default_https_context = ssl._create_unverified_context\n\nclass Conv2dBnAct(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, padding=0,\n                 stride=1, act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):\n        super().__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=False)\n        self.bn = norm_layer(out_channels)\n        self.act = act_layer(inplace=True)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.act(x)\n        return x\n\n\nclass DecoderBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, scale_factor=2.0, act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):\n        super().__init__()\n        conv_args = dict(kernel_size=3, padding=1, act_layer=act_layer)\n        self.scale_factor = scale_factor\n        if norm_layer is None:\n            self.conv1 = Conv2dBnAct(in_channels, out_channels, **conv_args)\n            self.conv2 = Conv2dBnAct(out_channels, out_channels,  **conv_args)\n        else:\n            self.conv1 = Conv2dBnAct(in_channels, out_channels, norm_layer=norm_layer, **conv_args)\n            self.conv2 = Conv2dBnAct(out_channels, out_channels, norm_layer=norm_layer, **conv_args)\n\n    def forward(self, x, skip: Optional[torch.Tensor] = None):\n        if self.scale_factor != 1.0:\n            x = F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n        if skip is not None:\n            x = torch.cat([x, skip], dim=1)\n        x = self.conv1(x)\n        x = self.conv2(x)\n        return x\n        \nclass UnetDecoder(nn.Module):\n\n    def __init__(\n            self,\n            encoder_channels,\n            decoder_channels=(256, 128, 64, 32, 16),\n            final_channels=1,\n            norm_layer=nn.BatchNorm2d,\n            center=False,\n    ):\n        super().__init__()\n\n        if center:\n            channels = encoder_channels[0]\n            self.center = DecoderBlock(channels, channels, scale_factor=1.0, norm_layer=norm_layer)\n        else:\n            self.center = nn.Identity()\n\n        in_channels = [in_chs + skip_chs for in_chs, skip_chs in zip(\n            [encoder_channels[0]] + list(decoder_channels[:-1]),\n            list(encoder_channels[1:]) + [0])]\n        out_channels = decoder_channels\n\n        self.blocks = nn.ModuleList()\n        for in_chs, out_chs in zip(in_channels, out_channels):\n            self.blocks.append(DecoderBlock(in_chs, out_chs, norm_layer=norm_layer))\n        self.final_conv = nn.Conv2d(out_channels[-1], final_channels, kernel_size=(1, 1))\n\n        self._init_weight()\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def forward(self, x: List[torch.Tensor]):\n        encoder_head = x[0]\n        skips = x[1:]\n        x = self.center(encoder_head)\n        for i, b in enumerate(self.blocks):\n            skip = skips[i] if i < len(skips) else None\n            x = b(x, skip)\n        x = self.final_conv(x)\n        return x\n    \nclass UnetSuperVision(nn.Module):\n    \"\"\"Unet is a fully convolution neural network for image semantic segmentation\n    Args:\n        encoder_name: name of classification model (without last dense layers) used as feature\n            extractor to build segmentation model.\n        encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).\n        decoder_channels: list of numbers of ``Conv2D`` layer filters in decoder blocks\n        decoder_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers\n            is used.\n        num_classes: a number of classes for output (output shape - ``(batch, classes, h, w)``).\n        center: if ``True`` add ``Conv2dReLU`` block on encoder head\n    NOTE: This is based off an old version of Unet in https://github.com/qubvel/segmentation_models.pytorch\n    \"\"\"\n\n    def __init__(\n            self,\n            backbone='resnet50',\n            backbone_kwargs=None,\n            backbone_indices=None,\n            decoder_use_batchnorm=True,\n            decoder_channels=(256, 128, 64, 32, 16),\n            in_chans=3,\n            num_classes=1,\n            center=True,\n            norm_layer=nn.BatchNorm2d,\n    ):\n        super().__init__()\n        backbone_kwargs = backbone_kwargs or {}\n        # NOTE some models need different backbone indices specified based on the alignment of features\n        # and some models won't have a full enough range of feature strides to work properly.\n        encoder = create_model(\n            backbone, features_only=True, out_indices=backbone_indices, in_chans=in_chans,\n            pretrained=False, **backbone_kwargs)\n        encoder_channels = encoder.feature_info.channels()[::-1]\n        self.encoder = encoder\n\n        if not decoder_use_batchnorm:\n            norm_layer = None\n        self.decoder = UnetDecoder(\n            encoder_channels=encoder_channels,\n            decoder_channels=decoder_channels,\n            final_channels=num_classes,\n            norm_layer=norm_layer,\n            center=center,\n        )\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(encoder.feature_info.channels()[-1], 1)\n\n    def forward(self, x: torch.Tensor):\n        x = self.encoder(x)\n        x.reverse()  # torchscript doesn't work with [::-1]\n        x_cls = self.avgpool(x[0])\n        x_cls = x_cls.view(x_cls.size(0), -1)\n        x_cls = self.fc(x_cls).view(x_cls.size(0))\n        x = self.decoder(x)\n        #return {\"x\":x,\"x_cls\":x_cls}\n        return x\n\n#  Core Config for all models\nclass Config:\n\n    # === Core Paths === \n    comp_name = 'contrails-identification'\n    root_path = '/kaggle/input'\n\n    target_size = 1 # target classes\n\n    num_workers = 2\n    seed = 38\n    use_th_search = False\n\n    device_ids = [0,1]\n\n# }}} \n\n# {{{ Dataset\n\ndef get_transforms(cfg):\n\n    aug = A.Compose([\n        A.Resize(cfg.image_size, cfg.image_size),\n        #A.Normalize(\n        #    mean= [0] * cfg.in_chans,\n        #    std= [1] * cfg.in_chans\n        #),\n        ToTensorV2(transpose_mask=True),\n    ])\n\n    return aug\n\nclass ContrailsDataset(torch.utils.data.Dataset):\n    def __init__(self, df, image_size=256, transform=None, mode='test'):\n        \n        self.df = df\n        self.df_idx: pd.DataFrame = pd.DataFrame({'idx': os.listdir(cfg.root_path + f'/google-research-identify-contrails-reduce-global-warming/test')})\n        self.normalize_image = T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n        self.image_size = image_size\n        if image_size != 256:\n            self.resize_image = T.transforms.Resize(image_size)\n        self.transform = transform \n        self.mode = mode\n    \n    def read_record(self, directory):\n        record_data = {}\n        for x in [\n            \"band_11\", \n            \"band_14\", \n            \"band_15\"\n        ]:\n\n            record_data[x] = np.load(os.path.join(directory, x + \".npy\"))\n\n        return record_data\n\n    def normalize_range(self, data, bounds):\n        \"\"\"Maps data to the range [0, 1].\"\"\"\n        return (data - bounds[0]) / (bounds[1] - bounds[0])\n    \n    def get_false_color(self, record_data):\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n        \n        N_TIMES_BEFORE = 4\n\n        r = self.normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n        g = self.normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n        b = self.normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n        false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n        img = false_color[..., N_TIMES_BEFORE]\n\n        return img\n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        con_path = row.path\n        data = self.read_record(con_path)    \n        \n        img = self.get_false_color(data)\n\n        ## Convert to float32 in the range [0,255]\n        #img = (img * 255).astype(np.float32)\n\n        #data = self.transform(image=img)\n        #img = data['image']\n\n        ## Convert back to original format and range\n        #img = img / 255.0\n        #\n        ##img = torch.tensor(np.reshape(img, (256, 256, 3))).to(torch.float32).permute(2, 0, 1)\n        #\n        ##if self.image_size != 256:\n        ##    img = self.resize_image(img)\n        #\n        #img = self.normalize_image(img)\n        #\n        #image_id = int(self.df_idx.iloc[index]['idx'])\n        #    \n        #return img.float(), torch.tensor(image_id)\n\n        if cfg.etl_version == '0.1':\n\n            img = (img * 255).astype(np.float32)\n\n            transformed = self.transform(image=img) # Resize and to Pytorch Tensor \n            img = transformed['image'] \n\n            # Convert back to original format and range\n            img = img / 255.0\n\n            if not cfg.pytorch_lightning:\n                img = self.normalize_image(img)\n\n            image_id = int(self.df_idx.iloc[index]['idx'])\n                \n            return img.float(), torch.tensor(image_id)\n\n        elif cfg.etl_version == '0.2':\n\n            img = torch.tensor(np.reshape(img, (256, 256, 3))).to(torch.float32).permute(2, 0, 1)\n            \n            if self.image_size != 256:\n                img = self.resize_image(img)\n            \n            img = self.normalize_image(img)\n            \n            image_id = int(self.df_idx.iloc[index]['idx'])\n                \n            return img.float(), torch.tensor(image_id)\n    \n    def __len__(self):\n        return len(self.df)\n\ndef make_test_dataset():\n\n    test_dir = cfg.root_path + \"/google-research-identify-contrails-reduce-global-warming/test/\"\n    test_files = os.listdir(test_dir)\n    test_df = pd.DataFrame(test_files, columns=['record_id'])\n    test_df['path'] = test_dir + test_df['record_id'].astype(str)\n\n    test_dataset = ContrailsDataset(test_df, cfg.image_size, transform=get_transforms(cfg=cfg), mode='test')\n    test_loader = DataLoader(test_dataset, batch_size=cfg.batch_size, num_workers=cfg.num_workers)\n\n    return test_loader\n\ndef TTA(x: torch.Tensor, model: nn.Module):\n    #x.shape=(batch,c,h,w)\n\n    shape=x.shape\n\n    if cfg.use_flip_tta:\n        # Horizontal Flip\n        x_hf = torch.flip(model(torch.flip(x,[3])),[3]).permute(1,0,2,3)\n\n        # Vertical Flip\n        x_vf = torch.flip(model(torch.flip(x,[2])),[2]).permute(1,0,2,3)\n\n    # 90 Degree Rotation TTA\n    x = [x,*[torch.rot90(x,k=i,dims=(-2,-1)) for i in range(1,4)]]\n    x = torch.cat(x,dim=0)\n\n    x = model(x)\n\n    if cfg.use_all_masks and not cfg.randomize_masks:\n        x = x[:,0,:,:]\n\n    x = torch.sigmoid(x)\n    x = x.reshape(4,shape[0],*shape[2:])\n    x = [torch.rot90(x[i],k=-i,dims=(-2,-1)) for i in range(4)]\n\n    x = torch.stack(x,dim=0)\n\n    if cfg.use_flip_tta:\n        x = torch.cat((x_hf,x_vf,x), dim=0)\n\n    x = x.mean(0) \n    x = x.unsqueeze(1)\n\n    return x\n\n\n# }}}\n\n# Metrics \n\ndef rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s\n# }}} \n\n# {{{ Model \n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, weight=None):\n        super().__init__()\n        self.cfg = cfg\n\n        if cfg.use_all_masks and not cfg.randomize_masks:\n            target_size = 5\n        else:\n            target_size = 1\n\n        if cfg.backbone == \"resnet3d\":\n            self.encoder = Resnet3dUnet(model_depth=cfg.model_depth)\n        else:\n\n            self.encoder = smp.Unet(\n                encoder_name=cfg.backbone,\n                encoder_weights=weight,\n                in_channels=cfg.in_chans,\n                classes=target_size,\n                activation=None,\n            )\n\n    def forward(self, image):\n\n        output = self.encoder(image)\n        #output = output.squeeze(-1)\n\n        return output\n\nclass ContrailModel(pl.LightningModule):\n\n    def __init__(self):\n        super().__init__()\n\n        self.CONFIG = dict (\n            config_name = \"e1-efficientnetv2_l_in21k-bestthreshold-flipsrotandbig_4tta\",\n            num_epochs = 1,\n            batch_size = 1,\n            img_size = 512,\n            threshold = 0.32,\n            loss = \"BCEWithLogitsLoss\",\n            optimizer = \"AdamW\",\n            lr = 0.0003,\n            scheduler = \"CosineAnnealingLR10\",\n            tta = \"4tta_rot\",\n            val_alb = \"resize_and_tensor\",\n            architecture = \"unet\",\n            encoder_name = \"tu-tf_efficientnetv2_l.in21k_ft_in1k\",\n        )\n\n        self.model = smp.Unet(self.CONFIG['encoder_name'], \n                         encoder_weights=None, \n                         classes=1, \n                         activation=None, \n                         encoder_depth=5)      \n        self.val_targets_step_outputs = []\n        self.val_predictions_step_outputs = []\n        self.test_results = []\n        self.threshold = self.CONFIG[\"threshold\"] # 0.32\n\n    def forward(self, x):\n        return self.model(x)\n\n    def test_step(self, batch, batch_idx):\n        images, file_names = batch\n\n        if (config.CONFIG[\"tta\"]==\"4tta_rot\"):\n            predictions = self._4tta_rot_test(images)\n        else:\n            print(\"Error. Unsuported tta\")\n\n        for i, file_name in enumerate(file_names):\n            prediction = predictions[i].detach().cpu()\n\n\t\t\t# Resize prediction back to 256\n            prediction = TF.resize(prediction, (256, 256))\n            mask = (prediction >= self.threshold).int() \n            mask = mask[0]\n            self.test_results.append([\n                file_name,\n                list_to_string(rle_encode(mask))                \n            ])\n\n        return {\"predictions\": predictions}\n\n        \n    def _4tta_rot_test(self, images):\n\n        outputs = self(images)\n        predictions = F.sigmoid(outputs)\n\n        # Horizontal flip\n        outputs = self(torch.flip(images, dims=[3]))\n        # Flipping predictions back\n        predictions += F.sigmoid(torch.flip(outputs, dims=[3]))\n\n        # Rotate 90\n        outputs = self(torch.rot90(images, k=1, dims=[2,3]))\n        # Rotate predictions back\n        predictions += F.sigmoid(torch.rot90(outputs, k=-1, dims=[2,3]))\n\n        # Rotate 270\n        outputs = self(torch.rot90(images, k=3, dims=[2,3]))\n        # Rotate predictions back\n        predictions += F.sigmoid(torch.rot90(outputs, k=-3, dims=[2,3]))\n\n        predictions /= 4\n\n        return predictions\n\n        \n    def _standard_validation(self, images, targets):\n        outputs = self(images)  \n        loss = self.criterion(outputs, targets)\n        predictions = F.sigmoid(outputs)\n        return loss, predictions\n        \n\n\ndef build_model(cfg, model_path):\n    print('model_name', cfg.model_name)\n    print('backbone', cfg.backbone)\n\n    if cfg.pytorch_lightning:\n        print('Load Pytorch Lightning Module')\n        model = ContrailModel.load_from_checkpoint(model_path)\n    elif 'vit' in cfg.model_name or 'tf_efficientnetv2_xl_in21ft1k' in cfg.model_name:\n        model = UnetSuperVision(backbone=cfg.backbone)\n        print(f'Load model from path: {model_path}')\n        model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\n    elif cfg.model_name.split('-')[0] == 'maxvit' or cfg.model_name.split('-')[0] == 'maxxvitsupervision' or cfg.model_name.split('-')[0] == 'maxvitlargecpt21':\n        model = UnetSuperVision(backbone=cfg.backbone)\n        print(f'Load model from path: {model_path}')\n        model.load_state_dict(torch.load(model_path, map_location=\"cpu\"))\n    else:\n        model = CustomModel(cfg)\n        print(f'Load model from path: {model_path}')\n        model.load_state_dict(torch.load(model_path, map_location=\"cpu\")['model'])\n\n    return model\n\nclass EnsembleModel(nn.Module):\n    def __init__(self, weights=None):\n        super().__init__()\n        self.models = nn.ModuleList()\n        self.weights = torch.tensor(weights)[:,None,None,None,None]\n\n    def __call__(self, x):\n        \"\"\"\n        Weighted average of models\n        \"\"\"\n        x = [model(x) for model in self.models]\n        x = torch.stack(x, dim=0)\n        x = x * self.weights.to(device=x.device)\n        return torch.sum(x, dim=0)\n\n    def add_model(self, model):\n        self.models.append(model)\n\ndef build_ensemble_model(cfg, model_path, cv_folds, weights):\n\n    model = EnsembleModel(weights)\n\n    for fold in cv_folds:\n        path = model_path.format(fold=fold, backbone=cfg.backbone)\n        _model = build_model(cfg, path)\n        model.add_model(_model)\n    \n    return model\n# }}}\n\n# Main\n\n# Global Config\ncfg = Config\n\nmode = 'test'\n\nmodel_template_list = [\n    {\n    'alias': 'unet-maxvit_base_tf_512.in21k_ft_in1k-32epochs_pseudo',\n    'weight': 1,\n    'config': {\n        'model_name': 'unet-maxvit_base_tf_512.in21k_ft_in1k-32epochs_pseudo',\n        'backbone': 'maxvit_base_tf_512.in21k_ft_in1k',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': False,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/maxvit-pseudo-512/unet-maxvit_base_tf_512.in21k_ft_in1k-32epochs_pseudo_best_fold0 (1).pth'\n        }\n    }\n},{\n    'alias': 'unet-maxvit_large_tf_512.in21k_ft_in1k-32epochs_pseudo',\n    'weight': 3,\n    'config': {\n        'model_name': 'unet-maxvit_large_tf_512.in21k_ft_in1k-32epochs_pseudo',\n        'backbone': 'maxvit_large_tf_512.in21k_ft_in1k',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': False,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/maxvit-pseudo-512/unet-maxvit_large_tf_512.in21k_ft_in1k-32epochs_pseudo_best_fold0.pth'\n        }\n    }\n},{\n    'alias': 'unet-tf_efficientnetv2_xl_in21ft1k-32epochs_pseudo',\n    'weight': 3,\n    'config': {\n        'model_name': 'unet-tf_efficientnetv2_xl_in21ft1k-32epochs_pseudo',\n        'backbone': 'tf_efficientnetv2_xl_in21ft1k',\n        'in_chans': 3,\n        'image_size': 640,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': False,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/maxvit-pseudo-512/unet-tf_efficientnetv2_xl_in21ft1k-32epochs_pseudo_best_fold0_697.pth'\n        }\n    }\n},{\n    'alias': 'tu-tf_efficientnetv2_l',\n    'weight': 8,\n    'config': {\n        'model_name': 'tu-tf_efficientnetv2_l-run31-epoch20',\n        'backbone': 'tu-tf_efficientnetv2_l.in21k_ft_in1k',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': True,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/weights-contrail-run31-epoch20/r31-e20-t28-model.ckpt'\n        }\n    }\n}, {\n    'alias': 'tu-tf_efficientnetv2_l',\n    'weight': 7,\n    'config': {\n        'model_name': 'tu-tf_efficientnetv2_l-run32-epoch17',\n        'backbone': 'tu-tf_efficientnetv2_l.in21k_ft_in1k',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': True,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/weights-contrail-run32-epoch17/r32-e17-t32-model.ckpt'\n        }\n    }\n}, {\n    'alias': 'tu-tf_efficientnetv2_l',\n    'weight': 6,\n    'config': {\n        'model_name': 'tu-tf_efficientnetv2_l-run33-epoch16',\n        'backbone': 'tu-tf_efficientnetv2_l.in21k_ft_in1k',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': True,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/weights-contrail-run33-epoch16/r33-e16-t38-model.ckpt'\n        }\n    }\n#}, {\n#    'alias': 'maxvitlargecpt21',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvitlargecpt21-fold0',\n#        'backbone': 'tf_efficientnetv2_l_in21ft1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold0.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold0.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvitlargecpt21',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvitlargecpt21-fold1',\n#        'backbone': 'tf_efficientnetv2_l_in21ft1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold1.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold1.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvitlargecpt21',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvitlargecpt21-fold2',\n#        'backbone': 'tf_efficientnetv2_l_in21ft1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold2.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold2.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvitlargecpt21',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvitlargecpt21-fold3',\n#        'backbone': 'tf_efficientnetv2_l_in21ft1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold3.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvitlargecpt21/unet-tf_efficientnetv2_l_in21ft1k_best_fold3.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxvitsupervision',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxvitsupervision-fold0',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold0.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold0.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxvitsupervision',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxvitsupervision-fold1',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold1.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold1.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxvitsupervision',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxvitsupervision-fold2',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold2.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold2.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxvitsupervision',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxvitsupervision-fold3',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold3.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxvitsupervision/unet-maxxvitv2_rmlp_base_rw_384_aug_best_fold3.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxunetcpt21epoch',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxunetcpt21epoch-fold0',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold0.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold0.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxunetcpt21epoch',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxunetcpt21epoch-fold1',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold1.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold1.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxunetcpt21epoch',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxunetcpt21epoch-fold2',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold2.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold2.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxunetcpt21epoch',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxunetcpt21epoch-fold3',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold3.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold3.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxxunetcpt21epoch',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxxunetcpt21epoch-fold4',\n#        'backbone': 'maxxvitv2_rmlp_base_rw_384',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold4.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxxunetcpt21epoch/unet-maxxvitv2_rmlp_base_rw_384_best_fold4.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvit-rmlp-384',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvit-rmlp-384-fold0',\n#        'backbone': 'maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold0.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold0.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvit-rmlp-384',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvit-rmlp-384-fold1',\n#        'backbone': 'maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold1.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold1.pth'\n#        }\n#    }\n#}, {\n#    'alias': 'maxvit-rmlp-384',\n#    'weight': 1,\n#    'config': {\n#        'model_name': 'maxvit-rmlp-384-fold2',\n#        'backbone': 'maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k',\n#        'in_chans': 3,\n#        'image_size': 384,\n#        'batch_size': 16,\n#        'inference_mode': 'eval',\n#        'use_tta': True,\n#        'use_flip_tta': True,\n#        'use_all_masks': False,\n#        'randomize_masks': False,\n#        'etl_version': '0.2',\n#        'pytorch_lightning': False,\n#        'model': {\n#            'ensemble': False,\n#            #'model_path': '/kaggle/input/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold2.pth'\n#            'model_path': '/home/flyyufelix/Documents/machine_learning/competitions/kaggle/contrails_identification/orkatz_models/maxvit-rmlp-384-cpt/unet-maxvit_rmlp_base_rw_384.sw_in12k_ft_in1k_best_fold2.pth'\n#        }\n#    }\n# }, {\n#     'alias': 'segformer-b3-rand-labels-exp042',\n#     'weight': 2,\n#     'config': {\n#         'model_name': 'mit-b3-rand-labels-exp042',\n#         'backbone': 'mit_b3',\n#         'in_chans': 3,\n#         'image_size': 512,\n#         'batch_size': 16,\n#         'inference_mode': 'eval',\n#         'use_tta': True,\n#         'use_flip_tta': False,\n#         'use_all_masks': False,\n#         'randomize_masks': False,\n#         'etl_version': '0.1',\n#         'pytorch_lightning': False,\n#         'model': {\n#             'ensemble': False,\n#             'model_path': '/kaggle/input/unet-mit-b3-exp042-model/Unet_mit_b3_last.pth'\n#         }\n#     }\n}, {\n    'alias': 'segformer-b3-all-labels-exp043',\n    'weight': 1,\n    'config': {\n        'model_name': 'mit-b3-all-labels-exp043',\n        'backbone': 'mit_b3',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': True,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-mit-b3-exp043-model/Unet_mit_b3_last.pth'\n        }\n    }\n# }, {\n#     'alias': 'segformer-b3-pseudo-labels-exp053',\n#     'weight': 2,\n#     'config': {\n#         'model_name': 'mit-b3-pseudo-labels-exp053',\n#         'backbone': 'mit_b3',\n#         'in_chans': 3,\n#         'image_size': 512,\n#         'batch_size': 16,\n#         'inference_mode': 'eval',\n#         'use_tta': True,\n#         'use_flip_tta': False,\n#         'use_all_masks': False,\n#         'randomize_masks': False,\n#         'etl_version': '0.1',\n#         'pytorch_lightning': False,\n#         'model': {\n#             'ensemble': False,\n#             'model_path': '/kaggle/input/unet-mit-b3-exp053-model/Unet_mit_b3_last.pth'\n#         }\n#     }\n# }, {\n#     'alias': 'efficientNet-b7-pseudo-labels-exp054',\n#     'weight': 1,\n#     'config': {\n#        'model_name': 'efficientNet-b7-pseudo-labels-exp054',\n#         'backbone': 'efficientnet-b7',\n#         'in_chans': 3,\n#         'image_size': 512,\n#         'batch_size': 16,\n#         'inference_mode': 'eval',\n#         'use_tta': True,\n#         'use_flip_tta': False,\n#         'use_all_masks': False,\n#         'randomize_masks': False,\n#         'etl_version': '0.1',\n#         'pytorch_lightning': False,\n#         'model': {\n#             'ensemble': False,\n#             'model_path': '/kaggle/input/unet-efficientnetb7-exp054-epoch70-model/Unet_efficientnet-b7_epoch70.pth'\n#         }\n#     }\n}, {\n    'alias': 'segformer-b3-768-exp45',\n    'weight': 1,\n    'config': {\n        'model_name': 'mit-b3-768-exp45',\n        'backbone': 'mit_b3',\n        'in_chans': 3,\n        'image_size': 768,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-mit-b3-exp045-model/Unet_mit_b3_last.pth'\n        }\n    }\n}, {\n    'alias': 'efficientNet-b7-all-labels-exp065',\n    'weight': 1,\n    'config': {\n       'model_name': 'efficientNet-b7-all-labels-exp065',\n        'backbone': 'efficientnet-b7',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': True,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-efficientnetb7-exp065-model/Unet_efficientnet-b7_ema.pth'\n        }\n    }\n}, {\n    'alias': 'segformer-b3-valid-data-exp038',\n    'weight': 1,\n    'config': {\n        'model_name': 'mit-b3-valid-data-exp038',\n        'backbone': 'mit_b3',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-mit-b3-exp038-model/Unet_mit_b3_epoch70.pth'\n        }\n    }\n# }, {\n#     'alias': 'segformer-b3-rand071-pseudo-labels-exp081', \n#     'weight': 1,\n#     'config': {\n#         'model_name': 'mit-b3-rand071-pseudo-labels-exp081',\n#         'backbone': 'mit_b3',\n#         'in_chans': 3,\n#         'image_size': 512,\n#         'batch_size': 16,\n#         'inference_mode': 'eval',\n#         'use_tta': True,\n#         'use_flip_tta': False,\n#         'use_all_masks': False,\n#         'randomize_masks': False,\n#         'etl_version': '0.1',\n#         'pytorch_lightning': False,\n#         'model': {\n#             'ensemble': False,\n#             'model_path': '/kaggle/input/unet-mit-b3-exp081-model/Unet_mit_b3_ema.pth'\n#         }\n#     }\n}, {\n    'alias': 'efficient-b7-rand071-pseudo-labels-exp083', \n    'weight': 3,\n    'config': {\n        'model_name': 'efficientnet-b7-rand-071-pseudo-labels-exp083',\n        'backbone': 'efficientnet-b7',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-efficientnetb7-exp083-model/Unet_efficientnet-b7_ema.pth'\n        }\n    }\n# }, {\n#     'alias': 'efficient-b7-full071-pseudo-labels-exp088', \n#     'weight': 1,\n#     'config': {\n#         'model_name': 'efficientnet-b7-full071-pseudo-labels-exp088',\n#         'backbone': 'efficientnet-b7',\n#         'in_chans': 3,\n#         'image_size': 512,\n#         'batch_size': 16,\n#         'inference_mode': 'eval',\n#         'use_tta': True,\n#         'use_flip_tta': False,\n#         'use_all_masks': False,\n#         'randomize_masks': False,\n#         'etl_version': '0.1',\n#         'pytorch_lightning': False,\n#         'model': {\n#             'ensemble': False,\n#             'model_path': '/kaggle/input/unet-efficientnetb7-exp088-model/Unet_efficientnet-b7_ema.pth'\n#         }\n#     }\n}, {\n    'alias': 'segformer-b3-768-full071-psuedo-labels-exp86', \n    'weight': 3,\n    'config': {\n        'model_name': 'segformer-b3-768-full071-psuedo-labels-exp86',\n        'backbone': 'mit_b3',\n        'in_chans': 3,\n        'image_size': 768,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-mit-b3-exp086-model/Unet_mit_b3_ema.pth'\n        }\n    }\n}, {\n    'alias': 'segformer-b3-rand071-psuedo-labels-valid-data-exp95',\n    'weight': 3,\n    'config': {\n        'model_name': 'segformer-b3-rand071-psuedo-labels-valid-data-exp95',\n        'backbone': 'mit_b3',\n        'in_chans': 3,\n        'image_size': 512,\n        'batch_size': 16,\n        'inference_mode': 'eval',\n        'use_tta': True,\n        'use_flip_tta': False,\n        'use_all_masks': False,\n        'randomize_masks': False,\n        'etl_version': '0.1',\n        'pytorch_lightning': False,\n        'normalize_image': True,\n        'run_classifier': False,\n        'model': {\n            'ensemble': False,\n            'model_path': '/kaggle/input/unet-mit-b3-exp095-model/Unet_mit_b3_ema.pth'\n        }\n    }\n}]\n\npred_masks_dict = defaultdict(list)\n\nfixed_TH = 0.35 # Arbitrary confidence threshold\n\nensemble_weights = []\n\n# TODO: Not running notebook if submission\n#a_file = cfg.dataset_path + f\"test/a/mask.png\"\n#with open(a_file,'rb') as f:\n#    hash_md5 = hashlib.md5(f.read()).hexdigest()\n#is_skip_test = hash_md5 == '0b0fffdc0e88be226673846a143bb3e0'\n#\n#is_skip_test = False\n#\n#debug = False\n#\n#if is_skip_test:\n#    submit_df = pd.DataFrame({\n#        'Id': ['a', 'b'],\n#        'Predicted':['1 2', '1 2']\n#    })\n#    submit_df.to_csv('submission.csv', index=False)\n\nOr_model_weight = 4\ntotal_weight = sum(model_template['weight'] for model_template in model_template_list) + Or_model_weight\n\nmodel_preds = defaultdict(float)\nfor model_template in model_template_list:\n\n    # Register model specific config\n    for k, v in model_template['config'].items():\n        setattr(cfg,k,v)\n\n    # Initialize model\n    model_path = model_template['config']['model']['model_path']\n    if model_template['config']['model']['ensemble']:\n        folds = model_template['config']['model']['folds']\n        weights = model_template['config']['model']['weights']\n        model = build_ensemble_model(cfg, model_path, folds, weights)\n    else:\n        model = build_model(cfg, model_path)\n        \n    if not cfg.pytorch_lightning:\n        model = nn.DataParallel(model, device_ids=Config.device_ids)\n    model = model.cuda()\n    if cfg.inference_mode == 'eval':\n        model.eval()\n\n    # Register ensemnble weights\n    model_weight = model_template['weight']\n    ensemble_weights.append(model_weight)\n\n    # Create dataloader for test data\n    test_loader = make_test_dataset()\n\n    for step, (images, image_ids) in tqdm(enumerate(test_loader), total=len(test_loader)):\n\n        images = images.cuda()\n\n        with torch.no_grad():\n            with autocast():\n                if cfg.pytorch_lightning:\n                    y_preds = model._4tta_rot_test(images)\n                else:\n                    if cfg.use_tta:\n                        y_preds = TTA(images, model)\n                    else:\n                        y_preds = model(images)\n                        y_preds = torch.sigmoid(y_preds)\n\n            # Resize back to original 256\n            y_preds = torch.nn.functional.interpolate(y_preds, size=256, mode='bilinear')\n            y_preds = y_preds.cpu().detach().numpy()\n\n        for img_idx in range(0, images.shape[0]):\n            pred_mask = y_preds[img_idx, :, :, :]\n            img_id = image_ids[img_idx].item()\n            norm_weight = model_weight / total_weight\n            model_preds[img_id] += norm_weight * pred_mask\n\n    del model, test_loader, images, y_preds, model_weight\n    gc.collect()\n    torch.cuda.empty_cache()\n    \n# Add Orkatz Pipeline\nbatch_size = 32\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndata = cfg.root_path + '/google-research-identify-contrails-reduce-global-warming'\ndata_root = cfg.root_path + '/google-research-identify-contrails-reduce-global-warming/test/'\nfilenames = os.listdir(data_root)\ntest_df = pd.DataFrame(filenames, columns=['record_id'])\ntest_df['path'] = data_root + test_df['record_id'].astype(str)\n\nclass ContrailsDataset(torch.utils.data.Dataset):\n    def __init__(self, df, image_size=256, train=True):\n\n        self.df = df\n        self.trn = train\n        self.df_idx: pd.DataFrame = pd.DataFrame({'idx': os.listdir(cfg.root_path + f'/google-research-identify-contrails-reduce-global-warming/test')})\n        self.normalize_image = T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n        self.image_size = image_size\n        if image_size != 256:\n            self.resize_image = T.transforms.Resize(image_size)\n\n    def read_record(self, directory):\n        record_data = {}\n        for x in [\n            \"band_11\",\n            \"band_14\",\n            \"band_15\"\n        ]:\n\n            record_data[x] = np.load(os.path.join(directory, x + \".npy\"))\n\n        return record_data\n\n    def normalize_range(self, data, bounds):\n        \"\"\"Maps data to the range [0, 1].\"\"\"\n        return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n    def get_false_color(self, record_data):\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n\n        N_TIMES_BEFORE = 4\n\n        r = self.normalize_range(record_data[\"band_15\"] - record_data[\"band_14\"], _TDIFF_BOUNDS)\n        g = self.normalize_range(record_data[\"band_14\"] - record_data[\"band_11\"], _CLOUD_TOP_TDIFF_BOUNDS)\n        b = self.normalize_range(record_data[\"band_14\"], _T11_BOUNDS)\n        false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n        img = false_color[..., N_TIMES_BEFORE]\n\n        return img\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        con_path = row.path\n        data = self.read_record(con_path)\n\n        img = self.get_false_color(data)\n        img = torch.tensor(np.reshape(img, (256, 256, 3))).to(torch.float32).permute(2, 0, 1)\n\n        if self.image_size != 256:\n            img = self.resize_image(img)\n\n        img = self.normalize_image(img)\n\n        image_id = int(self.df_idx.iloc[index]['idx'])\n\n        return img.float(), torch.tensor(image_id)\n\n    def __len__(self):\n        return len(self.df)\n\ntest_ds = ContrailsDataset(\n        test_df,\n        384,\n        train = False\n    )\n\nfrom typing import Optional, List\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom timm import create_model\n\nclass SegModel(nn.Module):\n    def __init__(self, backbone):\n        super(SegModel, self).__init__()\n        self.seg = smp.Unet(encoder_name=backbone, encoder_weights=None, classes=1, activation=None)\n    def forward(self,x):\n        global_features = self.seg.encoder(x)\n        seg_features = self.seg.decoder(*global_features)\n        seg_features = self.seg.segmentation_head(seg_features)\n        return seg_features\n    \nclass UnetSuperVision(nn.Module):\n    \"\"\"Unet is a fully convolution neural network for image semantic segmentation\n    Args:\n        encoder_name: name of classification model (without last dense layers) used as feature\n            extractor to build segmentation model.\n        encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).\n        decoder_channels: list of numbers of ``Conv2D`` layer filters in decoder blocks\n        decoder_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers\n            is used.\n        num_classes: a number of classes for output (output shape - ``(batch, classes, h, w)``).\n        center: if ``True`` add ``Conv2dReLU`` block on encoder head\n    NOTE: This is based off an old version of Unet in https://github.com/qubvel/segmentation_models.pytorch\n    \"\"\"\n\n    def __init__(\n            self,\n            backbone='resnet50',\n            backbone_kwargs=None,\n            backbone_indices=None,\n            decoder_use_batchnorm=True,\n            decoder_channels=(256, 128, 64, 32, 16),\n            in_chans=3,\n            num_classes=1,\n            center=True,\n            norm_layer=nn.BatchNorm2d,\n    ):\n        super().__init__()\n        backbone_kwargs = backbone_kwargs or {}\n        # NOTE some models need different backbone indices specified based on the alignment of features\n        # and some models won't have a full enough range of feature strides to work properly.\n        encoder = create_model(\n            backbone, features_only=True, out_indices=backbone_indices, in_chans=in_chans,\n            pretrained=False, **backbone_kwargs)\n        encoder_channels = encoder.feature_info.channels()[::-1]\n        self.encoder = encoder\n\n        if not decoder_use_batchnorm:\n            norm_layer = None\n        self.decoder = UnetDecoder(\n            encoder_channels=encoder_channels,\n            decoder_channels=decoder_channels,\n            final_channels=num_classes,\n            norm_layer=norm_layer,\n            center=center,\n        )\n        self.avgpool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Linear(encoder.feature_info.channels()[-1], 1)\n\n    def forward(self, x: torch.Tensor):\n        x = self.encoder(x)\n        x.reverse()  # torchscript doesn't work with [::-1]\n        x_cls = self.avgpool(x[0])\n        x_cls = x_cls.view(x_cls.size(0), -1)\n        x_cls = self.fc(x_cls).view(x_cls.size(0))\n        x = self.decoder(x)\n        #return {\"x\":x,\"x_cls\":x_cls}\n        return x\n    \n\nclass Unet(nn.Module):\n    \"\"\"Unet is a fully convolution neural network for image semantic segmentation\n    Args:\n        encoder_name: name of classification model (without last dense layers) used as feature\n            extractor to build segmentation model.\n        encoder_weights: one of ``None`` (random initialization), ``imagenet`` (pre-training on ImageNet).\n        decoder_channels: list of numbers of ``Conv2D`` layer filters in decoder blocks\n        decoder_use_batchnorm: if ``True``, ``BatchNormalisation`` layer between ``Conv2D`` and ``Activation`` layers\n            is used.\n        num_classes: a number of classes for output (output shape - ``(batch, classes, h, w)``).\n        center: if ``True`` add ``Conv2dReLU`` block on encoder head\n    NOTE: This is based off an old version of Unet in https://github.com/qubvel/segmentation_models.pytorch\n    \"\"\"\n\n    def __init__(\n            self,\n            backbone='resnet50',\n            backbone_kwargs=None,\n            backbone_indices=None,\n            decoder_use_batchnorm=True,\n            decoder_channels=(256, 128, 64, 32, 16),\n            in_chans=3,\n            num_classes=1,\n            center=True,\n            norm_layer=nn.BatchNorm2d,\n            cls_head = True,\n    ):\n        super().__init__()\n        backbone_kwargs = backbone_kwargs or {}\n        # NOTE some models need different backbone indices specified based on the alignment of features\n        # and some models won't have a full enough range of feature strides to work properly.\n        encoder = create_model(\n            backbone, features_only=True, out_indices=backbone_indices, in_chans=in_chans,\n            pretrained=False, **backbone_kwargs)\n        encoder_channels = encoder.feature_info.channels()[::-1]\n        self.encoder = encoder\n\n        if not decoder_use_batchnorm:\n            norm_layer = None\n        self.decoder = UnetDecoder(\n            encoder_channels=encoder_channels,\n            decoder_channels=decoder_channels,\n            final_channels=num_classes,\n            norm_layer=norm_layer,\n            center=center,\n        )\n        if cls_head:\n            self.avgpool = nn.AdaptiveAvgPool2d(1)\n            self.fc = nn.Linear(encoder.feature_info.channels()[-1], 1)\n\n    def forward(self, x: torch.Tensor):\n        x = self.encoder(x)\n        x.reverse()  # torchscript doesn't work with [::-1]\n#         x_cls = self.avgpool(x[0])\n#         x_cls = x_cls.view(x_cls.size(0), -1)\n#         x_cls = self.fc(x_cls).view(x_cls.size(0))\n        x = self.decoder(x)\n        return x\n    \nclass Conv2dBnAct(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, padding=0,\n                 stride=1, act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):\n        super().__init__()\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=False)\n        self.bn = norm_layer(out_channels)\n        self.act = act_layer(inplace=True)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.act(x)\n        return x\n\n\nclass DecoderBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, scale_factor=2.0, act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):\n        super().__init__()\n        conv_args = dict(kernel_size=3, padding=1, act_layer=act_layer)\n        self.scale_factor = scale_factor\n        if norm_layer is None:\n            self.conv1 = Conv2dBnAct(in_channels, out_channels, **conv_args)\n            self.conv2 = Conv2dBnAct(out_channels, out_channels,  **conv_args)\n        else:\n            self.conv1 = Conv2dBnAct(in_channels, out_channels, norm_layer=norm_layer, **conv_args)\n            self.conv2 = Conv2dBnAct(out_channels, out_channels, norm_layer=norm_layer, **conv_args)\n\n    def forward(self, x, skip: Optional[torch.Tensor] = None):\n        if self.scale_factor != 1.0:\n            x = F.interpolate(x, scale_factor=self.scale_factor, mode='nearest')\n        if skip is not None:\n            x = torch.cat([x, skip], dim=1)\n        x = self.conv1(x)\n        x = self.conv2(x)\n        return x\n    \nclass UnetDecoder(nn.Module):\n\n    def __init__(\n            self,\n            encoder_channels,\n            decoder_channels=(256, 128, 64, 32, 16),\n            final_channels=1,\n            norm_layer=nn.BatchNorm2d,\n            center=False,\n    ):\n        super().__init__()\n\n        if center:\n            channels = encoder_channels[0]\n            self.center = DecoderBlock(channels, channels, scale_factor=1.0, norm_layer=norm_layer)\n        else:\n            self.center = nn.Identity()\n\n        in_channels = [in_chs + skip_chs for in_chs, skip_chs in zip(\n            [encoder_channels[0]] + list(decoder_channels[:-1]),\n            list(encoder_channels[1:]) + [0])]\n        out_channels = decoder_channels\n\n        self.blocks = nn.ModuleList()\n        for in_chs, out_chs in zip(in_channels, out_channels):\n            self.blocks.append(DecoderBlock(in_chs, out_chs, norm_layer=norm_layer))\n        self.final_conv = nn.Conv2d(out_channels[-1], final_channels, kernel_size=(1, 1))\n\n        self._init_weight()\n\n    def _init_weight(self):\n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                torch.nn.init.kaiming_normal_(m.weight)\n            elif isinstance(m, nn.BatchNorm2d):\n                m.weight.data.fill_(1)\n                m.bias.data.zero_()\n\n    def forward(self, x: List[torch.Tensor]):\n        encoder_head = x[0]\n        skips = x[1:]\n        x = self.center(encoder_head)\n        for i, b in enumerate(self.blocks):\n            skip = skips[i] if i < len(skips) else None\n            x = b(x, skip)\n        x = self.final_conv(x)\n        return x\n    \ntest_dl = DataLoader(test_ds, batch_size=32, num_workers = 1)\nmodel_or = Unet(backbone = 'maxxvitv2_rmlp_base_rw_384')\nmodel_or.load_state_dict(torch.load(\"/kaggle/input/newfold1maxxvitwithppcpt/unet-maxxvitv2_rmlp_base_rw_384_clshead_best_fold1.pth\"))\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel_or.to(device)\nmodel_or.eval()\n\nfor i, data in tqdm(enumerate(test_dl), total=len(test_dl)):\n\n    images, image_id = data\n\n    images = images.to(device)\n    with torch.no_grad():\n        predicted_mask = 0.7*model_or.forward(images) + 0.3*model_or.forward(images.flip([-1])).flip([-1])\n    #if config[\"model\"][\"image_size\"] != 256:\n    predicted_mask = torch.nn.functional.interpolate(predicted_mask, size=256, mode='bilinear')\n    predicted_mask = torch.sigmoid(predicted_mask).cpu().detach().numpy()\n    norm_weight = Or_model_weight / total_weight\n    for img_num in range(0, images.shape[0]):\n        current_image_id = image_id[img_num].item()\n        model_preds[current_image_id] += norm_weight * predicted_mask[img_num]\n    \n\n# Generate Submission\nprint(\"Generating Submission\")\nresults = []\nfor img_id, pred in model_preds.items():\n\n    pred = (pred >= fixed_TH).astype(int)\n    pred_rle = list_to_string(rle_encode(pred))\n    results.append((img_id, pred_rle))\n\ndel model_preds, pred\ngc.collect()\ntorch.cuda.empty_cache()\n\nsub = pd.DataFrame(results, columns=['record_id', 'encoded_pixels'])\n\nsample_sub = pd.read_csv(cfg.root_path + '/google-research-identify-contrails-reduce-global-warming/sample_submission.csv')\nsample_sub = pd.merge(sample_sub[['record_id']], sub, on='record_id', how='left')\n\nsample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"_uuid":"4241b62a-7b8f-4dd2-b147-a5584bce0ef1","_cell_guid":"9a77996b-2ff6-4f8f-82de-7bc2bbfeb1be","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-07T19:36:05.167008Z","iopub.execute_input":"2023-08-07T19:36:05.167347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}