{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\nsys.path.append(\"../input/pretrained-models-pytorch\")\nsys.path.append(\"/kaggle/input/efficientnet-pytorch/EfficientNet-PyTorch-master\")\nsys.path.append(\"/kaggle/input/smp-github/segmentation_models.pytorch-master\")\nsys.path.append(\"/kaggle/input/timm-pretrained-resnest/resnest/\")\nimport segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-08-09T09:26:48.841143Z","iopub.execute_input":"2023-08-09T09:26:48.841601Z","iopub.status.idle":"2023-08-09T09:26:56.753982Z","shell.execute_reply.started":"2023-08-09T09:26:48.841563Z","shell.execute_reply":"2023-08-09T09:26:56.752973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\nimport sys\nimport pandas as pd\nimport os\nimport gc\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\nimport cv2\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport argparse\nimport importlib\nimport torch\nimport torch.nn as nn\nfrom torch.optim import Adam, SGD, AdamW\n\nimport datetime\nimport wandb\n\nimport torchvision.transforms as T","metadata":{"execution":{"iopub.status.busy":"2023-08-09T09:26:56.757669Z","iopub.execute_input":"2023-08-09T09:26:56.757962Z","iopub.status.idle":"2023-08-09T09:26:56.957903Z","shell.execute_reply.started":"2023-08-09T09:26:56.757938Z","shell.execute_reply":"2023-08-09T09:26:56.956959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# os.environ['KAGGLE_IS_COMPETITION_RERUN'] = \"True\"","metadata":{"execution":{"iopub.status.busy":"2023-08-09T09:27:09.488274Z","iopub.execute_input":"2023-08-09T09:27:09.488994Z","iopub.status.idle":"2023-08-09T09:27:09.497311Z","shell.execute_reply.started":"2023-08-09T09:27:09.488958Z","shell.execute_reply":"2023-08-09T09:27:09.496265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    !pip install --no-index --find-links=\"/kaggle/input/dependencies/\" openmim\n    !pip install --no-index --find-links=\"/kaggle/input/dependencies/\" mmengine\n    !cp -r /kaggle/input/dependencies/mmcv-2.0.1/mmcv-2.0.1 .\n    !pip install --no-index --find-links=\"/kaggle/input/dependencies/\" /kaggle/working/mmcv-2.0.1\n    !cp -r /kaggle/input/dependencies/mmsegmentation .\n    !pip install --no-index --find-links=\"/kaggle/input/dependencies/\" /kaggle/working/mmsegmentation\n    !pip install --no-index --find-links=\"/kaggle/input/mmpretrain2/\" /kaggle/input/mmpretrain2/mmpretrain-1.0.1-py2.py3-none-any.whl\n    from mmseg.apis import init_model","metadata":{"execution":{"iopub.status.busy":"2023-08-09T09:27:11.561495Z","iopub.execute_input":"2023-08-09T09:27:11.562215Z","iopub.status.idle":"2023-08-09T10:01:39.105767Z","shell.execute_reply.started":"2023-08-09T09:27:11.562182Z","shell.execute_reply":"2023-08-09T10:01:39.104732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:39.108083Z","iopub.execute_input":"2023-08-09T10:01:39.108443Z","iopub.status.idle":"2023-08-09T10:01:39.114294Z","shell.execute_reply.started":"2023-08-09T10:01:39.108412Z","shell.execute_reply":"2023-08-09T10:01:39.112946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom torch.utils.data import DataLoader, Dataset\nimport cv2\nimport torch\nimport os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:39.115646Z","iopub.execute_input":"2023-08-09T10:01:39.11605Z","iopub.status.idle":"2023-08-09T10:01:40.249305Z","shell.execute_reply.started":"2023-08-09T10:01:39.11602Z","shell.execute_reply":"2023-08-09T10:01:40.248383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## config","metadata":{}},{"cell_type":"code","source":"import os\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nclass CFG:\n    comp_name = 'contrails'\n    comp_dir_path = '/data/contrails/'\n    \n    exp_name = 'basev4'\n    starting_checkpoint = None\n    # ============== pred target =============\n    target_size = 1\n    # ============== model cfg =============\n    model_name = 'segformer_b3_big_out'\n    timestep_model = False\n    # ============== training cfg =============\n    size = 512\n    get_false_color = True\n    timesteps = [4]\n    bands = [0, 1, 2, 3, 4, 5, 6, 7, 8]\n    mean = [233.6771, 242.2548, 250.7509, 274.4108, 255.5268, 276.6016, 275.3604, 272.5643, 260.4260]\n    std = [ 7.0181,  9.1566, 11.3484, 19.6334, 13.1177, 20.7182, 21.0882, 20.5616, 15.8269]\n    normalize = False\n    if get_false_color:\n        in_chans = 3 * len(timesteps)\n    else:\n        in_chans = len(bands) * len(timesteps)\n    sample_labels = False\n    soft_multilabels = False\n    max_multilabels = False\n    softmax_multilabels = True\n\n    train_batch_size = 64\n    valid_batch_size = train_batch_size\n    use_amp = True\n\n    scheduler = 'Cosine'\n    epochs = 50\n\n    # adamW warmupあり\n    # warmup_factor = 10\n    if starting_checkpoint:\n        lr = 1e-5\n    else:\n        lr = 1e-4\n    # ============== fold =============\n    valid_id = \"folder\"\n    # ============== fixed =============\n    min_lr = 1e-6\n    weight_decay = 0\n    max_grad_norm = 1000\n    num_workers = 2\n    seed = 11\n    # ============== set dataset path =============\n    print('set dataset path')\n\n    outputs_path = f'working/outputs/{comp_name}/{exp_name}/'\n\n    submission_dir = outputs_path + 'submissions/'\n    submission_path = submission_dir + f'submission_{exp_name}.csv'\n\n    model_dir = outputs_path + \\\n        f'{comp_name}-models/'\n\n    figures_dir = outputs_path + 'figures/'\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.251774Z","iopub.execute_input":"2023-08-09T10:01:40.2521Z","iopub.status.idle":"2023-08-09T10:01:40.268294Z","shell.execute_reply.started":"2023-08-09T10:01:40.252074Z","shell.execute_reply":"2023-08-09T10:01:40.267389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_DEBUG = False\nmode = 'train' if IS_DEBUG else 'test'\nTH = 0.5","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.270573Z","iopub.execute_input":"2023-08-09T10:01:40.271372Z","iopub.status.idle":"2023-08-09T10:01:40.280071Z","shell.execute_reply.started":"2023-08-09T10:01:40.271253Z","shell.execute_reply":"2023-08-09T10:01:40.27901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.281721Z","iopub.execute_input":"2023-08-09T10:01:40.282214Z","iopub.status.idle":"2023-08-09T10:01:40.292422Z","shell.execute_reply.started":"2023-08-09T10:01:40.282184Z","shell.execute_reply":"2023-08-09T10:01:40.291314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## helper","metadata":{}},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.295403Z","iopub.execute_input":"2023-08-09T10:01:40.295696Z","iopub.status.idle":"2023-08-09T10:01:40.305197Z","shell.execute_reply.started":"2023-08-09T10:01:40.295673Z","shell.execute_reply":"2023-08-09T10:01:40.304198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dataset","metadata":{}},{"cell_type":"code","source":"def get_transforms(data, cfg):\n    if data == 'train':\n        aug = A.Compose(cfg.train_aug_list)\n    elif data == 'valid':\n        aug = A.Compose(cfg.valid_aug_list)\n\n    # print(aug)\n    return aug\n\nclass CustomDataset(Dataset):\n    def __init__(self, record_ids, cfg, transform=None):\n        self.record_ids = np.array(record_ids)\n        self.cfg = cfg\n        self.transform = transform\n        self.resize_img = T.Resize(self.cfg.size, interpolation=T.InterpolationMode.BICUBIC, antialias=False)\n\n    def __len__(self):\n        return len(self.record_ids)\n\n    def __getitem__(self, idx):\n        if self.cfg.timesteps:\n            image = self.load_image(self.record_ids[idx])[:, :, :, self.cfg.timesteps].astype(np.float32)\n            image = image[self.cfg.bands, :, :, :]\n            if self.cfg.normalize:\n                for i, band in enumerate(self.cfg.bands):\n                    image[i, :, :, :] = (image[i, :, :, :] - self.cfg.mean[band])/self.cfg.std[band]\n        else:\n            image = self.load_image(self.record_ids[idx])[:, :, :, 4].astype(np.float32)\n        if self.cfg.get_false_color:\n            image = self.get_false_color(image)\n        image = np.moveaxis(image, 0, -1)[:, :, 0, :]\n        if self.transform:\n            data = self.transform(image=image)\n            image = data['image']\n        if self.cfg.clamp:\n            img_max = image.max()\n            img_min = image.min()\n            image = self.resize_img(image).clamp(min=img_min, max=img_max)\n        else:\n            image = self.resize_img(image)\n        return image\n    \n    def normalize_range(self, data, bounds):\n        \"\"\"Maps data to the range [0, 1].\"\"\"\n        return (data - bounds[0]) / (bounds[1] - bounds[0])\n    def get_false_color(self, image):\n        _T11_BOUNDS = (243, 303)\n        _CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n        _TDIFF_BOUNDS = (-4, 2)\n\n        r = self.normalize_range(image[-2] - image[-3], _TDIFF_BOUNDS)\n        g = self.normalize_range(image[-3] - image[-6], _CLOUD_TOP_TDIFF_BOUNDS)\n        b = self.normalize_range(image[-3], _T11_BOUNDS)\n        false_color = np.clip(np.stack([r, g, b], axis=0), 0, 1)\n\n        return false_color\n    \n    def load_image(self, record):\n        images = []\n        for band in [\"08\", \"09\", \"10\", \"11\", \"12\", \"13\", \"14\", \"15\", \"16\"]:\n            image = np.load(record + f\"/band_{band}.npy\")\n            images.append(image)\n        image = np.stack(images, axis = 0)\n        return image\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.306744Z","iopub.execute_input":"2023-08-09T10:01:40.307274Z","iopub.status.idle":"2023-08-09T10:01:40.325621Z","shell.execute_reply.started":"2023-08-09T10:01:40.307242Z","shell.execute_reply":"2023-08-09T10:01:40.324778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\ndef make_test_dataset():\n    test_images_list = glob.glob(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/test/**\")\n    test_dataset = CustomDataset(test_images_list, CFG, transform=get_transforms(data='valid', cfg=CFG))\n    \n    test_loader = DataLoader(test_dataset,\n                          batch_size=CFG.batch_size,\n                          shuffle=False,\n                          num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    return test_loader, test_images_list","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.326966Z","iopub.execute_input":"2023-08-09T10:01:40.327355Z","iopub.status.idle":"2023-08-09T10:01:40.341339Z","shell.execute_reply.started":"2023-08-09T10:01:40.327325Z","shell.execute_reply":"2023-08-09T10:01:40.340236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## model","metadata":{}},{"cell_type":"code","source":"from transformers import SegformerForSemanticSegmentation, SegformerModel, SegformerConfig","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:40.345983Z","iopub.execute_input":"2023-08-09T10:01:40.346253Z","iopub.status.idle":"2023-08-09T10:01:49.417962Z","shell.execute_reply.started":"2023-08-09T10:01:40.346218Z","shell.execute_reply":"2023-08-09T10:01:49.416958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segformer_b3_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":128,\n  \"num_channels\":3})\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:49.41939Z","iopub.execute_input":"2023-08-09T10:01:49.420201Z","iopub.status.idle":"2023-08-09T10:01:49.431893Z","shell.execute_reply.started":"2023-08-09T10:01:49.420165Z","shell.execute_reply":"2023-08-09T10:01:49.430868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segformer_b3_medium_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":64,\n  \"num_channels\":3})","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:49.433096Z","iopub.execute_input":"2023-08-09T10:01:49.433485Z","iopub.status.idle":"2023-08-09T10:01:49.53721Z","shell.execute_reply.started":"2023-08-09T10:01:49.433434Z","shell.execute_reply":"2023-08-09T10:01:49.536275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segformer_b3_half_out_big_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":256,\n  \"num_channels\":3})","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:01:49.538656Z","iopub.execute_input":"2023-08-09T10:01:49.539034Z","iopub.status.idle":"2023-08-09T10:01:49.548789Z","shell.execute_reply.started":"2023-08-09T10:01:49.539008Z","shell.execute_reply":"2023-08-09T10:01:49.547606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segformer_b3_regular_config = SegformerConfig(**{\n  \"architectures\": [\n    \"SegformerForImageClassification\"\n  ],\n  \"attention_probs_dropout_prob\": 0.0,\n  \"classifier_dropout_prob\": 0.1,\n  \"decoder_hidden_size\": 768,\n  \"depths\": [\n    3,\n    4,\n    18,\n    3\n  ],\n  \"downsampling_rates\": [\n    1,\n    4,\n    8,\n    16\n  ],\n  \"drop_path_rate\": 0.1,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.0,\n  \"hidden_sizes\": [\n    64,\n    128,\n    320,\n    512\n  ],\n\n  \"image_size\": 224,\n  \"initializer_range\": 0.02,\n  \n  \"layer_norm_eps\": 1e-06,\n  \"mlp_ratios\": [\n    4,\n    4,\n    4,\n    4\n  ],\n  \"model_type\": \"segformer\",\n  \"num_attention_heads\": [\n    1,\n    2,\n    5,\n    8\n  ],\n  \"num_encoder_blocks\": 4,\n  \"patch_sizes\": [\n    7,\n    3,\n    3,\n    3\n  ],\n  \"sr_ratios\": [\n    8,\n    4,\n    2,\n    1\n  ],\n  \"strides\": [\n    4,\n    2,\n    2,\n    2\n  ],\n  \"torch_dtype\": \"float32\",\n  \"num_labels\":16,\n  \"num_channels\":3})\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:27.354502Z","iopub.execute_input":"2023-08-09T10:23:27.354892Z","iopub.status.idle":"2023-08-09T10:23:27.363243Z","shell.execute_reply.started":"2023-08-09T10:23:27.354858Z","shell.execute_reply":"2023-08-09T10:23:27.362256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class convnext_xl(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        config_file = '/kaggle/input/mmseg-models/convnext-xlarge_upernet_8xb2-amp-160k_ade20k-640x640.py'\n        checkpoint_file = None\n        # build the model from a config file and a checkpoint file\n        self.xy_encoder_2d = init_model(config_file)\n        self.upscaler1 = nn.ConvTranspose2d(\n            150, 1, kernel_size=(4, 4), stride=2, padding=1)\n\n    def forward(self, image):\n        output = self.xy_encoder_2d(image)\n        output = self.upscaler1(output)\n        return output\n    \nclass Segformer_2D_big_out(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            128, 64, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            64, 1, kernel_size=(4, 4), stride=2, padding=1)\n    def forward(self, image):\n        output = self.xy_encoder_2d(image).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n\nclass Segformer_2D_medium_out(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            64, 32, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            32, 1, kernel_size=(4, 4), stride=2, padding=1)\n    def forward(self, image):\n        output = self.xy_encoder_2d(image).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \nclass Segformer_2D_half_out(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            64, 1, kernel_size=(4, 4), stride=2, padding=1)\n    def forward(self, image):\n        output = self.xy_encoder_2d(image).logits\n        output = self.upscaler1(output)\n        return output\n    \nclass Segformer_2D_half_out_big(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            256, 1, kernel_size=(4, 4), stride=2, padding=1)\n    def forward(self, image):\n        output = self.xy_encoder_2d(image).logits\n        output = self.upscaler1(output)\n        return output\n    \n    \nclass Segformer_2D(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            16, 8, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            8, 1, kernel_size=(4, 4), stride=2, padding=1)\n\n    def forward(self, image):\n        output = self.xy_encoder_2d(image).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n    \n\nclass triple_channel_segformer(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n        self.act = nn.ReLU()\n        self.encoder1 = nn.Conv2d(self.cfg.in_chans, self.cfg.in_chans*2, kernel_size=1, padding=0)\n        self.encoder2 = nn.Conv2d(self.cfg.in_chans*2, self.cfg.in_chans*4, kernel_size=1, padding=0)\n        self.encoder3 = nn.Conv2d(self.cfg.in_chans*4, self.cfg.in_chans*8, kernel_size=1, padding=0)\n        self.encoder4 = nn.Conv2d(self.cfg.in_chans*8, self.cfg.in_chans*16, kernel_size=1, padding=0)\n        self.encoder5 = nn.Conv2d(self.cfg.in_chans*16, self.cfg.in_chans*32, kernel_size=1, padding=0)\n        self.encoder6 = nn.Conv2d(self.cfg.in_chans*32, 3, kernel_size=1, padding=0)\n        \n        self.xy_encoder_2d = SegformerForSemanticSegmentation(self.cfg.segformer_config)\n        self.upscaler1 = nn.ConvTranspose2d(\n            128, 64, kernel_size=(4, 4), stride=2, padding=1)\n        self.upscaler2 = nn.ConvTranspose2d(\n            64, 1, kernel_size=(4, 4), stride=2, padding=1)\n        \n    def forward(self, image):\n        output = self.act(self.encoder1(image))\n        output = self.act(self.encoder2(output))\n        output = self.act(self.encoder3(output))\n        output = self.act(self.encoder4(output))\n        output = self.act(self.encoder5(output))\n        output = self.encoder6(output)\n        output = self.xy_encoder_2d(output).logits\n        output = self.upscaler1(output)\n        output = self.upscaler2(output)\n        return output\n\nclass UnetPlusPlusRegNetX_320(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.xy_encoder_2d = smp.UnetPlusPlus(\n            # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n            encoder_name=\"timm-regnetx_320\",\n            encoder_weights=None,     # use `imagenet` pre-trained weights for encoder initialization\n            # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n            in_channels=3,\n            # model output channels (number of classes in your dataset)\n            classes=1,\n            activation=None\n        )\n    def forward(self, image):\n        output = self.xy_encoder_2d(image)\n        return output\n\nclass effnet_v2_xl(nn.Module):\n    def __init__(self, cfg):\n        super().__init__()\n        self.cfg = cfg\n\n        self.xy_encoder_2d = smp.UnetPlusPlus(\n            # choose encoder, e.g. mobilenet_v2 or efficientnet-b7\n            encoder_name=\"tu-tf_efficientnetv2_xl.in21k_ft_in1k\",\n            # use `imagenet` pre-trained weights for encoder initialization\n            encoder_weights=None,\n            # model input channels (1 for gray-scale images, 3 for RGB, etc.)\n            in_channels=3,\n            # model output channels (number of classes in your dataset)\n            classes=1,\n        )\n        self.downscaler = nn.Conv2d(\n            1, 1, kernel_size=(4, 4), stride=2, padding=1)\n\n    def forward(self, image):\n        output = self.xy_encoder_2d(image)\n        output = self.downscaler(output)\n        return output\n    \n    \ndef build_model(cfg, model_arch = None):\n    print('model_name', model_arch)\n    if model_arch == \"segformer_2d_big_out\":\n        model = Segformer_2D_big_out(cfg)\n    elif model_arch == \"segformer_2d_medium_out\":\n        model = Segformer_2D_medium_out(cfg)\n    elif model_arch == \"segformer_2d_half_out\":\n        model = Segformer_2D_half_out(cfg)\n    elif model_arch == \"segformer_2d_half_out_big\":\n        model = Segformer_2D_half_out_big(cfg)\n    elif model_arch == \"segformer_2d\":\n        model = Segformer_2D(cfg)\n    elif model_arch == \"triple_channel_segformer\":\n        model = triple_channel_segformer(cfg)\n    elif model_arch == \"UnetPlusPlusRegNetX_320\":\n        model = UnetPlusPlusRegNetX_320(cfg)\n    elif model_arch == \"convnext_xl\":\n        model = convnext_xl(cfg)\n    elif model_arch == \"effnet_v2_xl\":\n        model = effnet_v2_xl(cfg)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:27.629368Z","iopub.execute_input":"2023-08-09T10:23:27.629765Z","iopub.status.idle":"2023-08-09T10:23:27.665444Z","shell.execute_reply.started":"2023-08-09T10:23:27.629735Z","shell.execute_reply":"2023-08-09T10:23:27.664266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EnsembleModel:\n    def __init__(self, use_tta=False):\n        self.models = []\n        self.use_tta = use_tta\n    def tta_infer(self, model:nn.Module, x):\n        #x.shape=(batch,c,h,w)\n        shape=x.shape\n        x=[\n            *[torch.rot90(x,k=i,dims=(-2,-1)) for i in range(0,4)],\n            *[torch.flip(torch.rot90(x,k=i,dims=(-2,-1)), [2]) for i in range(0,4)],\n            *[torch.flip(torch.rot90(x,k=i,dims=(-2,-1)), [3]) for i in range(0,4)],\n            ]\n        x=[model(single_x) for single_x in x]\n        x=torch.cat(x,dim=0)\n        pred_height, pred_width = x.shape[2:] \n        x=x.reshape(12, shape[0], pred_height, pred_width)\n        x=[\n            *[torch.rot90(x[i],k=-i,dims=(-2,-1)) for i in range(0,4)],\n            *[torch.rot90(torch.flip(x[i], [1]),k=-i,dims=(-2,-1)) for i in range(4,8)],\n            *[torch.rot90(torch.flip(x[i], [2]),k=-i,dims=(-2,-1)) for i in range(8,12)],\n            ]\n        x=torch.stack(x,dim=0)\n#         x = torch.nn.functional.interpolate(x, size=256, mode='bilinear')\n        x = torch.sigmoid(x)\n        x = x.mean(0)\n        return x\n                \n    def __call__(self, x):\n        if self.use_tta:\n            outputs = [self.tta_infer(model, x).to('cpu').numpy()\n                   for model in self.models]\n        else:\n            outputs = []\n            for model in self.models:\n                output = model(x)\n#                 output = torch.nn.functional.interpolate(output, size=256, mode='bilinear')\n                output = output.mean(axis = 1)\n                output = torch.sigmoid(output)\n                outputs.append(output.to('cpu').numpy())\n        avg_preds = np.mean(outputs, axis=0)\n        new_preds = []\n        for pred in avg_preds:\n            pred = pred.astype(np.float32)\n            resized_prediction = cv2.resize(pred, dsize = (512, 512), interpolation = cv2.INTER_CUBIC)\n            resized_prediction = cv2.resize(resized_prediction, dsize = (256, 256), interpolation=cv2.INTER_NEAREST)\n            new_preds.append(resized_prediction)\n        avg_preds = np.stack(new_preds, axis = 0)\n        return avg_preds\n\n    def add_model(self, model):\n        self.models.append(model)\n\ndef build_ensemble_model(model_path, model_arch):\n    model = EnsembleModel(use_tta = True)\n    _model = build_model(CFG, model_arch)\n    _model.to(device)\n    state = torch.load(model_path)\n    try:\n        _model.load_state_dict(state)\n    except:\n        try:\n            _model = nn.DataParallel(_model)\n            _model.load_state_dict(state)\n        except:\n            _model = nn.DataParallel(_model)\n            _model.load_state_dict(state)\n    _model.eval()\n\n    model.add_model(_model)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:27.858967Z","iopub.execute_input":"2023-08-09T10:23:27.859265Z","iopub.status.idle":"2023-08-09T10:23:27.8786Z","shell.execute_reply.started":"2023-08-09T10:23:27.85924Z","shell.execute_reply":"2023-08-09T10:23:27.877359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_tuples = [\n    \n#     {\"model_arch\": \"effnet_v2_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/smp-models/effnet_v2_xl_all_train_effnet_v2_xl_300_95_final.pth\",\n#      \"segformer_config\": None},\n    \n#     {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/mmseg-models/all_train_convnext_xl_100_15_final.pth\",\n#      \"segformer_config\": None, \"score\":.696},\n    \n#     {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/mmseg-models/convnext_xl_all_train_convnext_xl_300_25_final.pth\",\n#      \"segformer_config\": None, \"clamp\":False},\n    \n#     {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/mmseg-models/convnext_xl_convnext_xl_500_90_final.pth\",\n#      \"segformer_config\": None, \"clamp\":True, 0.695},\n    \n    #best models\n#     {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/mmseg-models/spam_model_convnext_xl_0_best.pth\",\n#      \"segformer_config\": None, \"clamp\":False, \"score\":.696},\n    \n#     {\"model_arch\": \"effnet_v2_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/smp-models/effnet_v2_xl_effnet_v2_xl_200_70_final.pth\",\n#      \"segformer_config\": None, \"clamp\":False, \"score\":.696},\n    \n    {\"model_arch\": \"effnet_v2_xl\", \"size\": 512, \"batch_size\": 8,\n     \"weight_path\": \"/kaggle/input/smp-models/effnet_v2_xl_all_train_effnet_v2_xl_300_final_swa.pth\",\n     \"segformer_config\": None, \"clamp\":True, \"score\":.701},\n    \n#     {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n#      \"weight_path\": \"/kaggle/input/mmseg-models/convnext_xl_all_train_convnext_xl_300_final_swa.pth\",\n#      \"segformer_config\": None, \"clamp\":True, \"score\":0.695},\n    \n    {\"model_arch\": \"convnext_xl\", \"size\": 512, \"batch_size\": 8,\n     \"weight_path\": \"/kaggle/input/mmseg-models/convnext_xl_convnext_xl_500_final_swa.pth\",\n     \"segformer_config\": None, \"clamp\":True},\n    \n    {\"model_arch\": \"effnet_v2_xl\", \"size\": 512, \"batch_size\": 8,\n     \"weight_path\": \"/kaggle/input/smp-models/effnet_v2_xl_all_train_effnet_v2_xl_459_final_swa.pth\",\n     \"segformer_config\": None, \"clamp\":True},\n    \n    \n\n]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:28.098659Z","iopub.execute_input":"2023-08-09T10:23:28.099805Z","iopub.status.idle":"2023-08-09T10:23:28.108111Z","shell.execute_reply.started":"2023-08-09T10:23:28.099769Z","shell.execute_reply":"2023-08-09T10:23:28.106902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## main","metadata":{}},{"cell_type":"code","source":"import glob\nimport time","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:28.636186Z","iopub.execute_input":"2023-08-09T10:23:28.636575Z","iopub.status.idle":"2023-08-09T10:23:28.64185Z","shell.execute_reply.started":"2023-08-09T10:23:28.636545Z","shell.execute_reply":"2023-08-09T10:23:28.640766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nsample_sub = pd.read_csv(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:28.975645Z","iopub.execute_input":"2023-08-09T10:23:28.976021Z","iopub.status.idle":"2023-08-09T10:23:28.987799Z","shell.execute_reply.started":"2023-08-09T10:23:28.975994Z","shell.execute_reply":"2023-08-09T10:23:28.986698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:29.404778Z","iopub.execute_input":"2023-08-09T10:23:29.405516Z","iopub.status.idle":"2023-08-09T10:23:29.410257Z","shell.execute_reply.started":"2023-08-09T10:23:29.405461Z","shell.execute_reply":"2023-08-09T10:23:29.409155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as T","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:29.873946Z","iopub.execute_input":"2023-08-09T10:23:29.874693Z","iopub.status.idle":"2023-08-09T10:23:29.879651Z","shell.execute_reply.started":"2023-08-09T10:23:29.874657Z","shell.execute_reply":"2023-08-09T10:23:29.878669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nensemble_predictions = None\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    for model_config in model_tuples:\n        predictions = []\n        CFG.size = model_config[\"size\"]\n        CFG.batch_size = model_config[\"batch_size\"]\n        CFG.valid_aug_list = [\n    #         A.Resize(CFG.size, CFG.size, interpolation = cv2.INTER_LINEAR),\n            ToTensorV2(transpose_mask=True),\n        ]\n        CFG.segformer_config = model_config[\"segformer_config\"]\n        if model_config[\"clamp\"]:\n            CFG.clamp = True\n        else:\n            CFG.clamp = False\n        CFG.segformer_config = model_config[\"segformer_config\"]\n        model = build_ensemble_model(model_config[\"weight_path\"], model_config[\"model_arch\"])\n        test_loader, test_images_list = make_test_dataset()\n        for step, (images) in tqdm(enumerate(test_loader), total=len(test_loader)):\n            images = images.to(device)\n            with autocast():            \n                with torch.no_grad():\n                    predictions.append(model(images))\n\n        del test_loader\n        del model\n        gc.collect()\n        torch.cuda.empty_cache()\n        model_results = np.concatenate(predictions, axis = 0)\n        if ensemble_predictions is None:\n            ensemble_predictions = model_results\n        else:\n            ensemble_predictions += model_results\n    model_results = ensemble_predictions/len(model_tuples)\n    model_results = model_results > 0.46\n    for test_image, model_result in zip(test_images_list, model_results):\n        inklabels_rle = list_to_string(rle_encode(model_result))\n        results.append((test_image.split(\"/\")[-1], inklabels_rle))\n    gc.collect()\n    torch.cuda.empty_cache()\n    print(predictions[0].max())\n    sub = pd.DataFrame(results, columns=['record_id', 'encoded_pixels'])\n    sub[\"record_id\"] = sub[\"record_id\"].astype(np.int64)\n    sample_sub = pd.read_csv('/kaggle/input/google-research-identify-contrails-reduce-global-warming/' + 'sample_submission.csv')\n    sample_sub = pd.merge(sample_sub[['record_id']], sub, on='record_id', how='left')\n    !rm -rf /kaggle/working/\n    sample_sub.to_csv(\"submission.csv\", index=False)\nelse:\n    !rm -rf /kaggle/working/\n    sample_sub.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:23:30.217821Z","iopub.execute_input":"2023-08-09T10:23:30.218625Z","iopub.status.idle":"2023-08-09T10:24:18.192575Z","shell.execute_reply.started":"2023-08-09T10:23:30.218584Z","shell.execute_reply":"2023-08-09T10:24:18.191281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(ensemble_predictions[0])","metadata":{"execution":{"iopub.status.busy":"2023-08-09T10:24:18.19515Z","iopub.execute_input":"2023-08-09T10:24:18.196766Z","iopub.status.idle":"2023-08-09T10:24:18.479592Z","shell.execute_reply.started":"2023-08-09T10:24:18.196727Z","shell.execute_reply":"2023-08-09T10:24:18.478418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}