{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Identify Contrails with TensorFlow/Keras (Multi GPU)","metadata":{}},{"cell_type":"markdown","source":"**Features**\n\n- A data pipeline based on **TFRecordDataset**,\n  since the Keras dataset was not efficient enough for properly feeding the GPU:\n  includes a **multiprocessing parallelized generation** of TFRecords\n- Data augmentation:\n  **custom multi-frame image augmentation for segmentation** to apply the same random transform\n  to all time frames and the segmentation mask,\n  since the keras.layers preprocessing layers do not support this feature\n- Models:\n  - **Pseudo-3D ResNet (P3DResNet)** implementation inspired by Keras' ResNet50\n  - **U-Net** with self-made encoder, ResNet50 or P3DResNet\n  - **DeepLabV3+** with ResNet50 or P3DResNet\n- Gradient accumulation (the implementation did not show any benefit)\n- **Mixed-precision** GPU training: results in 2x speed up\n- **Multi-GPU** support (added in late submission): results in 2x speed up on 2x T4,\n  see the `<STRATEGY>` tag in the code comments","metadata":{}},{"cell_type":"code","source":"# reinstall tensorflow-io\n# to avoid the UserWarning: unable to load libtensorflow_io_plugins.so\n\n#!pip install tensorflow-io","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"import os","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ==============================\n\nLOAD_CHECKPOINT = False  # <DEVEL>\nTRAIN = True  # <DEVEL>\n\nif os.path.exists('/kaggle'):\n    PLATFORM = 'kaggle'\nelse:\n    PLATFORM = 'gcp'\n\n# ==============================\n\nprint(f'PLATFORM = {PLATFORM}')\nprint()\nprint(f'LOAD_CHECKPOINT = {LOAD_CHECKPOINT}')\nprint(f'TRAIN = {TRAIN}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PLATFORM == 'kaggle':\n\n    WORK_DIR = '/kaggle/working'  # preserved if notebook is saved\n    TEMP_DIR = '/kaggle/temp'  # just during current session\n\n    DATA_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n    \n    WEIGHTS_DIR = WORK_DIR\n    \n    resnet50_imagenet_weights =\\\n        '/kaggle/input/d/alexisbcook/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\n    # You can write up to 20GB to the current directory (/kaggle/working/)\n    # that gets preserved as output when you create a version using \"Save & Run All\" \n    # You can also write temporary files to /kaggle/temp/,\n    # but they won't be saved outside of the current session\n\nelif PLATFORM == 'gcp':\n\n    WORK_DIR = '/home/jupyter/kaggle/working'  # preserved if notebook is saved\n    TEMP_DIR = '/home/jupyter/kaggle/temp'  # just during current session\n\n    DATA_DIR = '/home/jupyter/kaggle/input/google-research-identify-contrails-reduce-global-warming'\n    \n    WEIGHTS_DIR = '/home/jupyter/identify-contrails-models'\n    WEIGHTS_DIR = WORK_DIR  # <DEVEL>\n    \n    resnet50_imagenet_weights = 'imagenet'\n    \n    %cd $WORK_DIR\n    \nelse:\n    raise NotImplementedError(f'unknown platform \"{PLATFORM}\"')\n\nprint('PWD =', os.getcwd())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"UPDATE_DATA = False\n\nif UPDATE_DATA:\n    %cp -v /kaggle/input/identify-contrails/identify-contrails_2023*.h5 .\n    %ll","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if LOAD_CHECKPOINT:\n    prev_checkpoint_path = os.path.join(WEIGHTS_DIR, 'identify-contrails_2023-07-31_23-01-22.h5')  # <DEVEL>\n    print(f'prev_checkpoint_path = {prev_checkpoint_path}')\n    if not os.path.exists(prev_checkpoint_path):\n        raise IOError(f'file does not exist {prev_checkpoint_path}')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!touch submission.csv\n%ll -h","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport datetime\nimport itertools\nimport math\nimport multiprocessing\nimport pathlib\nimport random\nimport shutil\nimport toml\n\nfrom pprint import pprint\nfrom pytz import timezone\nfrom tqdm.notebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport scipy","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PRINT_TIME_FORMAT = \"%Y-%m-%d %H:%M:%S %Z%z\"\nFILE_TIME_FORMAT = \"%Y-%m-%d_%H-%M-%S\"\n\nstart_time = datetime.datetime.now(timezone('CET'))\n\nfile_time_str = start_time.strftime(FILE_TIME_FORMAT)\n\nprint('Started', start_time.strftime(PRINT_TIME_FORMAT))","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import backend as backend\n\nAUTOTUNE = tf.data.AUTOTUNE","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('TensorFlow version:', tf.__version__)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Num CPUs Available: \", len(tf.config.list_physical_devices('CPU')))\nprint(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))\nprint('cpu_count: ', multiprocessing.cpu_count())","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = tf.distribute.MirroredStrategy()\nn_replicas = strategy.num_replicas_in_sync  # <STRATEGY>\n#n_replicas = 1  # <STRATEGY>\nprint(f'Number of devices in strategy: {n_replicas}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#---------------------------------------------------------------------------79","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset utils","metadata":{}},{"cell_type":"code","source":"class Paths:\n    train = os.path.join(DATA_DIR, 'train')\n    valid = os.path.join(DATA_DIR, 'validation')\n    test = os.path.join(DATA_DIR, 'test')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ids = sorted(os.listdir(Paths.train))\nvalid_ids = sorted(os.listdir(Paths.valid))\ntest_ids = sorted(os.listdir(Paths.test))\nprint('n_samples (train, validation, test) =', len(train_ids), len(valid_ids), len(test_ids))","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ABI:\n    bands = {name: idx for idx, name in enumerate([\n        '08', '09', '10', '11', '12', '13', '14', '15', '16'])}\n    colors = {name: idx for idx, name in enumerate([\n        'red', 'blue', 'green', 'orange', 'purple', 'cyan', 'magenta', 'yellow', 'black'])}","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_TIMES_BEFORE = 4\nN_TIMES_AFTER = 3\nN_TIMES = N_TIMES_BEFORE + N_TIMES_AFTER + 1","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FRAMES = 1  # <DEVEL> \nN_FRAMES_BEFORE = int((N_FRAMES - 1) / 2)  # N_FRAMES centered around N_TIMES_BEFORE\n#N_FRAMES_BEFORE = 3  # last frame of 4","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\n_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef get_ash_colors(sample_id, split_dir):\n    \"\"\"\n    Based on bands: 11, 14, 15\n    \n    Args:\n        sample_id(str): The id of the example i.e. '1000216489776414077'\n        split_dir(str): The split directoryu i.e. 'test', 'train', 'val'\n    \"\"\"\n    band15 = np.load(DATA_DIR + f\"/{split_dir}/{sample_id}/band_15.npy\")\n    band14 = np.load(DATA_DIR + f\"/{split_dir}/{sample_id}/band_14.npy\")\n    band11 = np.load(DATA_DIR + f\"/{split_dir}/{sample_id}/band_11.npy\")\n\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    ash_colors = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    \n    return ash_colors","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_individual_mask(sample_id, split_dir):\n    masks_path = DATA_DIR + f\"/{split_dir}/{sample_id}/human_individual_masks.npy\"\n    pixel_mask = np.load(masks_path)\n    return pixel_mask\n\ndef get_pixel_mask(sample_id, split_dir):\n    masks_path = DATA_DIR + f\"/{split_dir}/{sample_id}/human_pixel_masks.npy\"\n    pixel_mask = np.load(masks_path)\n    return pixel_mask","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Check some values**","metadata":{}},{"cell_type":"code","source":"sample_id = 7829917977180135058  # train_ids[3]\n\nprint(f'Check `ash_colors` on one of the samples: {sample_id}')\n\nash_colors = get_ash_colors(sample_id, 'train')[..., N_TIMES_BEFORE]\n\nprint(ash_colors.shape)\nfor color in range(3):\n    array = ash_colors[..., color]\n    print(array.min(), array.max())","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pixel_mask = get_pixel_mask(sample_id, 'train')\n\nprint(pixel_mask.shape)\nprint(pixel_mask.min(), pixel_mask.max())","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"execution":{"iopub.execute_input":"2023-07-10T19:30:33.222846Z","iopub.status.busy":"2023-07-10T19:30:33.222338Z","iopub.status.idle":"2023-07-10T19:30:33.228013Z","shell.execute_reply":"2023-07-10T19:30:33.227016Z","shell.execute_reply.started":"2023-07-10T19:30:33.222811Z"}}},{"cell_type":"markdown","source":"### Config","metadata":{}},{"cell_type":"code","source":"SEED = 42","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:  # <CONFIG>\n    \n    seed = SEED\n\n    img_size = (256, 256)\n    num_classes = 1\n    \n    augment = True  # <DEVEL>\n    upsample = False  # <DEVEL>\n    \n    model = 'unet'  # unet | deeplabv3plus | p3dresnet\n    preprocess = None  # 'resnet50' | None\n    backbone = None  # 'resnet50' | 'p3dresnet' | None\n    backbone_weights = resnet50_imagenet_weights  # resnet50_imagenet_weights | 'imagenet' | None\n    backbone_trainable = True\n    dropout = False\n    \n    num_epochs = 40  # <DEVEL> else 10\n    batch_size_per_replica = 16  # <DEVEL> else 16 or 32\n    batch_size = batch_size_per_replica * n_replicas\n    steps_per_update = 1  # gradient accumulation\n    \n    initial_learning_rate = 0.001\n    decay_steps = 10  # number of epochs: 5\n    decay_rate = 0.9\n\n    threshold = 'auto'","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://keras.io/examples/keras_recipes/reproducibility_recipes/\n\n# Set the seed using keras.utils.set_random_seed. This will set:\n# 1) `numpy` seed\n# 2) `tensorflow` random seed\n# 3) `python` random seed\nkeras.utils.set_random_seed(Config.seed)\n\n# See also:\n# tf.config.experimental.enable_op_determinism()","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Mixed Precision\n# https://www.tensorflow.org/guide/mixed_precision#supported_hardware\n\nif PLATFORM == 'kaggle':\n    MIXED_PRECISION = True\nelif PLATFORM == 'gcp':\n    # No mixed precision on QCP:\n    # \"Your GPU may run slowly with dtype policy mixed_float16 because it does not have compute capability of at least 7.0.\n    # Your GPU: Tesla P100-PCIE-16GB, compute capability 6.0\"\n    MIXED_PRECISION = False\nelse:\n    raise NotImplementedError(f'unknown platform \"{PLATFORM}\"')\n\nif MIXED_PRECISION:\n    print('setting mixed_precision')\n\n    NP_FLOAT = 'float16'\n    TF_FLOAT = tf.float16\n    TF_INT = tf.uint8\n    \n    keras.mixed_precision.set_global_policy('mixed_float16')\n    final_dtype = 'float32'\n    \nelse:\n    print('no mixed_precision')\n\n    NP_FLOAT = 'float32'\n    TF_FLOAT = tf.float32\n    TF_INT = tf.uint8\n\n    final_dtype = None","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pseudo-3D ResNet","metadata":{}},{"cell_type":"code","source":"class P3DResNet:\n    '''Pseudo-3D Resnet model.\n    \n    Adapted from:\n    - https://github.com/qijiezhao/pseudo-3d-pytorch/blob/master/p3d_model.py\n    - https://github.com/yfxc/pseudo-3d-tensorflow/blob/master/P3D.py\n    - https://github.com/keras-team/keras/blob/v2.13.1/keras/applications/resnet.py\n    \n    Reference:\n    - https://www.tensorflow.org/tutorials/video/video_classification\n    - https://www.tensorflow.org/tutorials/video/transfer_learning_with_movinet\n    '''\n    \n    def __init__(self, dropout=True):\n        \n        self.dropout = dropout\n        \n        self.filter_expansion = 4\n        self.blocks_per_stack = [3, 4, 6, 3]  # P3D63 modelbased on a ResNet-50-3D model\n        self.stack_is_3d = [True, True, True, False]\n    \n    def maxpool_s(self, x, name):\n        x = keras.layers.MaxPooling3D(\n            pool_size=(1, 3, 3), strides=(1, 2, 2), padding='same', name=name)(x)\n        if self.dropout:\n            x = tf.keras.layers.Dropout(rate=0.1)(x)\n        return x\n    \n    def maxpool_t(self, x, name):\n        x = keras.layers.MaxPooling3D(\n            pool_size=(2, 1, 1), strides=(2, 1, 1), padding='same', name=name)(x)\n        if self.dropout:\n            x = tf.keras.layers.Dropout(rate=0.1)(x)\n        return x\n    \n    @staticmethod\n    def avgpool(x):\n        return keras.layers.AveragePooling2D()(x)\n    \n    @staticmethod\n    def conv_2d(x, filters, kernel_size, strides, padding=\"valid\", activation=None, name=None):\n        '''2D convolution with normalization and optional activation'''\n        x = layers.Conv2D(\n            filters, kernel_size, strides=strides, padding=padding, name=name + \"_conv\"\n            )(x)\n        x = layers.BatchNormalization(\n            epsilon=1.001e-5, name=name + \"_bn\"\n        )(x)\n        if activation is not None:\n            x = layers.Activation(activation, name=name + \"_\" + activation)(x)\n        return x\n\n    @staticmethod\n    def conv_3d(x, filters, kernel_size, strides, padding=\"valid\", activation=None, name=None):\n        '''3D convolution with normalization and optional activation'''\n        use_bias = True  # TODO: use_bias or not?\n        x = layers.Conv3D(\n            filters, kernel_size, strides=strides, padding=padding, name=name + \"_conv\",\n            use_bias=use_bias\n        )(x)\n        x = layers.BatchNormalization(\n            epsilon=1.001e-5, name=name + \"_bn\"\n        )(x)\n        if activation is not None:\n            x = layers.Activation(activation, name=name + \"_\" + activation)(x)\n        return x\n\n    @classmethod\n    def conv_s(cls, x, filters, kernel_size, stride, activation, name):\n        '''Spatial (3D) convolution'''\n        x = cls.conv_3d(\n            x, filters, kernel_size=(1, kernel_size, kernel_size), strides=(1, stride, stride),\n            padding='same', activation=activation, name=name)\n        return x\n    \n    @classmethod\n    def conv_t(cls, x, filters, kernel_size, stride, activation, name):\n        '''Temporal (3D) convolution'''\n        x = cls.conv_3d(\n            x, filters, kernel_size=(kernel_size, 1, 1), strides=(stride, 1, 1),\n            padding='same', activation=activation, name=name)\n        return x\n    \n    def block(self, x, filters, kernel_size=3, stride=1, conv_shortcut=True, block_type='2d', name=None):\n        \"\"\"A residual block.\n\n        Args:\n          x: input tensor.\n          filters: integer, filters of the bottleneck layer.\n          kernel_size: default 3, kernel size of the bottleneck layer.\n          stride: default 1, stride of the first layer.\n          conv_shortcut: default True, use convolution shortcut if True,\n              otherwise identity shortcut.\n          name: string, block label.\n\n        Returns:\n          Output tensor for the residual block.\n        \"\"\"\n        bn_axis = \"channels_last\"\n        \n        if block_type == '2d':\n            conv_xd = self.conv_2d\n        else:\n            conv_xd = self.conv_3d\n        \n        if conv_shortcut:\n            # 1 x 1 conv\n            shortcut = conv_xd(\n                x, self.filter_expansion * filters, 1, strides=stride, name=name + \"_0\")\n        else:\n            shortcut = x\n        \n        # 1 x 1 conv\n        x = conv_xd(x, filters, 1, strides=stride, activation='relu', name=name + \"_1\")\n\n        # 3 x 3 conv\n        if block_type == '2d':\n            x = self.conv_2d(\n                x, filters, kernel_size, padding=\"same\", activation='relu', name=name + \"_2\")\n        elif block_type == 'A':\n            x = self.conv_s(x, filters, kernel_size, stride=1, activation='relu', name=name + \"_2_s\")\n            x = self.conv_t(x, filters, kernel_size, stride=1, activation='relu', name=name + \"_2_t\")\n\n        # 1 x 1 conv, with filter expansion\n        x = conv_xd(x, self.filter_expansion * filters, 1, 1, name=name + \"_3\")\n\n        x = layers.Add(name=name + \"_add\")([shortcut, x])\n        x = layers.Activation(\"relu\", name=name + \"_out\")(x)\n        \n        return x\n    \n    def stack(self, x, filters, blocks, stride, shortcut, block_type, name=None):\n        \"\"\"A set of stacked residual blocks.\n\n        Args:\n          x: input tensor.\n          filters: integer, filters of the bottleneck layer in a block.\n          blocks: integer, blocks in the stacked blocks.\n          stride1: default 2, stride of the first layer in the first block.\n          name: string, stack label.\n\n        Returns:\n          Output tensor for the stacked blocks.\n        \"\"\"\n        x = self.block(x, filters, stride=stride, block_type=block_type, name=name + \"_block1\")\n        for i in range(2, blocks + 1):\n            x = self.block(\n                x, filters, conv_shortcut=False, block_type=block_type, name=name + \"_block\" + str(i)\n            )\n        return x\n    \n    def initial(self, x, name, stride=2):\n        x = keras.layers.Conv3D(\n                64, kernel_size=(1, 7, 7), strides=(1, stride, stride), padding='same',\n                use_bias=False, name=name + '_conv')(x)\n        x = keras.layers.BatchNormalization(name=name + '_bn')(x)\n        x = keras.layers.ReLU(name=name + '_relu')(x)\n        return x\n        \n    def model(self, image_size, num_classes, initial_stride=2, model_input=None):\n        \n        # (B, T, H, W, C) in this code\n        # (B, C, T, H, W) in pseudo-3d-pytorch\n        # (B, T, H, W, C) in pseudo-3d-tensorflow\n    \n        if model_input is None:\n            model_input = keras.Input(shape=(N_FRAMES, image_size, image_size, 3), name='input')\n        \n        x = self.initial(model_input, name=\"conv1\", stride=initial_stride)\n        x = self.maxpool_s(x, name='maxpool1_s')\n\n        shortcut = 'n.a.'\n        x = self.stack(\n            x, 64, self.blocks_per_stack[0], stride=1, shortcut=shortcut, block_type='A', name=\"conv2\")\n        x = self.maxpool_t(x, name='maxpool2_t')\n        \n        x = self.stack(\n            x, 128, self.blocks_per_stack[1], stride=2, shortcut=shortcut, block_type='A', name=\"conv3\")\n        x = self.maxpool_t(x, name='maxpool3_t')\n        \n        x = self.stack(\n            x, 256, self.blocks_per_stack[2], stride=2, block_type='A', shortcut=shortcut, name=\"conv4\")\n        x = self.maxpool_t(x, name='maxpool4')\n        \n        x = self.stack(\n            x, 512, self.blocks_per_stack[3], stride=2, block_type='A', shortcut=shortcut, name=\"conv5\")\n        model_output = x\n        \n        return keras.Model(name=self.__class__.__name__, inputs=model_input, outputs=model_output)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### U-Net","metadata":{}},{"cell_type":"code","source":"class UNet:\n    '''U-Net model.\n    \n    Inspired by and adapted from:\n    - https://keras.io/examples/vision/oxford_pets_image_segmentation\n    - https://www.kaggle.com/code/shashwatraman/simple-unet-baseline-train-lb-0-580\n    - https://www.coursera.org/learn/advanced-computer-vision-with-tensorflow/home/week/3\n    '''\n    \n    def __init__(self, preprocess=None, backbone=None, weights='imagenet',\n                 backbone_trainable=True, dropout=True):\n        self.preprocess = preprocess\n        self.backbone = backbone\n        self.weights = weights\n        self.backbone_trainable = backbone_trainable\n        self.dropout = dropout\n        self.multiframe = (backbone == 'p3dresnet')\n        \n    def conv2d_block(self, input_tensor, n_filters, kernel_size=3):\n        x = input_tensor\n        for i in range(2):\n            x = tf.keras.layers.SeparableConv2D(\n                filters = n_filters, kernel_size=(kernel_size, kernel_size), padding='same')(x)\n            #? kernel_initializer = 'he_normal'\n            x = tf.keras.layers.BatchNormalization()(x)\n            x = tf.keras.layers.Activation('relu')(x)\n        return x\n\n    def encoder_block(self, inputs, n_filters, pool_size, dropout):\n        f = self.conv2d_block(inputs, n_filters=n_filters)\n        p = tf.keras.layers.MaxPooling2D(pool_size)(f)\n        p = tf.keras.layers.Dropout(dropout)(p)\n        return f, p\n\n    def basic_encoder(self, inputs, dropout=0.1):\n        f1, p1 = self.encoder_block(inputs, n_filters=64, pool_size=(2,2), dropout=dropout)\n        f2, p2 = self.encoder_block(p1, n_filters=128, pool_size=(2,2), dropout=dropout)\n        f3, p3 = self.encoder_block(p2, n_filters=256, pool_size=(2,2), dropout=dropout)\n        f4, p4 = self.encoder_block(p3, n_filters=512, pool_size=(2,2), dropout=dropout)\n        # activation_1/Relu:0 (None, 256, 256, 64)\n        # activation_3/Relu:0 (None, 128, 128, 128)\n        # activation_5/Relu:0 (None, 64, 64, 256)\n        # activation_7/Relu:0 (None, 32, 32, 512)        \n        return p4, (f1, f2, f3, f4)\n    \n    def effnet_encoder(self, inputs, dropout=0.1):\n        \n        effnetb3 = keras.applications.EfficientNetB3(\n            weights=self.weights, include_top=False, input_tensor=inputs,\n        )\n        effnetb3.trainable = self.backbone_trainable\n        print('backbone_trainable =', self.backbone_trainable)\n        \n        raise NotImplementedError('WIP')\n        \n        f1 = None  # effnetb3.get_layer(\"conv1_relu\").output\n        f2 = None  # effnetb3.get_layer(\"conv2_block3_out\").output\n        f3 = None  # effnetb3.get_layer(\"conv3_block4_out\").output\n        f4 = None  # effnetb3.get_layer(\"conv4_block5_out\").output\n        output = effnetb3.output\n        \n        return output, (f1, f2, f3, f4)\n    \n    def resnet_encoder(self, inputs, dropout=0.1):\n        \n        # The first Conv2D of the keras implementation of ResNet has strides=2:\n        # x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name=\"conv1_conv\")(x)\n        # i.e. the image is downsampled first.\n        # To avoid this, we apply an upsample in front.\n        upsample = keras.layers.UpSampling2D()(inputs) \n\n        resnet50 = keras.applications.ResNet50(\n            weights=self.weights, include_top=False, input_tensor=upsample,\n        )\n        resnet50.trainable = self.backbone_trainable\n        print('backbone_trainable =', self.backbone_trainable)\n        \n        f1 = resnet50.get_layer(\"conv1_relu\").output\n        f2 = resnet50.get_layer(\"conv2_block3_out\").output\n        f3 = resnet50.get_layer(\"conv3_block4_out\").output\n        f4 = resnet50.get_layer(\"conv4_block5_out\").output\n        output = resnet50.output\n\n        return output, (f1, f2, f3, f4)\n    \n    def p3dresnet_encoder(self, inputs, image_size, dropout=0.1):\n        \n        # The first Conv2D of the keras implementation of ResNet has strides=2:\n        # x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name=\"conv1_conv\")(x)\n        # i.e. the image is downsampled first.\n        # To avoid this, we apply an upsample in front.\n#        upsample = keras.layers.UpSampling2D()(inputs) \n\n        backbone = P3DResNet(dropout=self.dropout).model(\n            image_size=image_size, num_classes=Config.num_classes, initial_stride=1, model_input=inputs)\n\n#        x = backbone.get_layer(\"conv4_block6_2_t_relu\").output\n#        x = keras.layers.Reshape((16, 16, 256))(x)\n\n#        input_b = backbone.get_layer(\"conv2_block3_out\").output\n#        input_b = keras.layers.AveragePooling3D((5, 1, 1))(input_b)\n#        input_b = keras.layers.Reshape((64, 64, 256))(input_b)\n\n        f1 = backbone.get_layer(\"conv1_relu\").output\n        f2 = backbone.get_layer(\"conv2_block3_out\").output\n        f3 = backbone.get_layer(\"conv3_block4_out\").output\n        f4 = backbone.get_layer(\"conv4_block5_out\").output\n        output = backbone.output\n        \n        # conv1_relu/Relu:0 (None, 5, 256, 256, 64)\n        # conv2_block3_out/Relu:0 (None, 5, 128, 128, 256)\n        # conv3_block4_out/Relu:0 (None, 2, 64, 64, 512)\n        # conv4_block5_out/Relu:0 (None, 1, 32, 32, 1024)        \n        \n        f1 = keras.layers.Conv3D(filters=64, kernel_size=(5,1,1), padding='valid')(f1)\n        f1 = keras.layers.Reshape((256, 256, 64))(f1)\n        f2 = keras.layers.Conv3D(filters=256, kernel_size=(5,1,1), padding='valid')(f2)\n        f2 = keras.layers.Reshape((128, 128, 256))(f2)\n        f3 = keras.layers.Conv3D(filters=512, kernel_size=(2,1,1), padding='valid')(f3)\n        f3 = keras.layers.Reshape((64, 64, 512))(f3)\n        f4 = keras.layers.Reshape((32, 32, 1024))(f4)\n        output = keras.layers.Reshape((16, 16, 2048))(output)\n        \n        return output, (f1, f2, f3, f4)\n    \n    def encoder(self, inputs, image_size, dropout=0.1):\n        if self.backbone == 'resnet50':\n            return self.resnet_encoder(inputs, dropout)\n        elif self.backbone == 'p3dresnet':\n            return self.p3dresnet_encoder(inputs, image_size, dropout)\n        elif self.backbone == 'effnetb3':\n            return self.effnet_encoder(inputs, dropout)\n        elif self.backbone is None:\n            return self.basic_encoder(inputs, dropout)\n        raise NotImplementedError(f'unknown backbone \"{self.backbone}')\n\n    def bottleneck(self, inputs):\n        bottle_neck = self.conv2d_block(inputs, n_filters=1024)\n        return bottle_neck\n\n    def decoder_block(self, inputs, conv_output, n_filters, kernel_size, strides, dropout):\n        u = tf.keras.layers.Conv2DTranspose(\n            n_filters, kernel_size, strides=strides, padding='same')(inputs)\n        u = tf.keras.layers.BatchNormalization()(u)\n        c = tf.keras.layers.concatenate([u, conv_output])\n        c = tf.keras.layers.Dropout(dropout)(c)\n        c = self.conv2d_block(c, n_filters, kernel_size=3)\n        return c\n\n    def decoder(self, inputs, convs, num_classes, dropout=0.1):\n        if self.backbone == 'resnet50':\n            filters = [1024, 512, 256, 64]\n        if self.backbone == 'p3dresnet':\n            filters = [1024, 512, 256, 64]\n        elif self.backbone is None:\n            filters = [512, 256, 128, 64]\n        else:\n            raise NotImplementedError(f'unknown backbone \"{self.backbone}')\n        f1, f2, f3, f4 = convs\n        \n        kernel_size = (3, 3)\n        strides = (2, 2)\n\n        c6 = self.decoder_block(inputs, f4, n_filters=filters[0], kernel_size=kernel_size, strides=strides, dropout=dropout)\n        c7 = self.decoder_block(c6, f3, n_filters=filters[1], kernel_size=kernel_size, strides=strides, dropout=dropout)\n        c8 = self.decoder_block(c7, f2, n_filters=filters[2], kernel_size=kernel_size, strides=strides, dropout=dropout)\n        c9 = self.decoder_block(c8, f1, n_filters=filters[3], kernel_size=kernel_size, strides=strides, dropout=dropout)\n\n        if num_classes == 1:\n            activation = \"sigmoid\"\n        else:\n            activation = \"softmax\"\n        outputs = layers.Conv2D(\n            num_classes, kernel_size=3, activation=activation, padding=\"same\", dtype=final_dtype)(c9)\n        return outputs\n\n    def model(self, image_size, num_classes):\n        if self.multiframe:\n            inputs = keras.Input(shape=(N_FRAMES, image_size, image_size, 3), name='input')\n        else:\n            inputs = keras.layers.Input(shape=(image_size,image_size,3), name='input')\n        #inputs = keras.layers.UpSampling3D(size=(1,2,2))(inputs)\n        encoder_output, convs = self.encoder(inputs, image_size)\n        for conv in convs:\n            print(conv.name, conv.shape)\n        bottle_neck = self.bottleneck(encoder_output)\n        outputs = self.decoder(bottle_neck, convs, num_classes)\n        #outputs = keras.layers.MaxPooling3D(pool_size=(1,2,2))(outputs)\n        model = tf.keras.Model(name=self.__class__.__name__, inputs=inputs, outputs=outputs)\n        return model","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DeepLabV3+","metadata":{}},{"cell_type":"code","source":"class DeepLabV3Plus:\n    '''DeepLabV3+ model.\n    \n    Adapted from:\n    - https://keras.io/examples/vision/deeplabv3_plus/#inference-using-colormap-overlay\n    \n    Dropout from:\n    - https://github.com/smspillaz/seg-reg\n    '''\n    \n    def __init__(self, preprocess='resnet50', backbone='resnet50', weights='imagenet', backbone_trainable=True,\n                 dropout=False, upsample=Config.upsample):\n        self.preprocess = preprocess\n        self.backbone = backbone\n        self.weights = weights\n        self.backbone_trainable = backbone_trainable\n        self.dropout = dropout\n        self.upsample = upsample\n        self.multiframe = (backbone == 'p3dresnet')\n    \n    def convolution_block(\n        self,\n        block_input,\n        num_filters=256,\n        kernel_size=3,\n        dilation_rate=1,\n        padding=\"same\",\n        use_bias=False,\n    ):\n        x = layers.Conv2D(\n            num_filters,\n            kernel_size=kernel_size,\n            dilation_rate=dilation_rate,\n            padding=\"same\",\n            use_bias=use_bias,\n            kernel_initializer=keras.initializers.HeNormal(),\n        )(block_input)\n        x = layers.BatchNormalization()(x)\n        return tf.nn.relu(x)\n\n    def DilatedSpatialPyramidPooling(self, dspp_input):\n        dims = dspp_input.shape\n        x = layers.AveragePooling2D(pool_size=(dims[-3], dims[-2]))(dspp_input)\n        \n        x = self.convolution_block(x, kernel_size=1, use_bias=True)\n        out_pool = layers.UpSampling2D(\n            size=(dims[-3] // x.shape[1], dims[-2] // x.shape[2]), interpolation=\"bilinear\",\n        )(x)\n\n        out_1 = self.convolution_block(dspp_input, kernel_size=1, dilation_rate=1)\n        out_6 = self.convolution_block(dspp_input, kernel_size=3, dilation_rate=6)\n        out_12 = self.convolution_block(dspp_input, kernel_size=3, dilation_rate=12)\n        out_18 = self.convolution_block(dspp_input, kernel_size=3, dilation_rate=18)\n\n        x = layers.Concatenate(axis=-1)([out_pool, out_1, out_6, out_12, out_18])\n        output = self.convolution_block(x, kernel_size=1)\n\n        if self.dropout:\n            output = tf.keras.layers.Dropout(0.1)(output)\n        \n        return output\n    \n    def model(self, image_size, num_classes):\n        \n        if self.backbone == 'resnet50':\n\n            upsample = False\n\n            model_input = keras.Input(shape=(image_size, image_size, 3))\n        \n            backbone = keras.applications.ResNet50(\n                weights=self.weights, include_top=False, input_tensor=model_input,\n            )\n            backbone.trainable = self.backbone_trainable\n            print('backbone.trainable =', backbone.trainable)\n            \n            x = backbone.get_layer(\"conv4_block6_2_relu\").output\n            input_b = backbone.get_layer(\"conv2_block3_2_relu\").output\n        \n        elif self.backbone == 'p3dresnet':\n            \n            model_input = keras.Input(shape=(N_FRAMES, image_size, image_size, 3), name='input')\n            \n            if self.upsample:\n                re_input = keras.layers.UpSampling3D(size=(1,2,2))(model_input)\n                image_size_up = image_size * 2\n            else:\n                image_size_up = image_size\n                re_input = model_input\n\n            backbone = P3DResNet(dropout=self.dropout).model(\n                image_size=image_size_up, num_classes=Config.num_classes, model_input=re_input)\n            \n            x = backbone.get_layer(\"conv4_block6_2_t_relu\").output\n            if self.upsample:\n                x_shape = (32, 32, 256)\n            else:\n                x_shape = (16, 16, 256)\n            x = keras.layers.Reshape(x_shape)(x)\n\n            input_b = backbone.get_layer(\"conv2_block3_out\").output\n            if False:  # <DEVEL> Average or Max or Conv3D?\n                input_b = keras.layers.Conv3D(\n                    filters=256, kernel_size=(N_FRAMES,1,1), padding='valid')(input_b)\n            else:\n                input_b = keras.layers.AveragePooling3D((N_FRAMES, 1, 1))(input_b)\n            if self.upsample:\n                b_shape = (128, 128, 256)\n            else:\n                b_shape = (64, 64, 256)\n            input_b = keras.layers.Reshape(b_shape)(input_b)\n        \n        else:\n            raise NotImplementedError(f'unknown backbone \"{self.backbone}\"')\n        \n        x = self.DilatedSpatialPyramidPooling(x)\n\n        input_a = layers.UpSampling2D(\n            size=(image_size_up // 4 // x.shape[1], image_size_up // 4 // x.shape[2]),\n            interpolation=\"bilinear\",\n        )(x)\n        input_b = self.convolution_block(input_b, num_filters=48, kernel_size=1)\n\n        x = layers.Concatenate(axis=-1)([input_a, input_b])\n        x = self.convolution_block(x)\n        #if self.dropout:\n        #    x = tf.keras.layers.Dropout(0.5)(x)\n        x = self.convolution_block(x)\n        #if self.dropout:\n        #    x = tf.keras.layers.Dropout(0.1)(x)\n        x = layers.UpSampling2D(\n            size=(image_size_up // x.shape[1], image_size_up // x.shape[2]),\n            interpolation=\"bilinear\",\n        )(x)\n        \n        if self.dropout:\n            x = tf.keras.layers.Dropout(0.1)(x)\n\n        if num_classes == 1:\n            activation = \"sigmoid\"\n        else:\n            activation = \"softmax\"\n        model_output = layers.Conv2D(\n            num_classes, kernel_size=(1, 1), activation=activation, padding=\"same\", dtype=final_dtype)(x)\n        \n        if self.upsample:\n            model_output = keras.layers.MaxPooling2D(pool_size=(2,2))(model_output)\n        \n        return keras.Model(name=self.__class__.__name__, inputs=model_input, outputs=model_output)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Free up RAM in case the model definition cells were run multiple times\nkeras.backend.clear_session()\n\n# Build model\nif Config.model == 'unet':\n    builder = UNet(\n        preprocess=Config.preprocess,\n        backbone=Config.backbone,\n        weights=Config.backbone_weights,\n        backbone_trainable=Config.backbone_trainable,\n        dropout=Config.dropout\n    )\n\nelif Config.model == 'deeplabv3plus':\n    builder = DeepLabV3Plus(\n        preprocess=Config.preprocess,\n        backbone=Config.backbone,\n        weights=Config.backbone_weights,\n        backbone_trainable=Config.backbone_trainable,\n        dropout=Config.dropout\n    )\n\nelif Config.model =='p3dresnet':\n    builder = P3DResNet()\n\nelse:\n    raise NotImplementedError(f'model \"{Config.model}\"')\n\nwith strategy.scope():  # <STRATEGY>\n    model = builder.model(image_size=Config.img_size[0], num_classes=Config.num_classes)\n\nmodel.summary()","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:\n    # Visualize the model\n    keras.utils.plot_model(model, expand_nested=True, dpi=60, show_shapes=True)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configure datasets (keras)","metadata":{"execution":{"iopub.execute_input":"2023-07-10T19:30:33.222846Z","iopub.status.busy":"2023-07-10T19:30:33.222338Z","iopub.status.idle":"2023-07-10T19:30:33.228013Z","shell.execute_reply":"2023-07-10T19:30:33.227016Z","shell.execute_reply.started":"2023-07-10T19:30:33.222811Z"},"tags":[]}},{"cell_type":"code","source":"# The distributed dataset we use require to explicitly set the number of samples.\nN_TRAIN = 20529  # 512 <DEVEL> else None or 10240 (max 20529)\nN_VALID = 1856  # 128 <DEVEL> else None (max 1856)\nN_PARTIAL = 256  # 256 <DEVEL>","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('preprocess =', builder.preprocess)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Multi-frame","metadata":{}},{"cell_type":"code","source":"class AshColorMultiFrames(keras.utils.Sequence):\n    \"\"\"Helper to iterate over the data (as Numpy arrays).\"\"\"\n\n    def __init__(self, batch_size, img_size, sample_ids, split_dir, preprocess=None, n_samples=None):\n        self.batch_size = batch_size\n        self.img_size = img_size\n        self.split_dir = split_dir\n        self.sample_ids = sample_ids[:n_samples]\n        self.preprocess = preprocess\n\n    def __len__(self):\n        return math.ceil(len(self.sample_ids) / self.batch_size)\n    \n    def get_sample_ids(self, idx):\n        '''Get sample ids of batch idx.'''\n        i = idx * self.batch_size\n        return self.sample_ids[i : i + self.batch_size]\n    \n    def __getitem__(self, idx):\n        \"\"\"Returns tuple (input, target) correspond to batch #idx.\"\"\"\n        \n        batch_sample_ids = self.get_sample_ids(idx)\n        \n        sample_shape = self.img_size + (3,) + (N_FRAMES,)\n        \n        batch_shape = (self.batch_size,) + (N_FRAMES,) + self.img_size + (3,) \n        x = np.zeros(batch_shape, dtype=NP_FLOAT)\n        \n        for j, sample_id in enumerate(batch_sample_ids):\n            \n            # img shape (H x W x 3 x T)\n            img = get_ash_colors(sample_id, self.split_dir)\n            \n            # prep_img shape (T x H x W x 3)\n            prep_img = np.full(sample_shape, np.nan)\n            for frame_idx in range(N_FRAMES):\n                timestep_idx = N_TIMES_BEFORE - N_FRAMES_BEFORE + frame_idx\n                if self.preprocess == 'resnet50':\n                    img_t = keras.applications.resnet50.preprocess_input(img[..., timestep_idx])\n                elif self.preprocess is None:\n                    img_t = img[..., timestep_idx]\n                else:\n                    raise NotImplementedError(f'preprocess \"{self.preprocess}\"')\n                prep_img[..., frame_idx] = img_t\n            \n            x[j] = np.moveaxis(prep_img, 3, 0)\n\n        y = np.zeros((self.batch_size,) + self.img_size + (1,), dtype=\"uint8\")\n        if self.split_dir != 'test':\n            for j, sample_id in enumerate(batch_sample_ids):\n                # img shape (H x W x 1)\n                img = get_pixel_mask(sample_id, self.split_dir)\n                y[j] = img\n        \n        return x, y","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = AshColorMultiFrames(\n    Config.batch_size, Config.img_size, train_ids, 'train',\n    preprocess=builder.preprocess, n_samples=N_TRAIN)\nprint('number of batches:', len(train_set), 'train')\n\nvalid_set = AshColorMultiFrames(\n    Config.batch_size, Config.img_size, valid_ids, 'validation',\n    preprocess=builder.preprocess, n_samples=N_VALID)\nprint('number of batches:', len(valid_set), 'valid')\n\npartial_set = AshColorMultiFrames(\n    Config.batch_size, Config.img_size, valid_ids, 'validation',\n    preprocess=builder.preprocess, n_samples=N_PARTIAL)\nprint('number of batches:', len(partial_set), 'partial')\n\ntest_set = AshColorMultiFrames(\n    Config.batch_size, Config.img_size, test_ids, 'test',\n    preprocess=builder.preprocess)\nprint('number of batches:', len(test_set), 'test')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Check dimensions (x, y) of first batch:')\n\ntrain_set[0][0].shape, train_set[0][1].shape","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate TFRecords","metadata":{}},{"cell_type":"code","source":"class TFDataSetCreator:\n    '''Write TFRecords files and generate a TFRecordDataset from a keras.Sequence.\n    \n    Inspired by:\n    - https://www.tensorflow.org/tutorials/load_data/tfrecord\n    - https://keras.io/examples/keras_recipes/creating_tfrecords\n    - https://keras.io/examples/keras_recipes/tfrecord\n    - https://stackoverflow.com/questions/47861084/how-to-store-numpy-arrays-as-tfrecord\n    \n    References:\n    - https://www.tensorflow.org/guide/data - Build TensorFlow input pipelines\n    - https://www.tensorflow.org/guide/data_performance - Better performance with the tf.data API\n    '''\n    \n    def __init__(self, keras_sequence, train, augment):\n        self.keras_sequence = keras_sequence\n        self.split_dir = keras_sequence.split_dir\n        self.batch_size = keras_sequence.batch_size\n        \n        self.train = train\n        \n        self.augment = augment\n        self.crop_and_resize = False  # <DEVEL>\n        self.flip_left_right = True\n        self.rot90 = True\n        \n        self.keep_existing = True\n        self.records_dir = os.path.join(\n            TEMP_DIR, f'records-multi{N_FRAMES}-{self.batch_size}-{self.split_dir}')\n        self.record_paths = []\n        \n        self.progress_bar = True\n    \n    def write_tfrec(self, batch_idx):\n        #pid = multiprocessing.current_process().pid\n        \n        record_path = os.path.join(self.records_dir, f'batch_{batch_idx:04d}.tfrec')\n        \n        if self.keep_existing and os.path.exists(record_path):\n            return record_path\n        \n        if self.progress_bar:\n            if batch_idx % 100 == 0:\n                print('o', end='')\n            else:\n                print('.', end='')\n\n        x_b, y_b = self.keras_sequence[batch_idx]\n        with tf.io.TFRecordWriter(record_path) as writer:\n            for x, y in zip(x_b, y_b):\n                feature = {\n                    \"x\": tf.train.Feature(\n                        bytes_list=tf.train.BytesList(value=[tf.io.serialize_tensor(x).numpy()])),\n                    \"y\": tf.train.Feature(\n                        bytes_list=tf.train.BytesList(value=[tf.io.serialize_tensor(y).numpy()])),\n                }\n                example = tf.train.Example(features=tf.train.Features(feature=feature))\n                writer.write(example.SerializeToString())\n        \n        return record_path\n    \n    def generate_tfrec(self, keep_existing=True):\n        \n        self.keep_existing = keep_existing\n        \n        records_dir = self.records_dir\n        os.makedirs(records_dir, exist_ok=True)\n        \n        n_records = len(self.keras_sequence)\n        n_procs = multiprocessing.cpu_count() * 2\n        print(f'generating {n_records} records with {n_procs} processes in: {records_dir}')\n        %ll -hd $records_dir\n\n        pool = multiprocessing.Pool(processes=n_procs)\n        batch_indexes = range(n_records)\n        record_paths = sorted(pool.map(self.write_tfrec, batch_indexes))\n        if self.progress_bar:\n            print()\n        \n        !du -sh $records_dir\n        print()\n        \n        self.record_paths = record_paths\n        return self\n    \n    @staticmethod\n    def parse_tfrecord_sample(element):\n        parse_dic = {\n            'x': tf.io.FixedLenFeature([], tf.string),  # Note that it is tf.string, not tf.float32\n            'y': tf.io.FixedLenFeature([], tf.string),  # Note that it is tf.string, not tf.float32\n        }\n        feature = tf.io.parse_single_example(element, parse_dic)\n        feature['x'] = tf.io.parse_tensor(feature['x'], out_type=TF_FLOAT)\n        feature['y'] = tf.io.parse_tensor(feature['y'], out_type=TF_INT)\n        return feature\n\n    @staticmethod\n    def prepare_sample(features):\n        '''Return (x,y) both as Tensors.'''\n        # TODO: reshape tensor\n        #return (tf.reshape(features['x'], Config.img_size + (3,)),\n        #        tf.reshape(features['y'], Config.img_size + (1,)))\n        if builder.multiframe:\n            return (features['x'],\n                    features['y'])\n        else:\n            return (features['x'][N_FRAMES_BEFORE, :, :, :],\n                    features['y'])\n\n    def transform_image(self, boxes, rot90_k, flip_h, flip_v, image, is_label=False):\n        \"\"\"Randomly crop and resize and rotate image or label.\n        \n        - https://github.com/tensorflow/models/blob/.../research/deeplab/core/preprocess_utils.py\n        - https://github.com/keras-team/keras/blob/v2.13.1/keras/layers/preprocessing/image_preprocessing.py\n        \n        Args:\n        image: Image with shape [height, width, 3 or 1].\n\n        Returns:\n        Transformed image.\n        \"\"\"\n\n        if not self.train:\n            return image\n        \n        def transform(img, method):\n            identity = lambda: img\n            flip_left_right = lambda: tf.image.flip_left_right(img)\n            #flip_up_down = lambda: tf.image.flip_up_down(img)\n            if self.crop_and_resize:\n                img = tf.image.crop_and_resize(\n                    tf.expand_dims(img, 0),\n                    boxes, [0],\n                    [Config.img_size[0], Config.img_size[0]], \n                    method=method\n                )[0, ...]\n            if self.flip_left_right:\n                img = tf.case([(tf.equal(flip_h, 1), flip_left_right)],\n                    default=identity)\n            #img = tf.case([(tf.equal(flip_v, 1), flip_up_down)],\n            #    default=identity)\n            if self.rot90:\n                img = tf.image.rot90(img, rot90_k)\n            return img\n        \n        if is_label:\n            image = tf.reshape(image, Config.img_size + (1,))\n            image = transform(image, 'nearest')\n            image = tf.cast(image, TF_INT)\n        else:\n            image = tf.reshape(image, Config.img_size + (3,))\n            image = transform(image, 'bilinear')\n            image = tf.cast(image, TF_FLOAT)\n        \n        return image\n\n    def transform_sample(self, sample_images, sample_label=None):\n        \"\"\"Apply transform_image() to all frames of an image sequence.\"\"\"\n        \n        # samples_images shape (FRAMES, HEIGHT, WIDTH, CHANNELS)\n        \n        y1 = tf.random.uniform([1, 1], minval=0.0, maxval=0.2)\n        x1 = tf.random.uniform([1, 1], minval=0.0, maxval=0.2)\n        y2 = tf.random.uniform([1, 1], minval=0.8, maxval=1.0)\n        x2 = tf.random.uniform([1, 1], minval=0.8, maxval=1.0)\n        boxes = tf.concat([y1, x1, y2, x2], 1)\n        \n        rot90_k = tf.cast(tf.random.uniform([], minval=0., maxval=1.99), tf.int32)\n        flip_h = tf.cast(tf.random.uniform([], minval=0., maxval=1.99), tf.int32)\n        flip_v = tf.cast(tf.random.uniform([], minval=0., maxval=1.99), tf.int32)\n\n        if builder.multiframe:\n            # Images\n            images = tf.unstack(\n                sample_images, num=N_FRAMES, axis=0\n            )\n            transf_images = tf.stack([\n                self.transform_image(boxes, rot90_k, flip_h, flip_v, image)\n                for image in images\n            ])\n            # Label\n            transf_label = self.transform_image(boxes, rot90_k, flip_h, flip_v, sample_label, is_label=True)\n            return transf_images, transf_label\n        else:\n            transf_image = self.transform_image(boxes, rot90_k, flip_h, flip_v, sample_images)\n            transf_label = self.transform_image(boxes, rot90_k, flip_h, flip_v, sample_label, is_label=True)\n            return transf_image, transf_label\n\n    def dataset(self, batch=True, shuffle=True):\n        dataset = (\n            tf.data.TFRecordDataset(self.record_paths, num_parallel_reads=AUTOTUNE)\n            .map(self.parse_tfrecord_sample, num_parallel_calls=AUTOTUNE)\n            .map(self.prepare_sample, num_parallel_calls=AUTOTUNE)\n        )\n        if shuffle:\n            dataset = (\n                dataset\n                .shuffle(self.batch_size * 10)\n            )\n        if self.augment:\n            dataset = (\n                dataset\n                .map(self.transform_sample, num_parallel_calls=AUTOTUNE)\n            )\n        if batch:\n            dataset = (\n                dataset\n                .batch(self.batch_size)\n            )\n        dataset = (\n            dataset\n            .prefetch(AUTOTUNE)\n        )\n        # repeat() required for distributed dataset, see:\n        # - https://github.com/tensorflow/tensorflow/issues/56773\n        # In turn, it requires to set the number of steps in model.fit().\n        dataset = dataset.repeat()  # <STRATEGY>\n        return dataset","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize data augmentation","metadata":{}},{"cell_type":"code","source":"def plot_samples(images, labels):\n    \n    if images[0].ndim != 3:\n        raise IndexError(f'Not a single image, shape: {images[0].shape}')\n    \n    fig, axs = plt.subplots(2, len(images), figsize=(10, 4))\n\n    for idx, (image, label) in enumerate(zip(images, labels)):\n        axs[0, idx].imshow(image.astype('float32'))\n        axs[1, idx].imshow(label.astype('int8'))\n        axs[0, idx].set_axis_off()\n        axs[1, idx].set_axis_off()\n\n    plt.tight_layout() \n    plt.show()\n\n    return","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:  # <DEVEL>\n\n    KEEP_EXISTING = True\n\n    tf.debugging.set_log_device_placement(True)\n\n    # Place tensors on the CPU\n    with tf.device('/CPU:0'):\n\n        tf_train_set = (\n            TFDataSetCreator(train_set, train=True, augment=Config.augment)\n            .generate_tfrec(KEEP_EXISTING)\n            .dataset(batch=False, shuffle=False)\n        )\n\n    elements = []\n    for _ in range(10):\n        for idx, element in enumerate(tf_train_set.take(2)):\n            if idx == 1:\n                elements.append(element)\n    images = [el[0].numpy() for el in elements]\n    labels = [el[1].numpy() for el in elements]\n    if builder.multiframe:\n        images = [image[N_FRAMES_BEFORE] for image in images]\n    plot_samples(images, labels)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Write TFRecords","metadata":{"execution":{"iopub.execute_input":"2023-07-25T11:25:26.289666Z","iopub.status.busy":"2023-07-25T11:25:26.289139Z","iopub.status.idle":"2023-07-25T11:25:26.299911Z","shell.execute_reply":"2023-07-25T11:25:26.298556Z","shell.execute_reply.started":"2023-07-25T11:25:26.289625Z"}}},{"cell_type":"code","source":"KEEP_EXISTING = True  # <DEVEL>\n\ntimeit_start = datetime.datetime.now()\n\n#tf.config.run_functions_eagerly(False)\n#tf.data.experimental.enable_debug_mode()\n\ntf.debugging.set_log_device_placement(True)\n\n# Place tensors on the CPU\nwith tf.device('/CPU:0'):\n\n    tf_train_set = (\n        TFDataSetCreator(train_set, train=True, augment=Config.augment)\n        .generate_tfrec(KEEP_EXISTING)\n        .dataset()\n    )\n\n    tf_valid_set = (\n        TFDataSetCreator(valid_set, train=False, augment=False)\n        .generate_tfrec(KEEP_EXISTING)\n        .dataset(shuffle=False)\n    )\n\n    tf_partial_set = (\n        TFDataSetCreator(partial_set, train=False, augment=False)\n        .generate_tfrec(KEEP_EXISTING)\n        .dataset(shuffle=False)\n    )\n\n    tf_test_set = (\n        TFDataSetCreator(test_set, train=False, augment=False)\n        .generate_tfrec(KEEP_EXISTING)\n        .dataset(shuffle=False)\n    )\n\ntf.debugging.set_log_device_placement(False)\n\ntimeit_end = datetime.datetime.now()\nprint('datasets generated in:', timeit_end - timeit_start)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if False:  # <STRATEGY>\n    tf_train_set = strategy.experimental_distribute_dataset(tf_train_set)\n    tf_valid_set = strategy.experimental_distribute_dataset(tf_valid_set)\n    tf_partial_set = strategy.experimental_distribute_dataset(tf_partial_set)\n    tf_test_set = strategy.experimental_distribute_dataset(tf_test_set)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"def dice_coef(y_true, y_pred, smooth=0.001, threshold=None, tensors=True):\n    '''Dice coefficient.\n    \n    Adapted from:\n    - https://stackoverflow.com/questions/72195156/correct-implementation-of-dice-loss-in-tensorflow-keras\n    - https://www.kaggle.com/code/shashwatraman/simple-unet-baseline-train-lb-0-580\n    \n    The \"tensors\" argument is set to False for the final evaluation of the model,\n    since using the distributed model made trouble as this point.\n    \n    Parameters\n    ----------\n    tensors: bool\n        if True, considers y_true and y_pred as tensors, else considers as numpy arrays.\n    '''\n    if tensors:\n        y_true_f = backend.flatten(tf.cast(y_true, TF_FLOAT))\n        y_pred_f = backend.flatten(tf.cast(y_pred, TF_FLOAT))\n    else:\n        y_true_f = y_true.astype(NP_FLOAT).flatten()\n        y_pred_f = y_pred.astype(NP_FLOAT).flatten()\n    if threshold is not None:\n        if tensors:\n            y_pred_f = backend.flatten(\n                tf.cast(tf.math.greater(tf.cast(y_pred, TF_FLOAT), threshold), TF_FLOAT))\n        else:\n            y_pred_f = (y_pred_f > threshold).astype(NP_FLOAT)\n    if tensors:\n        #y_true_f = y_true_f.reshape((Config.img_size))\n        #y_pred_f = y_pred_f.reshape((Config.img_size))\n        intersection = backend.sum(y_true_f * y_pred_f)\n        y_true_sum = backend.sum(y_true_f)\n        y_pred_sum = backend.sum(y_pred_f)\n    else:\n        intersection = np.sum(y_true_f * y_pred_f)\n        y_true_sum = np.sum(y_true_f)\n        y_pred_sum = np.sum(y_pred_f)\n    dice = (2. * intersection + smooth) / (y_true_sum + y_pred_sum + smooth)\n    return dice\n\ndef dice_loss(y_true, y_pred):\n    return 1 - dice_coef(y_true, y_pred)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def threshold_dice_coef(y_true, y_pred, smooth=0.001):\n#    '''Dice coefficient with threshold set to Config.threshold.'''\n#    return dice_coef(y_true, y_pred, smooth=smooth, threshold=Config.threshold)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_id = 7829917977180135058  # train_ids[3]\n\nprint(f'Check dice_coef() on one of the samples: {sample_id}')\n\nmerged_mask = get_pixel_mask(sample_id, 'train')\nindiv_masks = get_individual_mask(sample_id, 'train')\n\nprint(dice_coef(tf.convert_to_tensor(merged_mask),\n                tf.convert_to_tensor(merged_mask)))\nfor idv in range(6):\n    print(dice_coef(tf.convert_to_tensor(merged_mask),\n                    tf.convert_to_tensor(indiv_masks[..., idv])))","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gradient accumulation optimizer","metadata":{}},{"cell_type":"code","source":"class AccumAdam(keras.optimizers.Adam):\n    \"\"\"Adam optimizer with gradient accumulation over specified number of batches.\n    \n    References:\n    - https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer\n    \"\"\"\n    def __init__(self, steps_per_update=1, **kwargs):\n        keras.optimizers.Adam.__init__(self, **kwargs)\n\n        self.steps_per_update = steps_per_update\n        self.step = 0\n        self.accum_grads = {}\n    \n    def build(self, var_list):\n        \n        keras.optimizers.Adam.build(self, var_list)\n        \n        for var in var_list:\n            var_key = var.name\n            self.accum_grads[var_key] = tf.zeros(var.shape)\n    \n    def update_step(self, gradient, variable):\n\n        step_mod = self.iterations % self.steps_per_update\n        var_key = variable.name\n        \n        self.accum_grads[var_key] =\\\n            tf.case([(tf.equal(step_mod, 0), lambda: tf.zeros_like(gradient))],\n                    default=lambda: self.accum_grads[var_key])\n        \n        self.accum_grads[var_key] += gradient\n        \n        accum_grad =\\\n            tf.case([(step_mod == self.steps_per_update - 1, lambda: self.accum_grads[var_key])],\n                    default=lambda: tf.zeros_like(gradient))\n    \n        keras.optimizers.Adam.update_step(self, accum_grad, variable)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Config.steps_per_update == 1:\n    optimizer = keras.optimizers.Adam()\nelse:\n    optimizer = AccumAdam(Config.steps_per_update)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoint_path = f'identify-contrails_{file_time_str}.h5'\nlogger_path = f'identify-contrails_{file_time_str}_log.csv'\nsetup_path = f'identify-contrails_{file_time_str}_setup.toml'\n\nprint(f'checkpoint file: {checkpoint_path}')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning rate scheduler:\n# - https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules\n# - https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/schedules/CosineDecay\n# Note: CosineDecay got warmup from v2.13.1 on\n\ncos_scheduler = keras.optimizers.schedules.CosineDecay(\n    Config.initial_learning_rate, Config.decay_steps)\n\nexp_scheduler = keras.optimizers.schedules.ExponentialDecay(\n    Config.initial_learning_rate, Config.decay_steps, Config.decay_rate)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Configure the model for training.\n\n# We use the \"sparse\" version of categorical_crossentropy\n# because our target data is integers.\n# See also:\n# loss = keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n\noptimizer = 'adam'  # <STRATEGY>\n\nwith strategy.scope():  # <STRATEGY>\n    model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=[dice_coef])\n\ncallbacks = [\n    keras.callbacks.LearningRateScheduler(exp_scheduler),\n    keras.callbacks.ModelCheckpoint(checkpoint_path, save_best_only=False),\n    keras.callbacks.CSVLogger(logger_path, append=True)\n]","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"setup = {\n    'Config': dict(filter(lambda x: not x[0].startswith('__'), Config.__dict__.items())),\n    'Dataset': {\n        'N_TRAIN': N_TRAIN,\n        'N_VALID': N_VALID,\n        'N_TIMES': N_TIMES,\n        'N_TIMES_BEFORE': N_TIMES_BEFORE,\n        'N_FRAMES': N_FRAMES,\n        'N_FRAMES_BEFORE': N_FRAMES_BEFORE,\n    }\n}\nwith open(setup_path, 'w') as setup_fp:\n    toml.dump(setup, setup_fp)\n#%cat *setup*","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LET'S TRAIN!","metadata":{}},{"cell_type":"code","source":"training_steps = N_TRAIN / Config.batch_size  # <STRATEGY>\n#training_steps = None\n\nvalidation_steps = N_VALID / Config.batch_size  # <STRATEGY>\n#validation_steps = None\n\npartial_steps = N_PARTIAL / Config.batch_size  # <STRATEGY>\n#partial_steps = None\n\ntest_steps = int(np.ceil(len(test_ids) / Config.batch_size))  # <STRATEGY>\n#test_steps = None","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if LOAD_CHECKPOINT:\n    # Loads the weights\n    model.load_weights(prev_checkpoint_path)\n    print(f'model loaded weights from {prev_checkpoint_path}')\n\nif TRAIN:\n\n    #tf.config.run_functions_eagerly(True)  # <DEVEL>\n    #tf.data.experimental.enable_debug_mode()\n\n    # Train the model, doing validation at the end of each epoch.\n    history = model.fit(\n        tf_train_set, epochs=Config.num_epochs, validation_data=tf_valid_set, callbacks=callbacks,\n        steps_per_epoch=training_steps, validation_steps=validation_steps,\n        workers=4, use_multiprocessing=True)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### History","metadata":{}},{"cell_type":"code","source":"if TRAIN:\n    print('History', history.history.keys())\n\n    for var, yrange in [('loss', [0.0, 0.02]),\n                        ('dice_coef', [0.0, 0.8])]:\n        plt.figure(figsize=(10, 3))\n        plt.plot(history.history[var])\n        plt.plot(history.history[f'val_{var}'])\n        plt.ylim(yrange[0], yrange[1])\n        plt.title(f'model {var}')\n        plt.xlabel('epoch')\n        plt.ylabel(var)\n        plt.legend(['train', 'val'], loc='upper left')\n        plt.show()","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"def apply_threshold(pred, threshold):\n    return (pred > threshold).astype(np.int8)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EVALUATE = True\nALL_BATCHES = True\nif Config.batch_size >= 16:\n    BATCH_IDX = 0\n    SAMPLE_IDX = 11\nelse:\n    BATCH_IDX = 1\n    SAMPLE_IDX = 3","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if EVALUATE:    \n    # Evaluate the model\n\n    if ALL_BATCHES:\n        loss, acc = model.evaluate(tf_valid_set, steps=validation_steps, verbose=2)\n    else:\n        eval_images, eval_masks = tf_valid_set[BATCH_IDX]\n        loss, acc = model.evaluate(eval_images, eval_masks, steps=validation_steps, verbose=2)\n\n    print(\"Model accuracy: {:5.2f}%\".format(100 * acc))","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def eval_dice_coef(sample_set, pred_set, n_batches, batch_size, threshold):\n    dice_coef_per_batch = np.full(n_batches, np.nan)\n    for idx, sample in enumerate(sample_set.take(n_batches)):\n        _x, y = sample\n        pred = pred_set[idx*batch_size:(idx + 1)*batch_size]\n        _coef = dice_coef(y.numpy(), pred, threshold=threshold, tensors=False)\n        dice_coef_per_batch[idx] = _coef\n    return dice_coef_per_batch\n\nif EVALUATE:\n    \n    predictions = model.predict(tf_partial_set, steps=partial_steps)\n    \n    _coefs = eval_dice_coef(\n        tf_partial_set, predictions, len(partial_set), batch_size=partial_set.batch_size,\n        threshold=None)\n    print(f'w/o threshold: {_coefs.mean():.2%}')\n\n    if Config.threshold == 'auto':\n        best_coef = 0.\n        for threshold in np.arange(0.1, 1.0, 0.1):\n            _coefs = eval_dice_coef(\n                tf_partial_set, predictions, len(partial_set), batch_size=partial_set.batch_size,\n                threshold=threshold)\n            mean_coef = _coefs.mean()\n            if mean_coef > best_coef:\n                best_coef = mean_coef\n                best_thresh = threshold\n            print(f'{threshold:.02} threshold: {mean_coef:.2%}')\n        print(f'best_threshold = {best_thresh:.02}')\n        Config.threshold = best_thresh\n        print('Config.threshold updated')\n    else:\n        threshold = Config.threshold\n        _coefs = eval_dice_coef(\n            tf_partial_set, predictions, len(partial_set), batch_size=partial_set.batch_size,\n            threshold=threshold)\n        mean_coef = _coefs.mean()\n        print(f'{threshold:.02} threshold: {mean_coef:.2%}')","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_prediction(img, truth, pred):\n\n    fig, axs = plt.subplots(1, 4, figsize=(16, 8))\n\n    axs[0].imshow(img)\n    axs[0].set_title(\"Ash Color Image\")\n\n    axs[1].imshow(truth)\n    axs[1].set_title(\"Ground Truth\")\n\n    axs[2].imshow(pred)\n    axs[2].set_title(\"Prediction\")\n\n    axs[3].imshow(img)\n    axs[3].imshow(truth, cmap='Reds', alpha=.3, interpolation='none')\n    axs[3].set_title('Contrail mask on ash color image')\n\n    plt.tight_layout() \n    plt.show()\n\n    return\n\nif EVALUATE:\n    eval_images, eval_masks = list(tf_partial_set.take(2))[BATCH_IDX]\n    #eval_images, eval_masks = partial_set[BATCH_IDX]\n    idx = SAMPLE_IDX\n    pred_idx = BATCH_IDX * Config.batch_size + SAMPLE_IDX\n    threshold = Config.threshold\n    \n    eval_image = eval_images[idx].numpy().astype(np.float32)\n    #eval_image = eval_images[idx][N_FRAMES_BEFORE].astype(np.float32)\n    if builder.multiframe:\n        eval_image = eval_image[N_FRAMES_BEFORE]\n    \n    plot_prediction(\n        eval_image, eval_masks[idx], apply_threshold(predictions[pred_idx], threshold))","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Make predictions on test dataset","metadata":{}},{"cell_type":"code","source":"predictions = model.predict(tf_test_set, steps=test_steps)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(predictions)","metadata":{"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a submission","metadata":{}},{"cell_type":"code","source":"def rle_encode(x, fg_val=1):\n    \"\"\"\n    Args:\n        x:  numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns: run length encoding as list\n    \"\"\"\n\n    dots = np.where(\n        x.T.flatten() == fg_val)[0]  # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if b > prev + 1:\n            run_lengths.extend((b + 1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return run_lengths\n\n\ndef list_to_string(x):\n    \"\"\"\n    Converts list to a string representation\n    Empty list returns '-'\n    \"\"\"\n    if x: # non-empty list\n        s = str(x).replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\")\n    else:\n        s = '-'\n    return s\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_recs = os.listdir(os.path.join(DATA_DIR, 'test'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission.csv'), index_col='record_id')[0:0]\n\nfor test_id, pred in zip(test_ids, predictions):\n    \n    mask = apply_threshold(pred, Config.threshold)\n    \n    # notice the we're converting rec to an `int` here:\n    submission.loc[int(test_id), 'encoded_pixels'] = list_to_string(rle_encode(mask))\n    \nsubmission.to_csv('submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def strf_timedelta(timedelta):\n    total_seconds = timedelta.total_seconds()\n    hours, remainder = divmod(total_seconds, 3600)\n    minutes, seconds = divmod(remainder, 60)\n    return '{:02}:{:02}:{:02}'.format(int(hours), int(minutes), int(seconds))\n\nend_time = datetime.datetime.now(timezone('CET'))\n\nprint('Terminated', end_time.strftime(PRINT_TIME_FORMAT),\n      'in', strf_timedelta(end_time - start_time))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"THIS IS THE END!","metadata":{}}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}