{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"ENV = 'kaggle'\n\nclass CFG:\n    DS_TYPE_SINGLE = 'SINGLE'\n    DS_TYPE_MULTIPLE = 'MULTIPLE'\n    DS_TYPE_SINGLE_ASH = 'SINGLE_ASH'\n    DS_TYPE_MULTIPLE_ASHS = 'MULTIPLE_ASHS'\n\n    DS_TYPE = DS_TYPE_MULTIPLE_ASHS\n    FLOAT_16 = True\n    BATCH_SIZE = 8\n    ORI_SIZE = 256\n    TRAIN_SIZE = 512\n    VALID_SIZE = 512\n    # TRAIN_CROP = True\n    # TRAIN_CROP_2 = False\n    AUG_ROT = 0\n    AUG_JPEG = None\n    AUG_COLOR = None\n#     AUG_NOISE = {'stddev':0.05}\n    AUG_NOISE = None\n#     AUG_FLIP_LR = None\n#     AUG_FLIP_UD = None\n#     AUG_ROT90 = None\n    \n    AUG_FLIP_LR = True\n    AUG_FLIP_UD = True\n    AUG_ROT90 = True\n\n    # ALIGN = None\n    # ALIGN = {'type': 'ds', 'x': 0.5, 'y': 0.5}\n    # ALIGN = {'type': 'rt', 'x': 0.5, 'y': 0.5}\n    ALIGN = {'type': 'ds', 'x': 0.408, 'y': 0.453}\n#     AUG_DROPOUT = None\n    AUG_DROPOUT = {'type': 'dropout2', 'rate': 0.2}\n    BACKBONES = [\n#             'EfficientNetV2B0', \n            # 'EfficientNetV2B1',\n            # 'EfficientNetV2B2', \n#             'EfficientNetV2B3', \n#             'EfficientNetV2S', \n#             'EfficientNetV2M',\n            'EfficientNetV2L',\n#             'EfficientNetB0',\n            # 'EfficientNetB1',\n            # 'EfficientNetB2',\n            # 'EfficientNetB3',\n            # 'EfficientNetB4',\n            # 'EfficientNetB5',\n            # 'EfficientNetB6',\n            # 'EfficientNetB7',\n#             'ResNet50',\n#             'ResNet101',\n#             'ResNet152',\n        ],\n    BACKBONE_CHANNEL = 3\n    WBC_WEIGHT = 10\n    LOSS_AUX = 0.25\n\n    EPOCHS = 35\n    SAVE = True\n    SEED = 23\n    FOLD_SPLITS = 5\n    \n    LOGGING = False\n    \ncfg = CFG()","metadata":{"papermill":{"duration":0.024542,"end_time":"2023-07-05T15:48:44.716052","exception":false,"start_time":"2023-07-05T15:48:44.69151","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport gc\nimport re\nimport json\nimport time\nimport math\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random as rn\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n\n# import icrgw","metadata":{"_kg_hide-output":true,"papermill":{"duration":44.162488,"end_time":"2023-07-05T15:49:28.883589","exception":false,"start_time":"2023-07-05T15:48:44.721101","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"papermill":{"duration":0.013521,"end_time":"2023-07-05T15:49:28.901976","exception":false,"start_time":"2023-07-05T15:49:28.888455","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/icrgw-ds-utils/icrgw.py ./\nimport icrgw","metadata":{"_kg_hide-output":true,"papermill":{"duration":10.446901,"end_time":"2023-07-05T15:49:39.354085","exception":false,"start_time":"2023-07-05T15:49:28.907184","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\n\nicrgw.init(cfg)","metadata":{"papermill":{"duration":1.281756,"end_time":"2023-07-05T15:49:40.645208","exception":false,"start_time":"2023-07-05T15:49:39.363452","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\n# icrgw.cfg = cfg","metadata":{"papermill":{"duration":0.014699,"end_time":"2023-07-05T15:49:40.667873","exception":false,"start_time":"2023-07-05T15:49:40.653174","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# 60 Sec\ndef TimeDistributed(layer, xs):\n    ys = []\n    for i in range (xs.shape.as_list()[1]):\n        y = layer(xs[:, i, :, :, :])\n        ys.append(y)\n    ys = tf.stack(ys, axis=1)\n    return ys\n'''\n'''\n# 50 Sec\ndef TimeDistributed(layer, xs):\n    shape = xs.shape.as_list()\n    print(shape)\n    xs = tf.concat(tf.unstack(xs, axis=1), axis=0)\n    xs = layer(xs)\n    xs = tf.stack(tf.split(xs, shape[1], axis=0), axis=1)\n    return xs\n'''\n'''\ndef TimeDistributed(layer, xs):\n    if isinstance(xs, list) == False:\n        xs = [xs]\n    xs = [tf.concat(tf.unstack(x, axis=1), axis=0) for x in xs]\n    xs = layer(*xs)\n    shape = xs.shape.as_list()\n    xs = tf.stack(tf.split(xs, 8, axis=0), axis=1)\n    \n#     xs = tf.reshape(xs, ([-1, 8] + shape[1:]))\n#     xs = [tf.reshape(x, ([-1, 8] + x.shape.as_list()[1:])) for x in xs]\n    return xs\n'''\n'''\ndef TimeDistributed(layer, xs):\n    if isinstance(xs, list) == False:\n        xs = [xs]\n    xs = [tf.reshape(x, ([-1] + x.shape.as_list()[2:])) for x in xs]\n    xs = layer(*xs)\n    shape = xs.shape.as_list()\n    xs = tf.reshape(xs, ([-1, 8] + shape[1:]))\n#     xs = [tf.reshape(x, ([-1, 8] + x.shape.as_list()[1:])) for x in xs]\n    return xs\n'''\ndef TimeDistributed(layer, xs):\n    xs = tf.keras.layers.TimeDistributed(layer)(xs)\n    return xs\n\ndef create_model(backbone_name):\n    xs_input = tf.keras.layers.Input((8, None, None, cfg.BACKBONE_CHANNEL))\n#     xs_input = tf.keras.layers.Input((8, cfg.TRAIN_SIZE, cfg.TRAIN_SIZE, cfg.BACKBONE_CHANNEL))\n    \n#     encoder_filters=(1536, 816, 224, 40, 16)\n    encoder_filters=(32, 64, 128, 256, 512)\n    decoder_filters=(256, 128, 64, 32, 16)\n    decoder_block = icrgw.DecoderUpsamplingX2Block\n#     decoder_block = DecoderTransposeX2Block\n\n    bs = icrgw.create_backbone(backbone_name, cfg.BACKBONE_CHANNEL)\n#     bs = icrgw.load_backbone('/kaggle/input/icrgw-train-single-ash-256-wbc-0611/', backbone_name)\n\n    xs = xs_input[:, 2:7]\n    ss = []\n\n    for i in range(len(bs)):\n        mt = bs[i]\n        xs = TimeDistributed(mt, xs)\n        ss.append(xs)\n\n    top = ss[-1]\n    \n    ss2 = []\n    for i in range(1, 5):\n        ts = ss[i]\n        ts = TimeDistributed(tf.keras.layers.Conv2D(encoder_filters[i-1], 3, padding='same'), ts)\n        ts = TimeDistributed(tf.keras.layers.BatchNormalization(), ts)\n        ts = TimeDistributed(tf.keras.layers.Activation('swish'), ts)\n        ts = TimeDistributed(tf.keras.layers.UpSampling2D(), ts)\n        ts = TimeDistributed(tf.keras.layers.Conv2D(encoder_filters[i-1], 3, padding='same'), ts)\n        ts = TimeDistributed(tf.keras.layers.BatchNormalization(), ts)\n        ts = TimeDistributed(tf.keras.layers.Activation('swish'), ts)\n        ss2.append(ts)\n    \n    ss3 = []\n    for i in range(4):\n        xp = tf.keras.layers.Conv3D(encoder_filters[i], (2, 1, 1), padding='valid')(ss2[i][:,:2])[:, 0]\n        xp = tf.keras.layers.BatchNormalization()(xp)\n        xp = tf.keras.layers.Activation('swish')(xp)\n        \n        xn = tf.keras.layers.Conv3D(encoder_filters[i], (2, 1, 1), padding='valid')(ss2[i][:,3:])[:, 0]\n        xn = tf.keras.layers.BatchNormalization()(xn)\n        xn = tf.keras.layers.Activation('swish')(xn)\n        xs = tf.concat([\n            ss[i][:,2],\n            xp,\n            xn\n        ], axis = 3)\n        ss3.append(xs)\n        \n    xs = ss[-1][:, 2]\n    skips = ss3[::-1]\n    \n    for i in range(5):\n        if i < len(skips):\n            skip = skips[i]\n        else:\n            skip = None\n            \n        xs = decoder_block(decoder_filters[i], stage=f'dc2_{i}', use_batchnorm=True)(xs, skip)\n        \n    # model head (define number of output classes)\n    ys = tf.keras.layers.Conv2D(\n        filters=1,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        activation='sigmoid',\n        name='ys',\n    )(xs)\n    ysa = tf.keras.layers.Conv2D(\n        filters=1,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        activation='sigmoid',\n        name='ysa',\n    )(xs)\n\n    # create keras model instance\n    model = tf.keras.models.Model(xs_input, [ys, ysa])\n\n    return model\n\n# mt = create_model('EfficientNetV2B0')","metadata":{"papermill":{"duration":10.365104,"end_time":"2023-07-05T15:49:51.04116","exception":false,"start_time":"2023-07-05T15:49:40.676056","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.keras.utils.plot_model(mt)","metadata":{"papermill":{"duration":0.016572,"end_time":"2023-07-05T15:49:51.067138","exception":false,"start_time":"2023-07-05T15:49:51.050566","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mt.summary()","metadata":{"papermill":{"duration":0.015923,"end_time":"2023-07-05T15:49:51.091736","exception":false,"start_time":"2023-07-05T15:49:51.075813","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with icrgw.strategy.scope():\n    train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)\n    train_accuracy = tf.keras.metrics.Mean('train_accuracy', dtype=tf.float32)\n    \n    train_i = tf.keras.metrics.Mean('train_i', dtype=tf.float32)    \n    train_u = tf.keras.metrics.Mean('train_u', dtype=tf.float32) \n    \n    valid_loss = tf.keras.metrics.Mean('valid_loss', dtype=tf.float32)\n    valid_accuracy = tf.keras.metrics.Mean('valid_accuracy', dtype=tf.float32)\n    \n    valid_i = tf.keras.metrics.Mean('valid_i', dtype=tf.float32)    \n    valid_u = tf.keras.metrics.Mean('valid_u', dtype=tf.float32) ","metadata":{"papermill":{"duration":0.285132,"end_time":"2023-07-05T15:49:51.38554","exception":false,"start_time":"2023-07-05T15:49:51.100408","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_dis, valid_dataset_dis, train_steps_per_epoch, valid_steps_per_epoch = icrgw.get_ds()\ntrain_iterator = iter(train_dataset_dis)","metadata":{"papermill":{"duration":0.759601,"end_time":"2023-07-05T15:49:52.154072","exception":false,"start_time":"2023-07-05T15:49:51.394471","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for bb in cfg.BACKBONES[0]:\n\n    with icrgw.strategy.scope():\n        model = create_model(bb)\n#         optimizer = tf.keras.optimizers.Adamax()\n        optimizer = icrgw.Lion()\n\n    train_step, valid_step = icrgw.build_2(model, optimizer, icrgw.wbc_dice_loss,\n            train_loss, train_accuracy, train_i, train_u,\n            valid_loss, valid_accuracy, valid_i, valid_u)\n\n    lr_cb = tf.keras.callbacks.ReduceLROnPlateau(factor=0.2, patience=3, verbose=1, min_lr=1e-8)\n    model.optimizer = optimizer\n    lr_cb.set_model(model)\n\n    target_epochs = cfg.EPOCHS\n\n#     if icrgw.strategy.num_replicas_in_sync == 1:\n#         train_steps_per_epoch = 1\n#     valid_step(valid_dataset_dis)\n\n    for epoch in range(target_epochs):\n        print('\\nEpoch: {}/{}'.format(epoch, target_epochs))\n        start = time.time()\n\n        train_step(train_iterator, train_steps_per_epoch)\n        print(f'train loss: {train_loss.result():.4f}, \\\n            accuracy: {train_accuracy.result():.4f}, \\\n            score: {(2 * train_i.result()/train_u.result()).numpy()}'\n            )\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        train_i.reset_states()\n        train_u.reset_states()\n        \n        if epoch > -1:\n            valid_loss.reset_states()\n            valid_accuracy.reset_states()\n            valid_i.reset_states()\n            valid_u.reset_states()\n        #     for data in valid_dataset_dis:\n        #         valid_step(data)\n            valid_step(valid_dataset_dis)\n            print(f'\\r\\r valid loss: {valid_loss.result():.4f}, \\\n                valid accuracy: {valid_accuracy.result():.4f}, \\\n                score: {(2 * valid_i.result()/valid_u.result()).numpy()}',\n                )\n\n            lr_cb.on_epoch_end(epoch, {'val_loss': valid_loss.result()})\n        \n        elapsed = time.time() - start\n        print(f'{elapsed:.2f} S')\n        \n        if optimizer.lr < 1e-7:\n            break\n            \n    if cfg.SAVE: \n        if cfg.FLOAT_16:\n            model.save('m_16.h5')\n            policy = tf.keras.mixed_precision.Policy('float32')\n            tf.keras.mixed_precision.set_global_policy(policy)\n            mt = create_model(bb)\n            mt.load_weights('m_16.h5')\n            mt.save(f'{bb}.h5')\n            if icrgw.tpu:\n                policy = tf.keras.mixed_precision.Policy('mixed_bfloat16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n            else:\n                policy = tf.keras.mixed_precision.Policy('mixed_float16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n        else:\n            model.save(f'{bb}.h5')","metadata":{"papermill":{"duration":11545.088426,"end_time":"2023-07-05T19:02:17.251234","exception":false,"start_time":"2023-07-05T15:49:52.162808","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.018996,"end_time":"2023-07-05T19:02:17.289665","exception":false,"start_time":"2023-07-05T19:02:17.270669","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}