{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"ENV = 'kaggle'\n\nclass CFG:\n    DS_TYPE_SINGLE = 'SINGLE'\n    DS_TYPE_MULTIPLE = 'MULTIPLE'\n    DS_TYPE_SINGLE_ASH = 'SINGLE_ASH'\n    DS_TYPE_MULTIPLE_ASHS = 'MULTIPLE_ASHS'\n    DS_TYPE_MULTIPLE_PREDS = 'MULTIPLE_PREDS'\n    DS_TYPE_MULTIPLE_C3 = 'MULTIPLE_C3'\n\n    DS_TYPE = DS_TYPE_SINGLE_ASH\n    FLOAT_16 = True\n    BATCH_SIZE = 8\n    ORI_SIZE = 256\n    TRAIN_SIZE = 480\n    VALID_SIZE = 512\n    AUG_ROT = 0\n    AUG_JPEG = None\n    AUG_COLOR = None\n#     AUG_NOISE = None\n    AUG_NOISE = {'stddev':0.18}\n#     AUG_FLIP_LR = None\n#     AUG_FLIP_UD = None\n#     AUG_ROT90 = None\n    \n    AUG_FLIP_LR = True\n    AUG_FLIP_UD = True\n    AUG_ROT90 = True\n\n    # ALIGN = None\n    # ALIGN = {'type': 'ds', 'x': 0.5, 'y': 0.5}\n    # ALIGN = {'type': 'rt', 'x': 0.5, 'y': 0.5}\n    ALIGN = {'type': 'ds', 'x': 0.408, 'y': 0.453}\n\n    AUG_DROPOUT = None\n#     AUG_DROPOUT = {'type': 'dropout', 'rate': 0.2}\n    AUG_COARSE_DROPOUT = None\n#     AUG_COARSE_DROPOUT = {'size': 128, 'threshold': 0.1}\n    BACKBONES = [\n#             'EfficientNetV2B0', \n            # 'EfficientNetV2B1',\n            # 'EfficientNetV2B2', \n            # 'EfficientNetV2B3', \n            # 'EfficientNetV2S', \n            # 'EfficientNetV2M',\n            'EfficientNetV2L',\n#             'EfficientNetB0',\n            # 'EfficientNetB1',\n            # 'EfficientNetB2',\n            # 'EfficientNetB3',\n            # 'EfficientNetB4',\n            # 'EfficientNetB5',\n            # 'EfficientNetB6',\n            # 'EfficientNetB7',\n#             'ResNet50',\n#             'ResNet101',\n#             'ResNet152',\n        ],\n    BACKBONE_CHANNEL = 3\n    WBC_WEIGHT = 10\n    LOSS_AUX = 0.25\n    \n    EPOCHS = 100\n    SAVE = True\n    SEED = 23\n    FOLD_SPLITS = 5\n    \n    LOGGING = False\n    \ncfg = CFG()","metadata":{"papermill":{"duration":0.026232,"end_time":"2023-06-08T13:14:52.348005","exception":false,"start_time":"2023-06-08T13:14:52.321773","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T15:59:27.765501Z","iopub.execute_input":"2023-07-31T15:59:27.765999Z","iopub.status.idle":"2023-07-31T15:59:27.777773Z","shell.execute_reply.started":"2023-07-31T15:59:27.765962Z","shell.execute_reply":"2023-07-31T15:59:27.776389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport gc\nimport re\nimport json\nimport time\nimport math\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random as rn\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedKFold\n\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\n\n# import icrgw","metadata":{"papermill":{"duration":45.53379,"end_time":"2023-06-08T13:15:37.91985","exception":false,"start_time":"2023-06-08T13:14:52.38606","status":"completed"},"tags":[],"execution":{"iopub.status.idle":"2023-07-31T12:30:36.370645Z","shell.execute_reply.started":"2023-07-31T12:30:05.790187Z","shell.execute_reply":"2023-07-31T12:30:36.369477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tf.__version__)","metadata":{"papermill":{"duration":0.014124,"end_time":"2023-06-08T13:15:37.938775","exception":false,"start_time":"2023-06-08T13:15:37.924651","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T12:30:36.37206Z","iopub.execute_input":"2023-07-31T12:30:36.372646Z","iopub.status.idle":"2023-07-31T12:30:36.377868Z","shell.execute_reply.started":"2023-07-31T12:30:36.37261Z","shell.execute_reply":"2023-07-31T12:30:36.377045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp /kaggle/input/icrgw-ds-utils/icrgw.py ./\nimport icrgw","metadata":{"papermill":{"duration":10.020854,"end_time":"2023-06-08T13:15:47.964217","exception":false,"start_time":"2023-06-08T13:15:37.943363","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T12:30:36.379797Z","iopub.execute_input":"2023-07-31T12:30:36.3802Z","iopub.status.idle":"2023-07-31T12:30:46.223817Z","shell.execute_reply.started":"2023-07-31T12:30:36.380173Z","shell.execute_reply":"2023-07-31T12:30:46.222688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\n\nicrgw.init(cfg)","metadata":{"papermill":{"duration":0.336761,"end_time":"2023-06-08T13:15:48.308812","exception":false,"start_time":"2023-06-08T13:15:47.972051","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T15:59:32.077476Z","iopub.execute_input":"2023-07-31T15:59:32.077956Z","iopub.status.idle":"2023-07-31T15:59:32.566641Z","shell.execute_reply.started":"2023-07-31T15:59:32.077921Z","shell.execute_reply":"2023-07-31T15:59:32.565132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import importlib\n# importlib.reload(icrgw)\nicrgw.cfg = cfg","metadata":{"papermill":{"duration":0.017286,"end_time":"2023-06-08T13:15:48.334896","exception":false,"start_time":"2023-06-08T13:15:48.31761","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T15:59:32.574898Z","iopub.execute_input":"2023-07-31T15:59:32.575264Z","iopub.status.idle":"2023-07-31T15:59:32.597237Z","shell.execute_reply.started":"2023-07-31T15:59:32.575226Z","shell.execute_reply":"2023-07-31T15:59:32.596049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model(backbone_name):\n    xs_input = tf.keras.layers.Input((None, None, cfg.BACKBONE_CHANNEL))\n    encoder_filters=(16, 32, 64, 128, 256)\n    decoder_filters=(256, 128, 64, 32, 16)\n#     decoder_block = icrgw.DecoderUpsamplingX2Block\n    decoder_block = icrgw.DecoderTransposeX2Block\n\n    bs = icrgw.create_backbone(backbone_name, cfg.BACKBONE_CHANNEL)\n\n    xs = xs_input\n    ss = []\n\n    for i in range(len(bs)):\n        xs = bs[i](xs)\n        ss.append(xs)\n\n    xs = ss[-1]\n    skips = ss[:-1][::-1]\n    \n    for i in range(5):\n        if i < len(skips):\n            skip = skips[i]\n        else:\n            skip = None\n        xs = decoder_block(decoder_filters[i], stage=i, use_batchnorm=True)(xs, skip)\n        \n    # model head (define number of output classes)\n    ys = tf.keras.layers.Conv2D(\n        filters=1,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        activation='sigmoid',\n        name='ys',\n    )(xs)\n    ysa = tf.keras.layers.Conv2D(\n        filters=5,\n        kernel_size=(3, 3),\n        padding='same',\n        use_bias=True,\n        kernel_initializer='glorot_uniform',\n        activation='softmax',\n        name='ysa',\n    )(xs)\n#     ysa = tf.keras.layers.Conv2D(\n#         filters=1,\n#         kernel_size=(3, 3),\n#         padding='same',\n#         use_bias=True,\n#         kernel_initializer='glorot_uniform',\n#         activation='sigmoid',\n#         name='ysa',\n#     )(xs)\n\n    # create keras model instance\n    model = tf.keras.models.Model(xs_input, [ys, ysa])\n\n    return model","metadata":{"papermill":{"duration":0.023127,"end_time":"2023-06-08T13:15:48.367207","exception":false,"start_time":"2023-06-08T13:15:48.34408","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T16:04:47.446522Z","iopub.execute_input":"2023-07-31T16:04:47.447731Z","iopub.status.idle":"2023-07-31T16:04:47.461293Z","shell.execute_reply.started":"2023-07-31T16:04:47.447693Z","shell.execute_reply":"2023-07-31T16:04:47.459912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with icrgw.strategy.scope():\n    train_loss = tf.keras.metrics.Mean('train_loss', dtype=tf.float32)\n    train_accuracy = tf.keras.metrics.Mean('train_accuracy', dtype=tf.float32)\n    \n    train_i = tf.keras.metrics.Mean('train_i', dtype=tf.float32)    \n    train_u = tf.keras.metrics.Mean('train_u', dtype=tf.float32) \n    \n    valid_loss = tf.keras.metrics.Mean('valid_loss', dtype=tf.float32)\n    valid_accuracy = tf.keras.metrics.Mean('valid_accuracy', dtype=tf.float32)\n    \n    valid_i = tf.keras.metrics.Mean('valid_i', dtype=tf.float32)    \n    valid_u = tf.keras.metrics.Mean('valid_u', dtype=tf.float32) ","metadata":{"papermill":{"duration":0.240873,"end_time":"2023-06-08T13:15:48.615605","exception":false,"start_time":"2023-06-08T13:15:48.374732","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T16:04:48.272619Z","iopub.execute_input":"2023-07-31T16:04:48.272989Z","iopub.status.idle":"2023-07-31T16:04:48.447757Z","shell.execute_reply.started":"2023-07-31T16:04:48.27296Z","shell.execute_reply":"2023-07-31T16:04:48.44652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset_dis, valid_dataset_dis, train_steps_per_epoch, valid_steps_per_epoch = icrgw.get_ds()\ntrain_iterator = iter(train_dataset_dis)","metadata":{"papermill":{"duration":0.66567,"end_time":"2023-06-08T13:15:49.29014","exception":false,"start_time":"2023-06-08T13:15:48.62447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T16:04:48.518755Z","iopub.execute_input":"2023-07-31T16:04:48.519094Z","iopub.status.idle":"2023-07-31T16:04:49.275757Z","shell.execute_reply.started":"2023-07-31T16:04:48.519066Z","shell.execute_reply":"2023-07-31T16:04:49.27432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def noise(xs, ys):\n    stddev = cfg.AUG_NOISE['stddev']\n    print(f'custom noise {stddev}')\n    ns = tf.random.normal((cfg.BATCH_SIZE, 1, 1, xs.shape[-1]), 0, stddev)\n    ns2 = tf.random.normal(xs.shape, 0, 0.001)\n#     ns = tf.image.resize(ns, (cfg.TRAIN_SIZE, cfg.TRAIN_SIZE))\n    xs = xs + ns + ns2\n    return xs, ys\n\nicrgw.noise = noise","metadata":{"execution":{"iopub.status.busy":"2023-07-31T16:04:49.28119Z","iopub.execute_input":"2023-07-31T16:04:49.281552Z","iopub.status.idle":"2023-07-31T16:04:49.288773Z","shell.execute_reply.started":"2023-07-31T16:04:49.281513Z","shell.execute_reply":"2023-07-31T16:04:49.287708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for bb in cfg.BACKBONES[0]:\n\n    with icrgw.strategy.scope():\n        model = create_model(bb)\n#         optimizer = tf.keras.optimizers.Adamax()\n        optimizer = icrgw.Lion()\n#         model.load_weights('/kaggle/working/EfficientNetV2B0.h5')\n        \n#         optimizer = icrgw.Lion(1e-6)\n#         model.load_weights('/kaggle/input/icrgw-train-single-512-480-b3-efv2l-0728/EfficientNetV2L.h5')\n\n    train_step, valid_step = icrgw.build_3(model, optimizer, icrgw.wbc_dice_loss,\n            train_loss, train_accuracy, train_i, train_u,\n            valid_loss, valid_accuracy, valid_i, valid_u)\n\n    lr_cb = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_score', mode='max', factor=0.2, patience=5, verbose=1, min_lr=1e-8)\n    model.optimizer = optimizer\n    lr_cb.set_model(model)\n\n    target_epochs = cfg.EPOCHS\n\n#     if icrgw.strategy.num_replicas_in_sync == 1:\n#         train_steps_per_epoch = 1\n#     valid_step(valid_dataset_dis)\n\n    for epoch in range(target_epochs):\n        print('\\nEpoch: {}/{}'.format(epoch, target_epochs))\n        start = time.time()\n\n        train_step(train_iterator, train_steps_per_epoch)\n        print(f'train loss: {train_loss.result():.4f}, \\\n            accuracy: {train_accuracy.result():.4f}, \\\n            score: {(2 * train_i.result()/train_u.result()).numpy()}'\n            )\n        train_loss.reset_states()\n        train_accuracy.reset_states()\n        train_i.reset_states()\n        train_u.reset_states()\n        \n        if epoch > -1:\n            valid_loss.reset_states()\n            valid_accuracy.reset_states()\n            valid_i.reset_states()\n            valid_u.reset_states()\n        #     for data in valid_dataset_dis:\n        #         valid_step(data)\n            valid_step(valid_dataset_dis)\n            score = (2 * valid_i.result()/valid_u.result()).numpy()\n            print(f'\\r\\r valid loss: {valid_loss.result():.4f}, \\\n                valid accuracy: {valid_accuracy.result():.4f}, \\\n                score: {score}',\n                )\n\n            lr_cb.on_epoch_end(epoch, {'val_score': score})\n        \n        elapsed = time.time() - start\n        print(f'{elapsed:.2f} S')\n        \n        if optimizer.lr < 1e-7:\n            break\n            \n    if cfg.SAVE: \n        if cfg.FLOAT_16:\n            model.save('m_16.h5')\n            policy = tf.keras.mixed_precision.Policy('float32')\n            tf.keras.mixed_precision.set_global_policy(policy)\n            mt = create_model(bb)\n            mt.load_weights('m_16.h5')\n            mt.save(f'{bb}.h5')\n            if icrgw.tpu:\n                policy = tf.keras.mixed_precision.Policy('mixed_bfloat16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n            else:\n                policy = tf.keras.mixed_precision.Policy('mixed_float16')\n                tf.keras.mixed_precision.set_global_policy(policy)\n        else:\n            model.save(f'{bb}.h5')","metadata":{"papermill":{"duration":1172.154483,"end_time":"2023-06-08T13:35:21.478765","exception":false,"start_time":"2023-06-08T13:15:49.324282","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-07-31T16:04:49.290041Z","iopub.execute_input":"2023-07-31T16:04:49.290324Z","iopub.status.idle":"2023-07-31T16:43:36.682878Z","shell.execute_reply.started":"2023-07-31T16:04:49.290299Z","shell.execute_reply":"2023-07-31T16:43:36.681436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}