{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nBASE_PATH = '/kaggle/input/rsna-breast-cancer-detection/'\nSAVE_PATH = './'\nTrain_png_image_path = '/kaggle/input/rsna-bcd-roi-1024x512-png-v2-dataset/'\nos.system(\"rm -rf \" + SAVE_PATH + \"/*\")\n#os.system(\"mkdir \" + SAVE_PATH + \"train_png_images\")\n\n\n%rm -rf /root/.keras/datasets\n%mkdir /root/.keras/datasets\n%cp /kaggle/input/gcvit-model-h5/model_h5/gcvit_tiny_weights.h5  /root/.keras/datasets/\n%rm -rf ./cuda\n%mkdir ./cuda\n%cp /kaggle/input/libcudnn8-1/libcudnn8_8.1.0.77-1cuda11.2_amd64.deb ./cuda/\n!sudo dpkg --remove libcudnn8-dev\n!sudo dpkg -i ./cuda/libcudnn8_8.1.0.77-1cuda11.2_amd64.deb\n!pip install -q --no-index --no-deps /kaggle/input/gcvit-py-wheel/wheelhouse/*.whl\n%rm -rf ./cuda\n# !dpkg -l libcudnn8*\n# !apt install --allow-change-held-packages libcudnn8=8.1.0.77-1+cuda11.2 -y\n\n# !pip wheel gcvit\n# !pip wheel dicomsdl\n# !pip wheel pylibjpeg\n# !pip wheel pydicom\n# !pip wheel tensorflow==2.11.0\n\n\n\nimport tensorflow as tf2, re, math\nprint(tf2.__version__)\nprint(tf2.config.list_physical_devices('GPU'))\ntf2.__path__","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-01T20:38:54.147497Z","iopub.execute_input":"2023-02-01T20:38:54.148236Z","iopub.status.idle":"2023-02-01T20:41:38.387908Z","shell.execute_reply.started":"2023-02-01T20:38:54.148194Z","shell.execute_reply":"2023-02-01T20:41:38.386671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport cv2\ncv2.setNumThreads(1)\n\nTARGET_HEIGHT = 1344\nTARGET_WIDTH = 768\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH\nCLAHE = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(32, 32))\nIMAGE_FORMAT = 'png'\n\n# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode='same')\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[:int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n    \n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n        \n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[1].imshow(image)\n        debug[1].set_title('X Offset')\n        vv_scale = H / vv.max() * 0.90\n        # Values\n        debug[1].plot(H - vv * vv_scale , c='red', label='vv')\n        # Threshold\n        debug[1].hlines(H - vv_threshold * vv_scale, 0, W -1, colors='orange', label='threshold')\n        # Max Value\n        debug[1].scatter(vv_argmax, H - vv[vv_argmax] * vv_scale, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[1].scatter(offset, H - vv[offset] * vv_scale, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[1].set_ylim(H, 0)\n        debug[1].legend()\n        debug[1].axis('off')\n        \n    return offset\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = int(image.shape[0] * 0.25) + vv[int(image.shape[0] * 0.25):int(image.shape[0] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n    \n    if isinstance(debug, np.ndarray):\n        debug[2].imshow(image)\n        debug[2].set_title('Y Bottom Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[2].plot(vv * vv_scale, np.arange(H), c='red', label='vv')\n        # Threshold\n        debug[2].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[2].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[2].scatter(vv[offset_bottom] * vv_scale, offset_bottom, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[2].legend()\n        debug[2].axis('off')\n            \n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[3].imshow(image)\n        debug[3].set_title('Y Top Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[3].plot(vv * vv_scale, np.arange(H) , c='red', label='vv')\n        # Threshold\n        debug[3].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[3].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[3].scatter(vv[offset_top] * vv_scale, offset_top, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[3].legend()\n        debug[3].axis('off')\n            \n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:,:x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n    \n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n        \n    # Crop Image\n    image = image[offset_bottom:offset_top:,:x_offset]\n        \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:53:30.004791Z","iopub.execute_input":"2023-02-01T20:53:30.005244Z","iopub.status.idle":"2023-02-01T20:53:30.037638Z","shell.execute_reply.started":"2023-02-01T20:53:30.005206Z","shell.execute_reply":"2023-02-01T20:53:30.036261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nimport dicomsdl \nimport cv2\nimport pydicom\n\nfrom joblib import Parallel, delayed\nimport glob\nimport time\nimport numpy as np\nimport os\nfrom matplotlib import pyplot as plt\n\n\ndef process(file_path, size=(TARGET_WIDTH, TARGET_HEIGHT), crop_image=False, apply_clahe=False, apply_eq_hist=False, debug=False, save=True):\n    # Read Dicom File\n    dicom = dicomsdl.open(file_path)\n    image = dicom.pixelData()\n    \n    # Save original image for debug purposes\n    if debug:\n        fig, axes = plt.subplots(1, 5, figsize=(20,10))\n        image0 = np.copy(image)\n        axes[0].imshow(image0)\n        axes[0].set_title('Original Image')\n        axes[0].axis('off')\n    else:\n        axes = False\n    \n    # voi_lut\n    try:\n        image = voi_lut(image, dicom)\n    except:\n        pass\n    \n    # Some images have 0 values as highest intensity and need to be inverted\n    if dicom.getPixelDataInfo()['PhotometricInterpretation'] == 'MONOCHROME1':\n        image = np.max(image) - image\n\n    # Normalize [0,1] range\n    image = (image - image.min()) / (image.max() - image.min())\n\n    # Convert to uint8 image in range [0, 255]\n    image = (image * 255).astype(np.uint8)\n    \n    # Flip T0 Left/Right Orientation\n    h0, w0 = image.shape\n    if image[:,int(-w0 * 0.10):].sum() > image[:,:int(w0 * 0.10)].sum():\n        image = np.flip(image, axis=1)\n    \n    # Crop Image\n    if crop_image:\n        image = crop(image, debug=axes)\n        \n    # Resize\n    if size is not None:\n        # Pad black pixels to make square image\n        \n        h, w = image.shape\n        if (h / w) > TARGET_HEIGHT_WIDTH_RATIO:\n            pad = int(h / TARGET_HEIGHT_WIDTH_RATIO - w)\n            image = np.pad(image, [[0,0], [0, pad]])\n            h, w = image.shape\n        else:\n            pad = int(0.50 * (w * TARGET_HEIGHT_WIDTH_RATIO - h))\n            image = np.pad(image, [[pad, pad], [0,0]])\n            h, w = image.shape\n        # Resize\n        image = cv2.resize(image, size, interpolation=cv2.INTER_AREA)\n        \n    # Apply CLAHE contrast enhancement\n    if apply_clahe:\n        image = CLAHE.apply(image)\n        \n     # Apply Histogram Equalization\n    if apply_eq_hist:\n        image = cv2.equalizeHist(image)\n        \n    # Show Processed Image    \n    if debug:\n        axes[4].imshow(image)\n        axes[4].set_title('Processed Image')\n        axes[4].axis('off')\n        plt.show()\n        \n    # Save Only\n    if save:\n        print('save image')\n        patient = file_path.split('/')[-2]\n        image_name = file_path.split('/')[-1][:-4]\n        #image_id = file_path.split('/')[-1].split('.')[0]\n        if IMAGE_FORMAT == 'png':\n            print('save png')\n            cv2.imwrite(f'{patient}_{image_name}.png', image)\n        else:\n            print('save jpeg')\n            cv2.imwrite(f'{patient}_{image_name}.jpg', image, [cv2.IMWRITE_JPEG_QUALITY, 95])","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:53:31.229223Z","iopub.execute_input":"2023-02-01T20:53:31.229763Z","iopub.status.idle":"2023-02-01T20:53:31.246812Z","shell.execute_reply.started":"2023-02-01T20:53:31.229728Z","shell.execute_reply":"2023-02-01T20:53:31.245433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:53:32.157539Z","iopub.execute_input":"2023-02-01T20:53:32.157902Z","iopub.status.idle":"2023-02-01T20:53:32.164989Z","shell.execute_reply.started":"2023-02-01T20:53:32.157871Z","shell.execute_reply":"2023-02-01T20:53:32.163927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl as dicoml\nimport dicomsdl\nprocess(test_images[0], size=(512, 512), crop_image=False, apply_clahe=False, apply_eq_hist=False, debug=False, save=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-01T20:53:32.783362Z","iopub.execute_input":"2023-02-01T20:53:32.783721Z","iopub.status.idle":"2023-02-01T20:53:33.511578Z","shell.execute_reply.started":"2023-02-01T20:53:32.78369Z","shell.execute_reply":"2023-02-01T20:53:33.510297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save_train_image_dir_pydicom = SAVE_PATH + '/train_png_images/'\n#save_test_image_dir_pydicom = '/kaggle/tmp/'\n\n#train_images = glob.glob(BASE_PATH + \"/train_images/*/*.dcm\")\ntest_images = glob.glob(BASE_PATH + \"/test_images/*/*.dcm\")\n\n# print('start processing training images')\n# start_time = time.time()\n# Parallel(n_jobs=-1)(\n#     delayed(process)(f, size = 512, save_folder = save_train_image_dir_pydicom, dicom_process = True)\n#     for f in train_images\n# )\n# print(time.time() - start_time)\nfrom multiprocessing import cpu_count\nprint('start processing test images')\nstart_time = time.time()\nParallel(n_jobs=cpu_count(),\n        backend='multiprocessing',\n        prefer='threads',)(\n    delayed(process)(f, size = (512, 512), crop_image=False, apply_clahe = False, apply_eq_hist=False)\n    for f in test_images\n)\nprint(time.time() - start_time)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-01T21:00:42.485954Z","iopub.execute_input":"2023-02-01T21:00:42.486764Z","iopub.status.idle":"2023-02-01T21:00:44.856178Z","shell.execute_reply.started":"2023-02-01T21:00:42.486727Z","shell.execute_reply":"2023-02-01T21:00:44.852583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#################\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # to avoid too many logging messages\nimport pandas as pd, numpy as np, random, shutil\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport sklearn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa \nimport tensorflow_probability as tfp\nimport wandb\nimport yaml\n\nfrom IPython import display as ipd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\n\nclass CFG:\n    wandb         = True\n    competition   = 'rsna-bcd' \n    _wandb_kernel = 'tom'\n    debug= False\n    comment       = 'GCViTTiny-224x224-check1'\n    exp_name      = 'submit_test' # name of the experiment, folds will be grouped using 'exp_name'\n    \n    # use verbose=0 for silent, vebose=1 for interactive,\n    #verbose      = 1 if debug else 0\n    verbose = 0\n    display_plot = True\n\n    # device\n    device = \"GPU\" #or \"GPU\"\n\n    model_name = 'GCViTTiny'\n    fine_tuned_model_h5 = '/kaggle/input/zh-gcvit-model-h5/gc_vit_model'\n\n    # seed for data-split, layer init, augs\n    seed = 42\n\n    # number of folds for data-split\n    folds = 5\n    \n    # which folds to train\n    selected_folds = [0, 1, 2]\n\n    # size of the image\n    img_size = [224, 224]\n\n    # batch_size and epochs\n    batch_size = 16\n    \n    epochs = 5\n    \n    # upsample\n    upsample = 10\n\n    # loss and optimizer\n    loss      = 'Focal'  # BCE, Focal\n    optimizer = 'Adam'\n\n    # augmentation\n    augment   = True\n\n    # scale-shift-rotate-shear\n    transform = True\n    fill_mode = 'constant'\n    rot    = 2.0\n    shr    = 2.0\n    hzoom  = 50.0\n    wzoom  = 50.0\n    hshift = 10.0\n    wshift = 10.0\n\n    # flip\n    hflip = True\n    vflip = True\n\n    # clip\n    clip = False\n\n    # lr-scheduler\n    scheduler   = 'exp' # cosine\n\n    # dropout\n    drop_prob   = 0.6\n    drop_cnt    = 10\n    drop_size   = 0.08\n    \n    # cut-mix-up\n    mixup_prob = 0.0\n    mixup_alpha = 0.5\n    \n    cutmix_prob = 0.0\n    cutmix_alpha = 2.5\n\n    # pixel-augment\n    pixel_aug = True\n    sat  = [0.7, 1.3]\n    cont = [0.8, 1.2]\n    bri  = 0.15\n    hue  = 0.05\n\n    # test-time augs\n    tta = 1\n    \n    # target column\n    target_col  = ['cancer']\n###########################################  \ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n    tf.random.set_seed(SEED)\n    print('seeding done!!!')\nseeding(CFG.seed)\n###########################################\nngpu = 1\nstrategy = tf.distribute.get_strategy()\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync\nprint(f'REPLICAS: {REPLICAS}')\n###########################################\n#BASE_PATH = '/home/lei/OWL/data/Kaggle/RSNA'\n\n# test\ntest_df = pd.read_csv(f'{BASE_PATH}/test.csv')\ntest_df['image_path'] =  test_df.patient_id.astype(str)\\\n                        + '_' + test_df.image_id.astype(str)\\\n                        + '.png'\nprint('\\nTest:')\n###########################################\ndef build_decoder(with_labels=True, target_size=CFG.img_size, ext='png'):\n    def decode(path):\n        file_bytes = tf.io.read_file(path)\n        if ext == 'png':\n            img = tf.image.decode_png(file_bytes, channels=3)\n        elif ext in ['jpg', 'jpeg']:\n            img = tf.image.decode_jpeg(file_bytes, channels=3)\n        else:\n            raise ValueError(\"Image extension not supported\")\n\n        img = tf.image.resize(img, target_size, method='bilinear')\n        img = tf.cast(img, tf.float32) / 255.0\n        img = tf.reshape(img, [*target_size, 3])\n\n        return img\n    \n    def decode_with_labels(path, label):\n        return decode(path), tf.cast(label, tf.float32)\n    \n    return decode_with_labels if with_labels else decode\n\n\ndef build_augmenter(with_labels=True, dim=CFG.img_size):\n    def augment(img, dim=dim):\n        img = tf.image.random_flip_left_right(img) if CFG.hflip else img\n        img = tf.image.random_flip_up_down(img) if CFG.vflip else img\n        if CFG.pixel_aug:\n            img = tf.image.random_hue(img, CFG.hue)\n            img = tf.image.random_saturation(img, CFG.sat[0], CFG.sat[1])\n            img = tf.image.random_contrast(img, CFG.cont[0], CFG.cont[1])\n            img = tf.image.random_brightness(img, CFG.bri)\n        img = tf.clip_by_value(img, 0, 1)  if CFG.clip else img         \n        img = tf.reshape(img, [*dim, 3])\n        return img\n    \n    def augment_with_labels(img, label):    \n        return augment(img), label\n    \n    return augment_with_labels if with_labels else augment\n\n\ndef build_dataset(paths, labels=None, batch_size=32, cache=True,\n                  decode_fn=None, augment_fn=None,\n                  augment=True, repeat=True, shuffle=1024, \n                  cache_dir=\"\", drop_remainder=False):\n    if cache_dir != \"\" and cache is True:\n        os.makedirs(cache_dir, exist_ok=True)\n    \n    if decode_fn is None:\n        decode_fn = build_decoder(labels is not None)\n    \n    if augment_fn is None:\n        augment_fn = build_augmenter(labels is not None)\n    \n    AUTO = tf.data.experimental.AUTOTUNE\n    slices = paths if labels is None else (paths, labels)\n    \n    ds = tf.data.Dataset.from_tensor_slices(slices)\n    ds = ds.map(decode_fn, num_parallel_calls=AUTO)\n    ds = ds.cache(cache_dir) if cache else ds\n    ds = ds.repeat() if repeat else ds\n    if shuffle: \n        ds = ds.shuffle(shuffle, seed=CFG.seed)\n        opt = tf.data.Options()\n        opt.experimental_deterministic = False\n        ds = ds.with_options(opt)\n    ds = ds.map(augment_fn, num_parallel_calls=AUTO) if augment else ds\n    ds = ds.batch(batch_size, drop_remainder=drop_remainder)\n    ds = ds.prefetch(AUTO)\n    return ds\n###########################################\n# tensorflow\ndef pFScore(labels, preds, beta=1):\n    eps = 1e-5\n    preds = tf.clip_by_value(preds, 0, 1)\n    y_true_count = tf.reduce_sum(labels)\n    ctp = tf.reduce_sum(preds[labels==1])\n    cfp = tf.reduce_sum(preds[labels==0])\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp + eps)\n    c_recall = ctp / (y_true_count + eps)\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + eps)\n        return result\n    else:\n        return 0.0\npFScore.__name__='pF1'\n\n# numpy\ndef pfbeta(labels, preds, beta=1):\n    eps = 1e-5\n    preds = preds.clip(0, 1)\n    y_true_count = labels.sum()\n    ctp = preds[labels==1].sum()\n    cfp = preds[labels==0].sum()\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp + eps)\n    c_recall = ctp / (y_true_count + eps)\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + eps)\n        return result\n    else:\n        return 0.0\n###########################################\nimport gcvit\ndef build_model(model_name=CFG.model_name,\n                loss_name=CFG.loss,\n                dim=CFG.img_size,\n                compile_model=True,\n                include_top=False):\n    model = getattr(gcvit, model_name)(input_shape=[*dim, 3],\n                                       pretrain=True)  # when pretrain=True, num_classes must be 1000\n    model.reset_classifier(num_classes=1,\n                           head_act='sigmoid')\n    if compile_model:\n        # optimizer\n        opt = tf.keras.optimizers.Adam(learning_rate=0.0001)\n        # loss\n        if loss_name == 'BCE':\n            loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=0.05)\n        elif loss_name == 'Focal':\n            loss = tfa.losses.SigmoidFocalCrossEntropy(alpha=0.80, gamma=2.0)\n        # metric\n        auc = tf.keras.metrics.AUC(name='auc')\n        pf1 = pFScore\n        metrics = [pf1, auc]\n        # compile\n        model.compile(optimizer=opt,\n                      loss=loss,\n                      metrics=metrics)\n    return model\n###########################################\ndef get_lr_callback(batch_size=8, plot=False):\n    lr_start   = 0.000005\n    lr_max     = 0.00000105 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        elif CFG.scheduler=='exp':\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        elif CFG.scheduler=='cosine':\n            decay_total_epochs = CFG.epochs - lr_ramp_ep - lr_sus_ep + 3\n            decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            cosine_decay = 0.4 * (1 + math.cos(phase))\n            lr = (lr_max - lr_min) * cosine_decay + lr_min\n        return lr\n    if plot:\n        plt.figure(figsize=(10,5))\n        plt.plot(np.arange(CFG.epochs), [lrfn(epoch) for epoch in np.arange(CFG.epochs)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('learnig rate')\n        plt.title('Learning Rate Scheduler')\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n###########################################\n###########################################\noof_pred = []; oof_tar = []; oof_val = []; oof_ids = []; oof_folds = []\npreds = np.zeros((test_df.shape[0],1))\n\nfor fold in np.arange(CFG.folds):\n    \n    # ignore not selected folds\n    if fold not in CFG.selected_folds:\n        continue\n    \n    test_paths  = test_df.image_path.values\n    # build model\n    K.clear_session()\n    with strategy.scope():\n        model = build_model(CFG.model_name, dim=CFG.img_size, compile_model=True)\n\n    # for gcvit resize layer\n    tf.config.set_soft_device_placement(True)\n    \n    # build dataset\n    #cache = 'TPU' not in CFG.device\n    cache = False\n    \n    # load best model for inference\n    print('Loading best model...' + '/fold-%i.h5'%fold)\n    model.load_weights(CFG.fine_tuned_model_h5 + '/fold-%i.h5'%fold)  \n    \n    #predict on test data\n    print('Predicting Test...')\n    ds_test = build_dataset(test_paths, labels=None, cache=False, \n                    batch_size=(CFG.batch_size*2 if len(test_df)>4 else 4)*REPLICAS,\n                   repeat=True, shuffle=False, augment=CFG.tta>1)\n    ct_test = len(test_paths); STEPS = 1 if len(test_df)<=4 else (CFG.tta * ct_test/CFG.batch_size/2/REPLICAS)\n    pred = model.predict(ds_test,steps=STEPS,verbose=CFG.verbose)[:CFG.tta*ct_test,] \n    preds[:ct_test, :] += np.mean(pred.reshape((CFG.tta, ct_test,-1)),axis=0) / CFG.folds # not meaningful for DIBUG = True\n    \n# save results\ndf_results = pd.DataFrame({'prediction_id':test_df['prediction_id'].values.tolist(),'cancer':preds[:, 0].tolist()})\ndf_results.groupby('prediction_id').agg({'cancer': 'max'}).reset_index().to_csv('submission.csv', index=False)\nprint('predict_done')","metadata":{"execution":{"iopub.status.busy":"2023-02-01T21:02:43.047199Z","iopub.execute_input":"2023-02-01T21:02:43.047719Z","iopub.status.idle":"2023-02-01T21:03:42.785572Z","shell.execute_reply.started":"2023-02-01T21:02:43.047677Z","shell.execute_reply":"2023-02-01T21:03:42.784354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf ./cuda\n!rm -rf ./*.png\n!ls","metadata":{"execution":{"iopub.status.busy":"2023-02-01T21:04:00.466244Z","iopub.execute_input":"2023-02-01T21:04:00.466649Z","iopub.status.idle":"2023-02-01T21:04:03.83754Z","shell.execute_reply.started":"2023-02-01T21:04:00.466617Z","shell.execute_reply":"2023-02-01T21:04:03.836154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}