{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Hello fellow Kagglers,\n\nThis notebook demonstrates the training process on a TPU in Tensorflow.\n\nThanks to the use of a [TPU (Tensor Processing Unit)](https://cloud.google.com/tpu) training takes just 12 minutes.\n\nThe TFREcord dataset contains the CC and MLO view of each laterality in 640x512, created in [this notebook](https://www.kaggle.com/code/markwijkhuizen/rsna-preprocessing-tfrecords-640x512-dataset).\n\n20% of the data is used for validation, which reaches 0.10+ pF1 with the best threshold.\n\n**Things that did not work for me:**\n\n* Increase resolution to 960x768\n* [SigmoidFocalCrossEntropy](https://www.tensorflow.org/addons/api_docs/python/tfa/losses/SigmoidFocalCrossEntropy)\n* Increasing model size to for example EfficientNetV2S\n* Heavier augmentations\n* Longer training\n\n**Things that did work for me:**\n\n* Class weights: give minority class weight of 10\n* Training on TPU instead of GPU: larger batch size (16x2->16x8) giving larger probability of having positive sample in batch\n\nI enjoy this competition and will update this notebook frequently, stay tuned!","metadata":{}},{"cell_type":"code","source":"# The Kaggle Tensorflow version is old and does not contain EfficientNetV2: get it from pip package\n!pip install -qq /kaggle/input/kerasefficientnetv2/keras_efficientnet_v2-1.2.2-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:34.931263Z","iopub.execute_input":"2022-12-19T18:23:34.931691Z","iopub.status.idle":"2022-12-19T18:23:42.769514Z","shell.execute_reply.started":"2022-12-19T18:23:34.93161Z","shell.execute_reply":"2022-12-19T18:23:42.768238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\n\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import train_test_split\n\nimport keras_efficientnet_v2\nimport os\nimport time\nimport pickle\nimport math\nimport random\nimport sys\nimport cv2\nimport gc\n\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Python Version: {sys.version}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:42.771451Z","iopub.execute_input":"2022-12-19T18:23:42.771678Z","iopub.status.idle":"2022-12-19T18:23:48.789085Z","shell.execute_reply.started":"2022-12-19T18:23:42.771649Z","shell.execute_reply":"2022-12-19T18:23:48.78833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mixed Precision Policy","metadata":{}},{"cell_type":"code","source":"# float32 or mixed_float16 (mixed precision: compute float16, variable float32)\n# TPU is fast enough and has enough memory to use float32\npolicy = tf.keras.mixed_precision.Policy('float32')\ntf.keras.mixed_precision.set_global_policy(policy)\n\nprint(f'Compute dtype: {tf.keras.mixed_precision.global_policy().compute_dtype}')\nprint(f'Variable dtype: {tf.keras.mixed_precision.global_policy().variable_dtype}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:48.790042Z","iopub.execute_input":"2022-12-19T18:23:48.79023Z","iopub.status.idle":"2022-12-19T18:23:48.795521Z","shell.execute_reply.started":"2022-12-19T18:23:48.790206Z","shell.execute_reply":"2022-12-19T18:23:48.79459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matplotlib Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:48.798033Z","iopub.execute_input":"2022-12-19T18:23:48.798232Z","iopub.status.idle":"2022-12-19T18:23:48.80735Z","shell.execute_reply.started":"2022-12-19T18:23:48.798208Z","shell.execute_reply":"2022-12-19T18:23:48.806464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    TPU = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', TPU.master())\nexcept ValueError:\n    print('Running on GPU')\n    TPU = None\n\nif TPU:\n    IS_TPU = True\n    tf.config.experimental_connect_to_cluster(TPU)\n    tf.tpu.experimental.initialize_tpu_system(TPU)\n    STRATEGY = tf.distribute.experimental.TPUStrategy(TPU)\nelse:\n    IS_TPU = False\n    STRATEGY = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nN_REPLICAS = STRATEGY.num_replicas_in_sync\nprint(f'N_REPLICAS: {N_REPLICAS}, IS_TPU: {IS_TPU}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:48.80852Z","iopub.execute_input":"2022-12-19T18:23:48.808808Z","iopub.status.idle":"2022-12-19T18:23:55.353846Z","shell.execute_reply.started":"2022-12-19T18:23:48.808775Z","shell.execute_reply":"2022-12-19T18:23:55.353035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# For TPU's the dataset needs to be stored in Google Cloud\n# Retrieve the Google Cloud location of the dataset\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('rsna-preprocessing-tfrecords-640x512-dataset-pub')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:55.354932Z","iopub.execute_input":"2022-12-19T18:23:55.356449Z","iopub.status.idle":"2022-12-19T18:23:59.259218Z","shell.execute_reply.started":"2022-12-19T18:23:55.356412Z","shell.execute_reply":"2022-12-19T18:23:59.258132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 43\nDEBUG = False\n\n# Image dimensions\nIMG_HEIGHT = 640\nIMG_WIDTH = 512\nINPUT_SHAPE = (IMG_HEIGHT, IMG_WIDTH, 1)\nN_SAMPLES_TFRECORDS = 239\n\n# Peak Learning Rate\nLR_MAX = 8e-4\n\nN_WARMUP_EPOCHS = 2\nN_EPOCHS = 8\n\n# Batch size\nBATCH_SIZE = 16 * N_REPLICAS\n\n# Is Interactive Flag and COrresponding Verbosity Method\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\nVERBOSE = 1 if IS_INTERACTIVE else 2\n\n# Tensorflow AUTO flag\nAUTO = tf.data.experimental.AUTOTUNE\n\nprint(f'BATCH_SIZE: {BATCH_SIZE}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.260431Z","iopub.execute_input":"2022-12-19T18:23:59.261201Z","iopub.status.idle":"2022-12-19T18:23:59.268544Z","shell.execute_reply.started":"2022-12-19T18:23:59.261149Z","shell.execute_reply":"2022-12-19T18:23:59.267557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{}},{"cell_type":"code","source":"# Seed all random number generators\ndef seed_everything(seed=SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.269635Z","iopub.execute_input":"2022-12-19T18:23:59.269859Z","iopub.status.idle":"2022-12-19T18:23:59.280712Z","shell.execute_reply.started":"2022-12-19T18:23:59.269837Z","shell.execute_reply":"2022-12-19T18:23:59.279888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Train DataFrame\ntrain = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n\ndisplay(train.head())\ndisplay(train.info())","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.282063Z","iopub.execute_input":"2022-12-19T18:23:59.282592Z","iopub.status.idle":"2022-12-19T18:23:59.424078Z","shell.execute_reply.started":"2022-12-19T18:23:59.282557Z","shell.execute_reply":"2022-12-19T18:23:59.423264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"code","source":"# short Tensorflow randin integer function\ndef tf_rand_int(minval, maxval, dtype=tf.int64):\n    minval = tf.cast(minval, dtype)\n    maxval = tf.cast(maxval, dtype)\n    return tf.random.uniform(shape=(), minval=minval, maxval=maxval, dtype=dtype)\n\n# chance of 1 in k\ndef one_in(k):\n    return 0 == tf_rand_int(0, k)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.426444Z","iopub.execute_input":"2022-12-19T18:23:59.426634Z","iopub.status.idle":"2022-12-19T18:23:59.431633Z","shell.execute_reply.started":"2022-12-19T18:23:59.426611Z","shell.execute_reply":"2022-12-19T18:23:59.430993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# Function to benchmark the dataset\ndef benchmark_dataset(dataset, num_epochs=3, n_steps_per_epoch=10, bs=BATCH_SIZE):\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        for idx, (images, labels) in enumerate(dataset.take(n_steps_per_epoch + 1)):\n            if idx == 0:\n                epoch_start = time.perf_counter()\n            elif idx == 1 and epoch_num == 0:\n                images_cc = images['image_cc']\n                images_mlo = images['image_mlo']\n                print(f'images_cc shape: {images_cc.shape}, images_mlo shape: {images_mlo.shape}, labels shape: {labels.shape}')\n                print(f'images_cc dtype: {images_cc.dtype}, images_mlo dtype: {images_mlo.dtype}, labels dtype: {labels.dtype}')\n            else:\n                pass\n        \n        epoch_t = time.perf_counter() - epoch_start\n        mean_step_t = round(epoch_t / n_steps_per_epoch * 1000, 1)\n        n_imgs_per_s = int(1 / (mean_step_t / 1000) * bs)\n        print(f'epoch {epoch_num} took: {round(epoch_t, 2)} sec, mean step duration: {mean_step_t}ms, images/s: {n_imgs_per_s}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.432611Z","iopub.execute_input":"2022-12-19T18:23:59.434082Z","iopub.status.idle":"2022-12-19T18:23:59.446571Z","shell.execute_reply.started":"2022-12-19T18:23:59.434051Z","shell.execute_reply":"2022-12-19T18:23:59.446037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plots a batch of images\ndef show_batch(dataset, rows=4, cols=4):\n    images, labels = next(iter(dataset))\n    images_cc = images['image_cc'].numpy()\n    images_mlo = images['image_mlo'].numpy()\n    # Plot\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*4, rows*5))\n    for r in range(rows):\n        for c in range(cols // 2):\n            idx = (r//2)*cols+c\n            img_cc = images_cc[idx]\n            img_mlo = images_mlo[idx]\n            lbl =  labels[idx]\n            \n            axes[r, c*2].imshow(img_cc)\n            axes[r, c*2].set_title(f'CC | Label: {lbl}')\n            \n            axes[r, c*2+1].imshow(img_mlo)\n            axes[r, c*2+1].set_title(f'MLO | Label: {lbl}')\n            \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.447865Z","iopub.execute_input":"2022-12-19T18:23:59.44836Z","iopub.status.idle":"2022-12-19T18:23:59.459856Z","shell.execute_reply.started":"2022-12-19T18:23:59.448327Z","shell.execute_reply":"2022-12-19T18:23:59.45865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Decodes the TFRecords\ndef decode_image(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image_cc': tf.io.FixedLenFeature([], tf.string),\n        'image_mlo': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n    })\n    \n    # Images are serialized, read them as uint8 and reshape to original dimensions\n    image_cc = tf.io.parse_tensor(features['image_cc'], out_type=tf.uint8)\n    image_cc = tf.reshape(image_cc, [IMG_HEIGHT, IMG_WIDTH, 1])\n\n    image_mlo = tf.io.parse_tensor(features['image_mlo'], out_type=tf.uint8)\n    image_mlo = tf.reshape(image_mlo, [IMG_HEIGHT, IMG_WIDTH, 1])\n\n    target = features['target']\n    target = tf.cast(target, tf.float32)\n    \n    return { 'image_cc': image_cc, 'image_mlo': image_mlo }, target","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.461116Z","iopub.execute_input":"2022-12-19T18:23:59.461408Z","iopub.status.idle":"2022-12-19T18:23:59.475431Z","shell.execute_reply.started":"2022-12-19T18:23:59.461378Z","shell.execute_reply":"2022-12-19T18:23:59.474834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def crop_resize_noise(X, y):\n    # Random crop image with maximum of 10%\n    ratio = tf.random.uniform([], 0.90, 1.00)\n    img_height_crop = tf.cast(ratio * IMG_HEIGHT, tf.int32)\n    img_width_crop = tf.cast(ratio * IMG_WIDTH, tf.int32)\n    # Random offset for crop\n    img_height_offset = IMG_HEIGHT - img_height_crop\n    img_width_offset = IMG_WIDTH - img_width_crop\n    # Crop And Resize\n    image_cc = tf.slice(X['image_cc'], [img_height_offset, img_width_offset, 0], [img_height_crop, img_width_crop, 1])\n    image_cc = tf.image.resize(image_cc, [IMG_HEIGHT, IMG_WIDTH], method=tf.image.ResizeMethod.BILINEAR)    \n    \n    image_mlo = tf.slice(X['image_mlo'], [img_height_offset, img_width_offset, 0], [img_height_crop, img_width_crop, 1])\n    image_mlo = tf.image.resize(image_mlo, [IMG_HEIGHT, IMG_WIDTH], method=tf.image.ResizeMethod.BILINEAR)\n    # Add Noise\n    image_cc = tf.random.uniform(X['image_cc'].shape, 0.90, 1.10) * image_cc\n    image_mlo = tf.random.uniform(X['image_mlo'].shape, 0.90, 1.10) * image_mlo\n    \n    return { 'image_cc': image_cc, 'image_mlo': image_mlo }, y","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.476434Z","iopub.execute_input":"2022-12-19T18:23:59.477178Z","iopub.status.idle":"2022-12-19T18:23:59.488456Z","shell.execute_reply.started":"2022-12-19T18:23:59.477144Z","shell.execute_reply":"2022-12-19T18:23:59.487496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Undersample majority class (0/negative) by randomly dropping them\ndef undersample_majority(X, y):\n    # Filter 2/3 of negative samples to upsample positive samples by a factor 3\n    return y == 1 or tf.random.uniform([]) > 0.66","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.489834Z","iopub.execute_input":"2022-12-19T18:23:59.490581Z","iopub.status.idle":"2022-12-19T18:23:59.501858Z","shell.execute_reply.started":"2022-12-19T18:23:59.490551Z","shell.execute_reply":"2022-12-19T18:23:59.501355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFRecord file paths\nTFRECORDS_FILE_PATHS = sorted(tf.io.gfile.glob(f'{GCS_DS_PATH}/*.tfrecords'))\nprint(f'Found {len(TFRECORDS_FILE_PATHS)} TFRecords')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.502669Z","iopub.execute_input":"2022-12-19T18:23:59.503065Z","iopub.status.idle":"2022-12-19T18:23:59.629015Z","shell.execute_reply.started":"2022-12-19T18:23:59.503041Z","shell.execute_reply":"2022-12-19T18:23:59.628098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train Test Split\nTFRECORDS_TRAIN, TFRECORDS_VAL = train_test_split(TFRECORDS_FILE_PATHS, train_size=0.80, random_state=SEED, shuffle=True)\nprint(f'# TFRECORDS_TRAIN: {len(TFRECORDS_TRAIN)}, # TFRECORDS_VAL: {len(TFRECORDS_VAL)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.630471Z","iopub.execute_input":"2022-12-19T18:23:59.630728Z","iopub.status.idle":"2022-12-19T18:23:59.63662Z","shell.execute_reply.started":"2022-12-19T18:23:59.630697Z","shell.execute_reply":"2022-12-19T18:23:59.635911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(tfrecords, bs=BATCH_SIZE, val=False, debug=True):\n    ignore_order = tf.data.Options()\n    ignore_order.experimental_deterministic = False\n    \n    # Initialize dataset with TFRecords\n    dataset = tf.data.TFRecordDataset(tfrecords, num_parallel_reads=AUTO, compression_type='GZIP')\n    \n    # Decode mapping\n    dataset = dataset.map(decode_image, num_parallel_calls=AUTO)\n\n    if not val:\n        dataset = dataset.filter(undersample_majority)\n        dataset = dataset.map(crop_resize_noise, num_parallel_calls=AUTO)\n        dataset = dataset.with_options(ignore_order)\n        if not debug:\n            dataset = dataset.shuffle(1024)\n        dataset = dataset.repeat()        \n\n    dataset = dataset.batch(bs, drop_remainder=not val)\n    dataset = dataset.prefetch(AUTO)\n    \n    return dataset","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.637681Z","iopub.execute_input":"2022-12-19T18:23:59.638436Z","iopub.status.idle":"2022-12-19T18:23:59.649218Z","shell.execute_reply.started":"2022-12-19T18:23:59.638404Z","shell.execute_reply":"2022-12-19T18:23:59.648656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = get_dataset(TFRECORDS_TRAIN, val=False, debug=False)\nval_dataset = get_dataset(TFRECORDS_VAL, val=True, debug=False)\n\nTRAIN_STEPS_PER_EPOCH = len(TFRECORDS_TRAIN) * N_SAMPLES_TFRECORDS // BATCH_SIZE\nVAL_STEPS_PER_EPOCH = len(TFRECORDS_VAL) * N_SAMPLES_TFRECORDS // BATCH_SIZE\nprint(f'TRAIN_STEPS_PER_EPOCH: {TRAIN_STEPS_PER_EPOCH}, VAL_STEPS_PER_EPOCH: {VAL_STEPS_PER_EPOCH}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:23:59.650926Z","iopub.execute_input":"2022-12-19T18:23:59.651222Z","iopub.status.idle":"2022-12-19T18:24:00.008342Z","shell.execute_reply.started":"2022-12-19T18:23:59.651189Z","shell.execute_reply":"2022-12-19T18:24:00.007558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity check, image and label statistics\nX_batch, y_batch = next(iter(get_dataset(TFRECORDS_TRAIN, val=False)))\nimages_cc = X_batch['image_cc'].numpy()\nimages_mlo = X_batch['image_mlo'].numpy()\nprint(f'images_cc shape: {images_cc.shape}, images_mlo shape: {images_mlo.shape}, y_batch shape: {y_batch.shape}')\nprint(f'images_cc dtype: {images_cc.dtype}, images_mlo dtype: {images_mlo.dtype}, y_batch dtype: {y_batch.dtype}')\nprint(f'images_cc min: {images_cc.min():.2f}, max: {images_cc.max():.2f}, images_mlo min: {images_mlo.min():.2f}, max: {images_mlo.max():.2f}')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:24:00.009312Z","iopub.execute_input":"2022-12-19T18:24:00.009505Z","iopub.status.idle":"2022-12-19T18:24:06.94848Z","shell.execute_reply.started":"2022-12-19T18:24:00.009482Z","shell.execute_reply":"2022-12-19T18:24:06.947293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Benchmark Dataset\nbenchmark_dataset(get_dataset(TFRECORDS_TRAIN, val=False))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:24:06.949735Z","iopub.execute_input":"2022-12-19T18:24:06.949992Z","iopub.status.idle":"2022-12-19T18:24:31.591303Z","shell.execute_reply.started":"2022-12-19T18:24:06.949937Z","shell.execute_reply":"2022-12-19T18:24:31.590362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show what we will be training on\nshow_batch(get_dataset(TFRECORDS_TRAIN, bs=8, val=False))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:24:31.592714Z","iopub.execute_input":"2022-12-19T18:24:31.592939Z","iopub.status.idle":"2022-12-19T18:24:36.986476Z","shell.execute_reply.started":"2022-12-19T18:24:31.592911Z","shell.execute_reply":"2022-12-19T18:24:36.984094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class Imbalance","metadata":{}},{"cell_type":"code","source":"# Label Distribution Train With Undersampled Majority Class\nN = 128\ntrain_labels = []\nfor _, labels in tqdm(get_dataset(TFRECORDS_TRAIN, val=False).take(N), total=N):\n    train_labels += labels.numpy().tolist()\n    \ndisplay(pd.concat((\n        pd.Series(train_labels).value_counts(normalize=True).to_frame('Train Label Ratio'),\n        pd.Series(train_labels).value_counts().to_frame('Train Label Count'),\n    ), axis=1)\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:03.923089Z","iopub.execute_input":"2022-12-19T18:39:03.923341Z","iopub.status.idle":"2022-12-19T18:39:25.675915Z","shell.execute_reply.started":"2022-12-19T18:39:03.923315Z","shell.execute_reply":"2022-12-19T18:39:25.675357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Label Distribution Validation (Unchanged)\nval_labels = []\nfor _, labels in tqdm(get_dataset(TFRECORDS_VAL, val=True), total=VAL_STEPS_PER_EPOCH):\n    val_labels += labels.numpy().tolist()\n    \ndisplay(pd.concat((\n        pd.Series(val_labels).value_counts(normalize=True).to_frame('Val Label Ratio'),\n        pd.Series(val_labels).value_counts().to_frame('Val Label Count'),\n    ), axis=1)\n)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:25.677165Z","iopub.execute_input":"2022-12-19T18:39:25.677453Z","iopub.status.idle":"2022-12-19T18:39:27.52626Z","shell.execute_reply.started":"2022-12-19T18:39:25.677427Z","shell.execute_reply":"2022-12-19T18:39:27.525414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# pF1 Metric\n\nsource: [RSNA-BCD: EfficientNet [TF][TPU-1VM][Train]](https://www.kaggle.com/code/awsaf49/rsna-bcd-efficientnet-tf-tpu-1vm-train#Metric)","metadata":{}},{"cell_type":"code","source":"# tensorflow\ndef pfbeta_tf(labels, preds, beta=1):\n    eps = 1e-5\n    preds = tf.clip_by_value(preds, 0, 1)\n    y_true_count = tf.reduce_sum(labels)\n    ctp = tf.reduce_sum(preds[labels==1])\n    cfp = tf.reduce_sum(preds[labels==0])\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp + eps)\n    c_recall = ctp / (y_true_count + eps)\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + eps)\n        return result\n    else:\n        return tf.constant(0, dtype=tf.float32)\npfbeta_tf.__name__='pF1'\n\n\n# finds best pf1 using thresholds\ndef pfbeta_thr(labels, preds):\n    thrs = tf.range(0, 1, 0.05)\n    best_score = tf.constant(0, dtype=tf.float32)\n    for thr in thrs:\n        score = pfbeta_tf(labels, tf.cast(preds>thr, tf.float32))\n        best_score = tf.cond(score > best_score, lambda: score, lambda: best_score)\n    return best_score\n\npfbeta_thr.__name__='pF1_thr'","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:27.527293Z","iopub.execute_input":"2022-12-19T18:39:27.52758Z","iopub.status.idle":"2022-12-19T18:39:27.536249Z","shell.execute_reply.started":"2022-12-19T18:39:27.527554Z","shell.execute_reply":"2022-12-19T18:39:27.535627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def get_model():\n    # Verify Mixed Policy Settings\n    print(f'Compute dtype: {tf.keras.mixed_precision.global_policy().compute_dtype}')\n    print(f'Variable dtype: {tf.keras.mixed_precision.global_policy().variable_dtype}')\n    \n    with STRATEGY.scope():\n        # Set seed for deterministic weights initialization\n        seed_everything()\n    \n        def normalize(image):\n            image = tf.repeat(image, repeats=3, axis=3)\n            image = tf.cast(image, tf.float32)\n            image = tf.keras.applications.imagenet_utils.preprocess_input(image, mode='torch')\n            \n            return image\n        \n        # Inputs, note the names are equal to the dictionary keys in the dataset\n        image_cc = tf.keras.layers.Input(INPUT_SHAPE, name='image_cc', dtype=tf.uint8)\n        image_mlo = tf.keras.layers.Input(INPUT_SHAPE, name='image_mlo', dtype=tf.uint8)\n        \n        # EfficientNetV2 Backbone # \n        cnn_cc = keras_efficientnet_v2.EfficientNetV2B0(input_shape=[IMG_HEIGHT, IMG_WIDTH, 3], pretrained=\"imagenet21k-ft1k\", num_classes=0)\n        cnn_cc._name = 'efficientnetv2b0_cc'\n        cnn_mlo = keras_efficientnet_v2.EfficientNetV2B0(input_shape=[IMG_HEIGHT, IMG_WIDTH, 3],pretrained=\"imagenet21k-ft1k\", num_classes=0)\n        cnn_mlo._name = 'efficientnetv2b0_mlo'\n        \n        # Normalize Input\n        image_cc_norm = normalize(image_cc)\n        image_mlo_norm = normalize(image_mlo)\n\n        # CNN\n        embedding_cc = cnn_cc(image_cc_norm)\n        embedding_mlo = cnn_mlo(image_mlo_norm)\n        \n        # GAP to Get Cc/MLO Embeddings\n        embedding_cc = tf.keras.layers.GlobalAveragePooling2D()(embedding_cc)\n        embedding_mlo = tf.keras.layers.GlobalAveragePooling2D()(embedding_mlo)\n\n        # Head\n        x = tf.keras.layers.Concatenate()([embedding_cc, embedding_mlo])\n        x = tf.keras.layers.Dropout(0.50)(x)\n        x = tf.keras.layers.Dense(128)(x)\n        x = tf.keras.layers.Dropout(0.30)(x)\n        outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n\n        # We will use the famous Adam optimizer for fast learning\n        optimizer = tf.optimizers.Adam(learning_rate=LR_MAX, epsilon=1e-7, clipnorm=5.0)\n\n        # Loss\n        loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n        \n        # Metrics\n        metrics = [\n            pfbeta_tf,\n            pfbeta_thr,\n            tfa.metrics.F1Score(num_classes=1, threshold=0.50),\n            tf.keras.metrics.Precision(),\n            tf.keras.metrics.Recall(),\n            tf.keras.metrics.AUC(),\n            tf.keras.metrics.BinaryAccuracy(),\n        ]\n\n        model = tf.keras.models.Model(inputs=[image_cc, image_mlo], outputs=outputs)\n        \n        model.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\n        return model","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:27.538621Z","iopub.execute_input":"2022-12-19T18:39:27.538939Z","iopub.status.idle":"2022-12-19T18:39:27.553239Z","shell.execute_reply.started":"2022-12-19T18:39:27.538905Z","shell.execute_reply":"2022-12-19T18:39:27.55255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained File Path: '/kaggle/input/sartorius-training-dataset/model.h5'\ntf.keras.backend.clear_session()\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\nmodel = get_model()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:27.554244Z","iopub.execute_input":"2022-12-19T18:39:27.554901Z","iopub.status.idle":"2022-12-19T18:39:48.115677Z","shell.execute_reply.started":"2022-12-19T18:39:27.554874Z","shell.execute_reply":"2022-12-19T18:39:48.115175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot model summary\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:48.116717Z","iopub.execute_input":"2022-12-19T18:39:48.117344Z","iopub.status.idle":"2022-12-19T18:39:48.158643Z","shell.execute_reply.started":"2022-12-19T18:39:48.117311Z","shell.execute_reply":"2022-12-19T18:39:48.157916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Model architecture\ntf.keras.utils.plot_model(model, show_shapes=True, show_dtype=True, show_layer_names=True, expand_nested=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:48.159524Z","iopub.execute_input":"2022-12-19T18:39:48.15971Z","iopub.status.idle":"2022-12-19T18:39:48.48446Z","shell.execute_reply.started":"2022-12-19T18:39:48.159686Z","shell.execute_reply":"2022-12-19T18:39:48.483362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weight Initilization","metadata":{}},{"cell_type":"code","source":"# Training Output Baseline\ntrain_preds = model.predict(\n        get_dataset(TFRECORDS_TRAIN, val=True),\n        verbose=VERBOSE,\n        steps=128,\n    ).squeeze()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:39:48.486045Z","iopub.execute_input":"2022-12-19T18:39:48.486267Z","iopub.status.idle":"2022-12-19T18:40:14.729664Z","shell.execute_reply.started":"2022-12-19T18:39:48.48624Z","shell.execute_reply":"2022-12-19T18:40:14.728374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialized model train predictions: should not be saturated (all 0/1)\ndisplay(pd.Series(train_preds).describe().to_frame('Value'))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:14.730899Z","iopub.execute_input":"2022-12-19T18:40:14.731154Z","iopub.status.idle":"2022-12-19T18:40:14.745638Z","shell.execute_reply.started":"2022-12-19T18:40:14.731128Z","shell.execute_reply":"2022-12-19T18:40:14.744785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title(f'Train Predictions Initialized Model')\npd.Series(train_preds).plot(kind='hist')\nplt.xticks(np.arange(0, 1.1, 0.1))\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:14.74781Z","iopub.execute_input":"2022-12-19T18:40:14.74801Z","iopub.status.idle":"2022-12-19T18:40:15.014663Z","shell.execute_reply.started":"2022-12-19T18:40:14.747988Z","shell.execute_reply":"2022-12-19T18:40:15.014035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"# Learning rate scheduler with exponential warmup and cosine decay\ndef lrfn(current_step, num_warmup_steps, lr_max, num_cycles=0.50, num_training_steps=N_EPOCHS):\n    \n    if current_step < num_warmup_steps:\n        return lr_max * 0.50 ** (num_warmup_steps - current_step)\n    else:\n        progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr_max","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:15.015841Z","iopub.execute_input":"2022-12-19T18:40:15.016129Z","iopub.status.idle":"2022-12-19T18:40:15.022276Z","shell.execute_reply.started":"2022-12-19T18:40:15.01609Z","shell.execute_reply":"2022-12-19T18:40:15.021297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the learning rate scheduler\ndef plot_lr_schedule(lr_schedule, epochs):\n    fig = plt.figure(figsize=(20, 10))\n    plt.plot([None] + lr_schedule + [None])\n    # X Labels\n    x = np.arange(1, epochs + 1)\n    x_axis_labels = [i if epochs <= 40 or i % 5 == 0 or i == 1 else None for i in range(1, epochs + 1)]\n    plt.xlim([1, epochs])\n    plt.xticks(x, x_axis_labels) # set tick step to 1 and let x axis start at 1\n    \n    # Increase y-limit for better readability\n    plt.ylim([0, max(lr_schedule) * 1.1])\n    \n    # Title\n    schedule_info = f'start: {lr_schedule[0]:.1E}, max: {max(lr_schedule):.1E}, final: {lr_schedule[-1]:.1E}'\n    plt.title(f'Step Learning Rate Schedule, {schedule_info}', size=18, pad=12)\n    \n    # Plot Learning Rates\n    for x, val in enumerate(lr_schedule):\n        if epochs <= 40 or x % 5 == 0 or x is epochs - 1:\n            if x < len(lr_schedule) - 1:\n                if lr_schedule[x - 1] < val:\n                    ha = 'right'\n                else:\n                    ha = 'left'\n            elif x == 0:\n                ha = 'right'\n            else:\n                ha = 'left'\n            plt.plot(x + 1, val, 'o', color='black');\n            offset_y = (max(lr_schedule) - min(lr_schedule)) * 0.02\n            plt.annotate(f'{val:.1E}', xy=(x + 1, val + offset_y), size=12, ha=ha)\n    \n    plt.xlabel('Epoch', size=16, labelpad=5)\n    plt.ylabel('Learning Rate', size=16, labelpad=5)\n    plt.grid()\n    plt.show()\n\n# Learning rate for encoder\nLR_SCHEDULE = [lrfn(step, num_warmup_steps=N_WARMUP_EPOCHS, lr_max=LR_MAX, num_cycles=0.50) for step in range(N_EPOCHS)]\nplot_lr_schedule(LR_SCHEDULE, epochs=N_EPOCHS)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:15.024768Z","iopub.execute_input":"2022-12-19T18:40:15.025146Z","iopub.status.idle":"2022-12-19T18:40:15.380339Z","shell.execute_reply.started":"2022-12-19T18:40:15.02512Z","shell.execute_reply":"2022-12-19T18:40:15.379419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Learning Rate Callback\nlr_callback = tf.keras.callbacks.LearningRateScheduler(lambda step: LR_SCHEDULE[step], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:15.3814Z","iopub.execute_input":"2022-12-19T18:40:15.381591Z","iopub.status.idle":"2022-12-19T18:40:15.386455Z","shell.execute_reply.started":"2022-12-19T18:40:15.381568Z","shell.execute_reply":"2022-12-19T18:40:15.385586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n        train_dataset,\n        steps_per_epoch = TRAIN_STEPS_PER_EPOCH,\n        validation_data = val_dataset,\n        epochs = N_EPOCHS,\n        verbose = VERBOSE,\n        callbacks = [\n            lr_callback,\n        ],\n        class_weight = {\n            0:  1.0,\n            1: 10.0,\n        },\n    )","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:40:15.387786Z","iopub.execute_input":"2022-12-19T18:40:15.388085Z","iopub.status.idle":"2022-12-19T18:49:03.547083Z","shell.execute_reply.started":"2022-12-19T18:40:15.38805Z","shell.execute_reply":"2022-12-19T18:49:03.545946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save model weights for inference\nmodel.save_weights('model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:49:03.54868Z","iopub.execute_input":"2022-12-19T18:49:03.548995Z","iopub.status.idle":"2022-12-19T18:49:07.097451Z","shell.execute_reply.started":"2022-12-19T18:49:03.548934Z","shell.execute_reply":"2022-12-19T18:49:07.096465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# F1 By Threshold","metadata":{}},{"cell_type":"code","source":"# Get true labels and predictions for validation set\ny_true_val = []\ny_pred_val = []\nfor X_batch, y_batch in tqdm(get_dataset(TFRECORDS_VAL, val=True), total=VAL_STEPS_PER_EPOCH):\n    y_true_val += y_batch.numpy().tolist()\n    y_pred_val += model.predict_on_batch(X_batch).squeeze().tolist()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:49:07.099395Z","iopub.execute_input":"2022-12-19T18:49:07.09968Z","iopub.status.idle":"2022-12-19T18:49:53.462205Z","shell.execute_reply.started":"2022-12-19T18:49:07.099644Z","shell.execute_reply":"2022-12-19T18:49:53.461111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source: https://www.kaggle.com/code/sohier/probabilistic-f-score\n# Competition Leaderboard Metric\ndef pfbeta(labels, predictions, beta=1):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:49:53.463629Z","iopub.execute_input":"2022-12-19T18:49:53.463925Z","iopub.status.idle":"2022-12-19T18:49:53.471985Z","shell.execute_reply.started":"2022-12-19T18:49:53.463886Z","shell.execute_reply":"2022-12-19T18:49:53.470918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show\npf1_by_threshold = []\nfor t in tqdm(np.arange(0, 1.01, 0.01)):\n    pf1_by_threshold.append(\n        pfbeta(y_true_val, y_pred_val > t)\n    )\n    \nplt.figure(figsize=(15,8))\nplt.title('F1 By Threshold', size=24)\nplt.plot(pf1_by_threshold, label='F1 Score')\n\narg_max = np.argmax(pf1_by_threshold)\nval_max = np.max(pf1_by_threshold)\nplt.scatter(arg_max, val_max, color='red', label=f'Best Threshold {t:.2f}, pF1 Score: {val_max:.2f}')\n\nplt.xticks(np.arange(0, 110, 10), [f'{t:.2f}' for t in np.arange(0, 1.1, 0.1)])\nplt.yticks(np.arange(0, 1.1, 0.1))\nplt.xlim(0, 100)\nplt.ylim(0, 1)\nplt.xlabel('Threshold')\nplt.ylabel('pF1 Score')\nplt.legend(fontsize=12)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:50:22.433892Z","iopub.execute_input":"2022-12-19T18:50:22.434157Z","iopub.status.idle":"2022-12-19T18:50:25.232265Z","shell.execute_reply.started":"2022-12-19T18:50:22.434132Z","shell.execute_reply":"2022-12-19T18:50:25.231274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training History","metadata":{}},{"cell_type":"code","source":"def plot_history_metric(metric, f_best=np.argmax, ylim=None, yscale=None, yticks=None):\n    plt.figure(figsize=(20, 10))\n    \n    values = history.history[metric]\n    N_EPOCHS = len(values)\n    val = 'val' in ''.join(history.history.keys())\n    # Epoch Ticks\n    if N_EPOCHS <= 20:\n        x = np.arange(1, N_EPOCHS + 1)\n    else:\n        x = [1, 5] + [10 + 5 * idx for idx in range((N_EPOCHS - 10) // 5 + 1)]\n\n    x_ticks = np.arange(1, N_EPOCHS+1)\n\n    # Validation\n    if val:\n        val_values = history.history[f'val_{metric}']\n        val_argmin = f_best(val_values)\n        plt.plot(x_ticks, val_values, label=f'val')\n\n    # summarize history for accuracy\n    plt.plot(x_ticks, values, label=f'train')\n    argmin = f_best(values)\n    plt.scatter(argmin + 1, values[argmin], color='red', s=75, marker='o', label=f'train_best')\n    if val:\n        plt.scatter(val_argmin + 1, val_values[val_argmin], color='purple', s=75, marker='o', label=f'val_best')\n\n    plt.title(f'Model {metric}', fontsize=24, pad=10)\n    plt.ylabel(metric, fontsize=20, labelpad=10)\n\n    if ylim:\n        plt.ylim(ylim)\n\n    if yscale is not None:\n        plt.yscale(yscale)\n        \n    if yticks is not None:\n        plt.yticks(yticks, fontsize=16)\n\n    plt.xlabel('epoch', fontsize=20, labelpad=10)        \n    plt.tick_params(axis='x', labelsize=8)\n    plt.xticks(x, fontsize=16) # set tick step to 1 and let x axis start at 1\n    plt.yticks(fontsize=16)\n    \n    plt.legend(prop={'size': 10})\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:49:56.150912Z","iopub.execute_input":"2022-12-19T18:49:56.151199Z","iopub.status.idle":"2022-12-19T18:49:56.162889Z","shell.execute_reply.started":"2022-12-19T18:49:56.151165Z","shell.execute_reply":"2022-12-19T18:49:56.161939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('loss', f_best=np.argmin)","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:49:56.163943Z","iopub.execute_input":"2022-12-19T18:49:56.164138Z","iopub.status.idle":"2022-12-19T18:49:56.518227Z","shell.execute_reply.started":"2022-12-19T18:49:56.164115Z","shell.execute_reply":"2022-12-19T18:49:56.517165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('pF1', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:50:54.44099Z","iopub.execute_input":"2022-12-19T18:50:54.441532Z","iopub.status.idle":"2022-12-19T18:50:54.757997Z","shell.execute_reply.started":"2022-12-19T18:50:54.441498Z","shell.execute_reply":"2022-12-19T18:50:54.757018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('pF1_thr', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:50:58.228015Z","iopub.execute_input":"2022-12-19T18:50:58.22829Z","iopub.status.idle":"2022-12-19T18:50:58.54577Z","shell.execute_reply.started":"2022-12-19T18:50:58.228261Z","shell.execute_reply":"2022-12-19T18:50:58.544779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('f1_score', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:51:01.402327Z","iopub.execute_input":"2022-12-19T18:51:01.402764Z","iopub.status.idle":"2022-12-19T18:51:01.74247Z","shell.execute_reply.started":"2022-12-19T18:51:01.402735Z","shell.execute_reply":"2022-12-19T18:51:01.74132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('precision', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:51:05.489794Z","iopub.execute_input":"2022-12-19T18:51:05.490064Z","iopub.status.idle":"2022-12-19T18:51:05.827168Z","shell.execute_reply.started":"2022-12-19T18:51:05.490037Z","shell.execute_reply":"2022-12-19T18:51:05.825858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('recall', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:51:07.946178Z","iopub.execute_input":"2022-12-19T18:51:07.946441Z","iopub.status.idle":"2022-12-19T18:51:08.283944Z","shell.execute_reply.started":"2022-12-19T18:51:07.946412Z","shell.execute_reply":"2022-12-19T18:51:08.283019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('auc', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:51:12.191124Z","iopub.execute_input":"2022-12-19T18:51:12.191396Z","iopub.status.idle":"2022-12-19T18:51:12.506707Z","shell.execute_reply.started":"2022-12-19T18:51:12.191371Z","shell.execute_reply":"2022-12-19T18:51:12.505983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_history_metric('binary_accuracy', ylim=[0,1], yticks=np.arange(0.0, 1.1, 0.1))","metadata":{"execution":{"iopub.status.busy":"2022-12-19T18:51:15.172538Z","iopub.execute_input":"2022-12-19T18:51:15.172811Z","iopub.status.idle":"2022-12-19T18:51:15.526492Z","shell.execute_reply.started":"2022-12-19T18:51:15.172784Z","shell.execute_reply":"2022-12-19T18:51:15.525485Z"},"trusted":true},"execution_count":null,"outputs":[]}]}