{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -q /lib/wheels/tensorflow-2.9.1-cp38-cp38-linux_x86_64.whl\n!pip install -q tensorflow-addons==0.18.0\n!pip install -q tensorflow-probability==0.17.0\n!pip install -q opencv-python-headless\n!pip install -q seaborn","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-12-16T10:40:23.048426Z","iopub.execute_input":"2022-12-16T10:40:23.048715Z","iopub.status.idle":"2022-12-16T10:41:20.542372Z","shell.execute_reply.started":"2022-12-16T10:40:23.048656Z","shell.execute_reply":"2022-12-16T10:41:20.541439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Install Custom Libraries","metadata":{}},{"cell_type":"code","source":"!pip install -q efficientnet >> /dev/null\n!pip install -qU wandb\n!pip install -qU scikit-learn","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-12-16T10:41:20.544099Z","iopub.execute_input":"2022-12-16T10:41:20.544373Z","iopub.status.idle":"2022-12-16T10:41:42.255065Z","shell.execute_reply.started":"2022-12-16T10:41:20.544349Z","shell.execute_reply":"2022-12-16T10:41:42.254131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'  # to avoid too many logging messages\nimport pandas as pd, numpy as np, random, shutil\nimport tensorflow as tf, re, math\nimport tensorflow.keras.backend as K\nimport sklearn\nimport matplotlib.pyplot as plt\nimport tensorflow_addons as tfa\nimport tensorflow_probability as tfp\nimport wandb\nimport yaml\nimport random\n\nfrom IPython import display as ipd\nfrom glob import glob\nfrom tqdm import tqdm\nfrom sklearn.model_selection import KFold, StratifiedKFold, GroupKFold, StratifiedGroupKFold\nfrom sklearn.metrics import roc_auc_score\nfrom tensorflow import keras","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:41:42.256387Z","iopub.execute_input":"2022-12-16T10:41:42.256668Z","iopub.status.idle":"2022-12-16T10:41:56.68885Z","shell.execute_reply.started":"2022-12-16T10:41:42.256642Z","shell.execute_reply":"2022-12-16T10:41:56.687535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Version Check","metadata":{}},{"cell_type":"code","source":"print('np:', np.__version__)\nprint('pd:', pd.__version__)\nprint('sklearn:', sklearn.__version__)\nprint('tf:',tf.__version__)\nprint('tfp:', tfp.__version__)\nprint('tfa:', tfa.__version__)\nprint('w&b:', wandb.__version__)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:41:56.691401Z","iopub.execute_input":"2022-12-16T10:41:56.692265Z","iopub.status.idle":"2022-12-16T10:41:56.699984Z","shell.execute_reply.started":"2022-12-16T10:41:56.692227Z","shell.execute_reply":"2022-12-16T10:41:56.699071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_FOLD = 5\nSEED = 42\nALPHA = 0.5\nMIXUP_PROB = 0.2\nBATCH_SIZE = 4\nUPSAMPLE_RATE = 10\nNUM_EPOCH = 10\nCUTOUT = False\nIMG_SIZE = [1024, 512]\nAUGMENTATION = True\nDEVICE = \"TPU-1VM\"\nIMG_PATH = '/kaggle/input/rsna-bcd-roi-1024x-png-dataset/train_images'","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:41:56.701001Z","iopub.execute_input":"2022-12-16T10:41:56.701255Z","iopub.status.idle":"2022-12-16T10:41:56.707526Z","shell.execute_reply.started":"2022-12-16T10:41:56.70123Z","shell.execute_reply":"2022-12-16T10:41:56.706725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"TPU\" in DEVICE:\n    tpu = 'local' if DEVICE=='TPU-1VM' else None\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu=tpu)\n        strategy = tf.distribute.TPUStrategy(tpu)\n    except:\n        DEVICE = \"GPU\"\n        \nAUTOTUNE = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:41:56.708589Z","iopub.execute_input":"2022-12-16T10:41:56.708852Z","iopub.status.idle":"2022-12-16T10:42:08.756503Z","shell.execute_reply.started":"2022-12-16T10:41:56.708829Z","shell.execute_reply":"2022-12-16T10:42:08.75564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\nprint(train_df.head())\n\ndf = train_df\n\ndf.drop('BIRADS',axis = 1,inplace = True)\ndf.drop('density',axis = 1,inplace = True)\n\nnum_bins = 5\ndf[\"age_bin\"] = pd.cut(df['age'].values.reshape(-1), bins=num_bins, labels=False)\n\nstrat_cols = [\n    'laterality', 'view', 'biopsy','invasive', 'age_bin',\n    'implant','machine_id', 'difficult_negative_case',\n    'cancer',\n]\n\ndf['stratify'] = ''\nfor col in strat_cols:\n    df['stratify'] += df[col].astype(str)\n    \ntrain_ = []\nval_ = []\nskf = StratifiedGroupKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)\nfor fold, (train_idx, val_idx) in enumerate(skf.split(df, df['stratify'], df[\"patient_id\"])):\n    #df.loc[val_idx, 'fold'] = fold\n    train_.append(train_idx)\n    val_.append(val_idx)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:08.75762Z","iopub.execute_input":"2022-12-16T10:42:08.757888Z","iopub.status.idle":"2022-12-16T10:42:15.456229Z","shell.execute_reply.started":"2022-12-16T10:42:08.757862Z","shell.execute_reply":"2022-12-16T10:42:15.455192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_image(image_path):\n    img = tf.io.read_file(image_path)\n    img = tf.image.decode_jpeg(img, channels = 3)\n    img = tf.image.resize(img, IMG_SIZE)\n    img = tf.cast(img, dtype = tf.float32)\n    img = img/255.0\n    return img\n\ndef random_float(shape=[], minval=0.0, maxval=1.0):\n    rnd = tf.random.uniform(\n        shape=shape, minval=minval, maxval=maxval, dtype=tf.float32)\n    return rnd\n\n@tf.function\ndef mixup(images, labels, alpha=ALPHA, prob=MIXUP_PROB):\n    if random_float() > prob:\n        return images, labels\n\n    image_shape = tf.shape(images)\n    label_shape = tf.shape(labels)\n\n    beta = np.random.beta(ALPHA, ALPHA, 1)\n    lam = beta\n\n    images = lam * images + (1.0 - lam) * tf.roll(images, shift=1, axis=0)\n    labels = lam * labels + (1.0 - lam) * tf.roll(labels, shift=1, axis=0)\n\n    images = tf.reshape(images, image_shape)\n    labels = tf.reshape(labels, label_shape)\n    return images, labels\n\ndef augment(image, label):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_flip_up_down(image)\n    degree = random.uniform(-10, 10)\n    #image = tfa.image.rotate(image, degree * math.pi / 180)\n    image = tf.image.random_hue(image, 0.01)\n    image = tf.image.random_saturation(image, 0.70, 1.30)\n    image = tf.image.random_contrast(image, 0.80, 1.20)\n    image = tf.image.random_brightness(image, 0.10)\n    return image, label\n\ndef CountPos(df, train_list, eval_list, num_fold):\n    for i in range(num_fold):\n        train_df = df.iloc[train_list[i]]\n        cancer_list = train_df['cancer'].tolist()\n        t_num_pos = cancer_list.count(1)\n        \n        eval_df = df.iloc[eval_list[i]]\n        cancer_list = eval_df['cancer'].tolist()\n        e_num_pos = cancer_list.count(1)\n        print('In fold %d, there are %d positive in train dataset, %d positive in evaluation dataset'%(i, t_num_pos, e_num_pos))\n\ndef FindIdx(df, train_idx):\n    t_pos_idx = []\n    t_neg_idx = []\n    train_df = df.iloc[train_idx]\n    original_idx = list(train_df.index)\n    cancer_list = train_df['cancer'].tolist()\n    for t in range(len(cancer_list)):\n        if cancer_list[t] == 1:\n            t_pos_idx.append(original_idx[t])\n        else:\n            t_neg_idx.append(original_idx[t])\n    return t_pos_idx, t_neg_idx\n\ndef GeneratePathLabel(train_idx, val_idx, cancer_list, img_ids, patient_ids):\n    t_path = []\n    t_label = []\n    v_path = []\n    v_label = []\n    for i in range(len(train_idx)):\n        img_path = IMG_PATH + '/' + str(patient_ids[train_idx[i]]) + '/' + str(img_ids[train_idx[i]]) + '.png'\n        t_path.append(img_path)\n        t_label.append([cancer_list[train_idx[i]]])\n    for i in range(len(val_idx)):\n        img_path = IMG_PATH + '/' + str(patient_ids[val_idx[i]]) + '/' + str(img_ids[val_idx[i]]) + '.png'\n        v_path.append(img_path)\n        v_label.append([cancer_list[val_idx[i]]])\n    \n    return t_path, v_path, t_label, v_label\n\ndef CheckBalance(batch_size, labels):\n    start = 0\n    end = batch_size\n    num = int(len(labels)/batch_size)\n    num_p_list = []\n    for i in range(num):\n        num_p = labels[start:end].count([1])\n        num_p_list.append(num_p)\n        start = end\n        end += batch_size\n    \n    print(\"Let's find out how many positive in each batch: \", num_p_list[0:50])","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:15.457395Z","iopub.execute_input":"2022-12-16T10:42:15.457689Z","iopub.status.idle":"2022-12-16T10:42:15.481996Z","shell.execute_reply.started":"2022-12-16T10:42:15.457664Z","shell.execute_reply":"2022-12-16T10:42:15.481181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(SEED)\nimage_ids = df['image_id'].tolist()\npatient_ids = df['patient_id'].tolist()\ncancer_list = df['cancer'].tolist()\n\nall_folds_train = []\nall_folds_val = []\nall_train_label = []\nall_val_label = []\nfor k in range(N_FOLD):\n    pos_idx, neg_idx = FindIdx(df = df, train_idx = list(train_[k]))\n    ups_pos_idx = []\n    for i in range(UPSAMPLE_RATE):\n        ups_pos_idx += pos_idx\n    #random.shuffle(neg_idx)\n    ups_train_idx = ups_pos_idx + neg_idx\n    random.shuffle(ups_train_idx)\n    print(\"In fold%d there are %d positive images and %d negative images\"%(k, len(ups_pos_idx), len(neg_idx)))\n    train_path, val_path, train_label, val_label = GeneratePathLabel(train_idx = ups_train_idx, \n                                                                     val_idx = list(val_[k]), \n                                                                     cancer_list = cancer_list, \n                                                                     img_ids = image_ids, \n                                                                     patient_ids = patient_ids)\n    all_folds_train.append(train_path)\n    all_folds_val.append(val_path)\n    all_train_label.append(train_label)\n    all_val_label.append(val_label)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:15.482945Z","iopub.execute_input":"2022-12-16T10:42:15.483211Z","iopub.status.idle":"2022-12-16T10:42:16.889005Z","shell.execute_reply.started":"2022-12-16T10:42:15.483188Z","shell.execute_reply":"2022-12-16T10:42:16.888066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GCS_DS_PATH = 'gs://kds-571ac0e16a7cf8c2edaf15e8516462560975b834b27369573714d3be'\ndef GCS(gcs_path, img_path):\n    gcs_path_ = []\n    for i in range(len(img_path)):\n        split = img_path[i].split('/')\n        path = gcs_path + '/' + 'train_images' + '/' + split[5] + '/' + split[6]\n        gcs_path_.append(path)\n    return gcs_path_","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:16.891718Z","iopub.execute_input":"2022-12-16T10:42:16.892016Z","iopub.status.idle":"2022-12-16T10:42:16.897642Z","shell.execute_reply.started":"2022-12-16T10:42:16.891992Z","shell.execute_reply":"2022-12-16T10:42:16.896868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fold = []\nval_fold = []\nfor fold in range(N_FOLD):\n    train_path = GCS(GCS_DS_PATH, all_folds_train[fold])\n    train_path_ds = tf.data.Dataset.from_tensor_slices(train_path)\n    train_label_ds = tf.data.Dataset.from_tensor_slices(all_train_label[fold])\n    train_image_ds = train_path_ds.map(load_image, num_parallel_calls = AUTOTUNE)\n    train_ds = tf.data.Dataset.zip((train_image_ds, train_label_ds))\n    #train_ds = train_ds.cache()\n    train_ds = train_ds.repeat()\n    #train_ds = train_ds.shuffle(1024, seed=SEED)\n    opt = tf.data.Options()\n    opt.experimental_deterministic = False\n    train_ds = train_ds.with_options(opt)\n    train_ds = train_ds.batch(BATCH_SIZE * REPLICAS).prefetch(buffer_size=AUTOTUNE)\n    if AUGMENTATION:\n        #train_ds = train_ds.map(data_augment, num_parallel_calls = AUTOTUNE)\n        train_ds = train_ds.map(augment, num_parallel_calls = AUTOTUNE)\n        train_ds = train_ds.map(mixup, num_parallel_calls = AUTOTUNE)\n    train_fold.append(train_ds)\n    \n    val_path = GCS(GCS_DS_PATH, all_folds_val[fold])\n    val_path_ds = tf.data.Dataset.from_tensor_slices(val_path)\n    #val_label_ds = tf.data.Dataset.from_tensor_slices(all_val_label[fold])\n    val_ds = val_path_ds.map(load_image, num_parallel_calls = AUTOTUNE)\n    #val_ds = tf.data.Dataset.zip((val_image_ds, val_label_ds))\n    val_ds = val_ds.batch(BATCH_SIZE * REPLICAS).prefetch(buffer_size=AUTOTUNE)\n    val_fold.append(val_ds)","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:16.898519Z","iopub.execute_input":"2022-12-16T10:42:16.898761Z","iopub.status.idle":"2022-12-16T10:42:21.623251Z","shell.execute_reply.started":"2022-12-16T10:42:16.898739Z","shell.execute_reply":"2022-12-16T10:42:21.62225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    encoder = tf.keras.applications.EfficientNetB4(include_top=False, \n                                                   weights=\"imagenet\", \n                                                   input_shape = [IMG_SIZE[0], IMG_SIZE[1], 3])\ndef build_model():\n    x = encoder.output\n    x = keras.layers.GlobalAveragePooling2D()(x)\n    #x = keras.layers.BatchNormalization()(x)\n    x = keras.layers.Dense(32, activation = 'silu')(x)\n    #x = keras.layers.Dropout(0.5)(x)\n    outputs = keras.layers.Dense(1, activation = 'sigmoid')(x)\n    model = tf.keras.models.Model(inputs = encoder.inputs, outputs = outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:21.624374Z","iopub.execute_input":"2022-12-16T10:42:21.624719Z","iopub.status.idle":"2022-12-16T10:42:58.423938Z","shell.execute_reply.started":"2022-12-16T10:42:21.624689Z","shell.execute_reply":"2022-12-16T10:42:58.422689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"REPLICAS = strategy.num_replicas_in_sync\nscheduler = 'exp'\n\ndef pFScore(labels, preds, beta=1):\n    eps = 1e-5\n    preds = tf.clip_by_value(preds, 0, 1)\n    y_true_count = tf.reduce_sum(labels)\n    ctp = tf.reduce_sum(preds[labels==1])\n    cfp = tf.reduce_sum(preds[labels==0])\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp + eps)\n    c_recall = ctp / (y_true_count + eps)\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + eps)\n        return result\n    else:\n        return 0.0\npFScore.__name__='pF1'\n\npf1 = pFScore\n\ndef get_lr_callback(batch_size=BATCH_SIZE, plot=False):\n    lr_start   = 0.000003\n    lr_max     = 0.00000095 * REPLICAS * batch_size\n    lr_min     = 0.000001\n    lr_ramp_ep = 4\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n   \n    def lrfn(epoch):\n        if epoch < lr_ramp_ep:\n            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n            \n        elif epoch < lr_ramp_ep + lr_sus_ep:\n            lr = lr_max\n            \n        elif scheduler=='exp':\n            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n            \n        elif scheduler=='cosine':\n            decay_total_epochs = NUM_EPOCH - lr_ramp_ep - lr_sus_ep + 3\n            decay_epoch_index = epoch - lr_ramp_ep - lr_sus_ep\n            phase = math.pi * decay_epoch_index / decay_total_epochs\n            cosine_decay = 0.4 * (1 + math.cos(phase))\n            lr = (lr_max - lr_min) * cosine_decay + lr_min\n        return lr\n    if plot:\n        plt.figure(figsize=(10,5))\n        plt.plot(np.arange(NUM_EPOCH), [lrfn(epoch) for epoch in np.arange(NUM_EPOCH)], marker='o')\n        plt.xlabel('epoch'); plt.ylabel('learnig rate')\n        plt.title('Learning Rate Scheduler')\n        plt.show()\n\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback\n\nwith strategy.scope():\n    METRICS = [keras.metrics.TruePositives(thresholds = 0.5, name='tp'),\n               keras.metrics.FalsePositives(thresholds = 0.5, name='fp'),\n               keras.metrics.TrueNegatives(thresholds = 0.5, name='tn'),\n               keras.metrics.FalseNegatives(thresholds = 0.5, name='fn'), \n               keras.metrics.BinaryAccuracy(name='accuracy'),\n               keras.metrics.Precision(name='precision'),\n               keras.metrics.Recall(name='recall')]\n    METRICS += [pf1]","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:58.425334Z","iopub.execute_input":"2022-12-16T10:42:58.426011Z","iopub.status.idle":"2022-12-16T10:42:58.58447Z","shell.execute_reply.started":"2022-12-16T10:42:58.42598Z","shell.execute_reply":"2022-12-16T10:42:58.583229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CROSS_VAL = False\nTRAIN = True\n\nif TRAIN:\n    if CROSS_VAL:\n        for fold in range(N_FOLD):\n    \n            with strategy.scope():\n                model = build_model()\n                model.compile(#optimizer = tfa.optimizers.AdamW(learning_rate=3e-4, weight_decay = 1e-4),\n                              optimizer=keras.optimizers.Adam(),\n                              loss=tf.keras.losses.BinaryCrossentropy(from_logits = False),\n                              metrics=METRICS)\n            \n            model.fit(train_fold[fold], \n                      epochs = NUM_EPOCH, \n                      batch_size = BATCH_SIZE,\n                      verbose = 1, \n                      callbacks = [warm_up_lr], \n                      validation_data = val_fold[fold])\n            \n    else:\n        with strategy.scope():\n            model = build_model()\n            model.compile(#optimizer = tfa.optimizers.AdamW(weight_decay = 0.001),\n                          optimizer=keras.optimizers.Adam(),\n                          loss=tfa.losses.SigmoidFocalCrossEntropy(alpha=0.80, gamma=2.0),\n                          #loss=tf.keras.losses.BinaryCrossentropy(from_logits = False),\n                          metrics=METRICS)\n            \n        model.fit(train_fold[0], \n                  epochs = NUM_EPOCH, \n                  #batch_size = BATCH_SIZE,\n                  verbose = 1, \n                  callbacks = [get_lr_callback(BATCH_SIZE)], \n                  steps_per_epoch = len(all_folds_train[0])/BATCH_SIZE//REPLICAS,\n                  #validation_data = val_fold[0],\n                  #class_weight=class_weight\n                 )","metadata":{"execution":{"iopub.status.busy":"2022-12-16T10:42:58.585819Z","iopub.execute_input":"2022-12-16T10:42:58.586139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(val_fold[0], verbose = 1)\npreds = np.reshape(preds, (preds.shape[0], ))\nlabels = np.reshape(all_val_label[0], (preds.shape[0], ))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"STEP = 0.001\nN_ITER = int(1/STEP)\n\ndef pfbeta(labels, predictions, beta = 1):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0\n\ndef FindTh(labels, preds, step, n_iter):\n    th = 0.0\n    pf1 = []\n    thr = []\n    for i in range(n_iter):\n        th += step\n        pred = (preds > th).astype(int)\n        pf1_ = pfbeta(labels = labels, predictions = pred)\n        pf1.append(pf1_)\n        thr.append(th)\n    return np.array(pf1), np.array(thr)\n\npf1, th = FindTh(labels = labels,\n                 preds = preds,\n                 step = STEP,\n                 n_iter = N_ITER)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.max(pf1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_idx = np.argmax(pf1)\nmax_th = th[max_idx]\nmax_th","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}