{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport PIL\nimport cv2\nimport sklearn\nfrom sklearn.model_selection import train_test_split\nimport tifffile\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nimport os\nimport tqdm\nimport tensorflow as tf\nimport re\nfrom itertools import zip_longest\nimport pickle\nimport random\nimport io\nimport matplotlib\n\n\ndirname = \"/kaggle/input/prostate-cancer-grade-assessment\"\nyolo_max_boxes = 20\nyolo_score_threshold = 0.05\n\ngenerate_yolo_ds = False\nuse_precomputed_yolo_ds = True\ntrain_yolo = False\nuse_precomputed_yolo_model = True\n\n#yolo code from: https://github.com/zzh8829/yolov3-tf2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def read_image(filename, mask = True, series = 1, split = \"train\"):\n    with tifffile.TiffFile(dirname + \"/\" + split + \"_images/\" + filename + \".tiff\") as infile:\n        img = infile.asarray(series = series)\n    if mask:\n        with tifffile.TiffFile(dirname + \"/\" + split + \"_label_masks/\" + filename + \"_mask.tiff\") as infile:\n            mask = infile.asarray(series = series)[:,:,0]\n        return img, mask\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_patches(image, mask = None, patch_size = 416):\n    xs = image.shape[0] // patch_size\n    ys = image.shape[1] // patch_size\n    for i in range(xs):\n        for j in range(ys):\n            img = image[(i*patch_size):((i+1)*patch_size), (j*patch_size):((j+1)*patch_size)]\n            if mask is not None:\n                msk = mask[(i*patch_size):((i+1)*patch_size), (j*patch_size):((j+1)*patch_size)]\n                yield img, msk\n            else:\n                yield img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_masked_dataframe():\n    d = pd.read_csv(dirname + \"/train.csv\")\n    d = d[d['data_provider'] == \"radboud\"]\n    masklist = os.listdir(dirname + \"/train_label_masks/\")\n    masklist = [name[:-10] for name in masklist]\n    d_masked = d.merge(pd.DataFrame({'image_id': masklist}))\n    return d_masked","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_box_level_area(box, mask, level):\n    box_area = box[2] * box[3]\n    box_mask = mask[box[1]:(box[1] + box[3]), box[0]:(box[0]+box[2])]\n    level_area = len(box_mask[box_mask == level])\n    #if level > 2:\n    #    print(level)\n    return np.array([level, box[0], box[1], box[2], box[3], level_area, box_area])\n    \ndef get_boxes(mask_original, resampling_factor = 8, min_box_area = 500, level_to_box_min_ratio = 0.5, box_to_image_min_ratio = 0.05, levels = [2,3,4,5]):\n    results = []\n    for level in levels:\n        mask = mask_original.copy()\n        mask = np.array(PIL.Image.fromarray(mask).resize([mask.shape[i] // resampling_factor for i in range(mask.ndim)]).resize(mask_original.shape))\n        mask[mask != level] = 0\n        _, _, stats, _ = cv2.connectedComponentsWithStats(mask, stats = cv2.CC_STAT_AREA)\n        #if level > 3:\n        #    print(\"initial:\", len(stats))\n        stats = stats[np.where(stats[:,2] * stats[:,3] > min_box_area)]\n        #if level > 3:\n        #    print(\"min box filter:\", len(stats))\n        mask_area = np.prod(mask.shape[:2])\n        stats = stats[np.where(stats[:,2] * stats[:,3] > mask_area * box_to_image_min_ratio), :][0]\n        #if level > 3:\n        #    print(\"mask area\", len(stats))\n        if len(stats) > 0:\n            stats = np.apply_along_axis(get_box_level_area, 1, stats, mask, level)\n            #if level > 3:\n            #    print(\"beforelevelbox:\", len(stats), stats)\n            stats = stats[stats[:,5] > level_to_box_min_ratio * stats[:,6]]\n            #if level > 3:\n            #    print(\"levelbox:\", len(stats))\n            results.append(stats)\n    if len(results) >= 1:\n        results = np.concatenate(results, axis = 0)\n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_plots(img, mask):\n    fig, ax = plt.subplots(1, 2)\n    ax[0].imshow(img)\n    ax[1].imshow(mask)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_class_file(classes = ['benign', 'grade3', 'grade4', 'grade5']):\n    classes = \"\\n\".join(classes)\n    with open(\"/kaggle/working/annotations/classes.names\", 'w') as ofile:\n        ofile.write(classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%mkdir /kaggle/working/annotations\nclasses = ['benign', 'grade3', 'grade4', 'grade5']\ncreate_class_file()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_tfrecord_example(img, boxes, classes = ['benign', 'grade3', 'grade4', 'grade5']):\n    height = img.shape[0]\n    width = img.shape[1]\n    classes_text = [classes[i].encode('utf-8') for i in list(boxes[:,0] - 2)]\n    xmin = list(boxes[:,1]/width)\n    ymin = list(boxes[:,2]/height)\n    xmax = list((boxes[:,1] + boxes[:,3] - 1)/width)\n    ymax = list((boxes[:,2] + boxes[:,4] - 1)/height)\n    img = PIL.Image.fromarray(img)\n    imgByteArr = io.BytesIO()\n    img.save(imgByteArr, format='JPEG')\n    imgByteArr = imgByteArr.getvalue()\n    example = tf.train.Example(features=tf.train.Features(feature={\n            'image/height': tf.train.Feature(int64_list=tf.train.Int64List(value=[height])),\n            'image/width': tf.train.Feature(int64_list=tf.train.Int64List(value=[width])),\n            'image/encoded': tf.train.Feature(bytes_list=tf.train.BytesList(value=[imgByteArr])),\n            'image/object/bbox/xmin': tf.train.Feature(float_list=tf.train.FloatList(value=xmin)),\n            'image/object/bbox/xmax': tf.train.Feature(float_list=tf.train.FloatList(value=xmax)),\n            'image/object/bbox/ymin': tf.train.Feature(float_list=tf.train.FloatList(value=ymin)),\n            'image/object/bbox/ymax': tf.train.Feature(float_list=tf.train.FloatList(value=ymax)),\n            'image/object/class/text': tf.train.Feature(bytes_list=tf.train.BytesList(value=classes_text)),\n        }))\n    return example","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_image(name, classes, patch_size = 416, split = \"train\", box_to_image_min_ratio = 0.05, level_to_box_min_ratio = 0.5, min_box_area = 100, writer = None):\n    image_complete, mask_complete = read_image(name)\n    examples = []\n    for patch_number, (img, msk) in enumerate(generate_patches(image_complete, mask_complete, patch_size)):\n        means = np.mean(img, axis = 2)\n        if len(means[means > 230]) > 0.8 * (patch_size * patch_size):\n            continue\n        boxes = get_boxes(msk, box_to_image_min_ratio = box_to_image_min_ratio, \n                          level_to_box_min_ratio = level_to_box_min_ratio, min_box_area = min_box_area)\n        #if name == \"8aa35680beee94df712c211da5bda211\" and patch_number == 22:\n            #print_plots(img, msk)\n        if boxes.shape[0] > 0:\n            if len(boxes) >= 100:\n                raise Exception(\"too many boxes: \" + str(len(boxes)))\n            example = create_tfrecord_example(img, boxes)\n            if writer:\n                writer.write(example.SerializeToString())\n            else:\n                examples.append(example)\n    return examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_example(example):\n    ymins = example.features.feature['image/object/bbox/ymin'].float_list.value\n    ymaxs = example.features.feature['image/object/bbox/ymax'].float_list.value\n    xmins = example.features.feature['image/object/bbox/xmin'].float_list.value\n    xmaxs = example.features.feature['image/object/bbox/xmax'].float_list.value\n    classes = example.features.feature['image/object/class/text'].bytes_list.value\n    img = example.features.feature['image/encoded'].bytes_list.value\n    nboxes = len(ymins)\n    boxes = [[xmins[i] * patch_size, ymins[i] * patch_size, xmaxs[i] * patch_size, ymaxs[i] * patch_size] for i in range(nboxes)]\n    classes = [c.decode('utf-8') for c in classes]\n    img = tf.io.decode_jpeg(tf.cast(img[0], tf.string)).numpy()\n    return boxes, classes, img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"box_to_image_min_ratio = 0.02\nlevel_to_box_min_ratio = 0.3\nmin_box_area = 30\npatch_size = 416\ntrain_filename = f\"train_ps{patch_size}bir{box_to_image_min_ratio}_lbr{level_to_box_min_ratio}_mba{min_box_area}.tfrecord\"\nvalid_filename = f\"val_ps{patch_size}bir{box_to_image_min_ratio}_lbr{level_to_box_min_ratio}_mba{min_box_area}.tfrecord\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if generate_yolo_ds:\n    d_masked = get_masked_dataframe()\n    train, valid = sklearn.model_selection.train_test_split(d_masked, test_size = 0.2, random_state = 0, stratify = d_masked['isup_grade'])\n    print(train.shape, valid.shape)\n\n    writer = tf.io.TFRecordWriter(train_filename)\n    for fname in tqdm.tqdm(train['image_id']):\n        #print(train[train['image_id'] == fname]['isup_grade'])\n        process_image(fname, classes = classes, patch_size = patch_size, split = \"train\", \n                      box_to_image_min_ratio = box_to_image_min_ratio, \n                      level_to_box_min_ratio = level_to_box_min_ratio, \n                      min_box_area = min_box_area, writer = writer)\n    writer.close()     \n\n    upload_to_gcs(train_filename, destination = \"prostate\")\n    del train\n\n    writer = tf.io.TFRecordWriter(valid_filename)\n    for fname in tqdm.tqdm(valid['image_id']):\n        #print(train[train['image_id'] == fname]['isup_grade'])\n        process_image(fname, classes = classes, patch_size = 416, split = \"train\", \n                      box_to_image_min_ratio = box_to_image_min_ratio, \n                      level_to_box_min_ratio = level_to_box_min_ratio, \n                      min_box_area = min_box_area, writer = writer)\n    writer.close()     \n    upload_to_gcs(valid_filename, destination = \"prostate\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_precomputed_yolo_ds:\n    #train_records = tf.data.TFRecordDataset(os.path.join(\"/kaggle\", \"input\", \"precomputedprostate\", train_filename))\n    valid_records = tf.data.TFRecordDataset(os.path.join(\"/kaggle\", \"input\", \"precomputedprostate\", valid_filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if generate_yolo_ds or use_precomputed_yolo_ds:\n    d_masked = get_masked_dataframe()\n    d_masked = d_masked[d_masked['isup_grade'] > 1]\n    printed = False\n    while (not printed):\n        sample_img=random.randint(0, d_masked.shape[0] - 1)\n        examples = process_image(d_masked.iloc[sample_img]['image_id'], classes = classes, patch_size = patch_size, split = \"train\", \n                              box_to_image_min_ratio = box_to_image_min_ratio, \n                              level_to_box_min_ratio = level_to_box_min_ratio, \n                              min_box_area = min_box_area, writer = None)\n        if len(examples) > 1:\n            rands = np.random.choice(range(len(examples)), min(4, len(examples)), replace = False)\n            nfig = len(rands)\n            fig, ax = plt.subplots(1, nfig, figsize = (20, 10))\n            for i in range(nfig):\n                boxes, class_preds, img = parse_example(examples[rands[i]])\n                for j in range(len(boxes)):\n                    box = boxes[j]\n                    xmin, ymin = box[0], box[1]\n                    xmax, ymax = box[2], box[3]\n                    w, h = xmax - xmin, ymax - ymin\n                    rect = matplotlib.patches.Rectangle((xmin, ymin), w, h, fill = None)\n                    ax[i].add_patch(rect)\n                    ax[i].text(xmin, ymin, class_preds[j])\n                ax[i].imshow(img)\n            printed = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if train_yolo:\n    %mkdir yolo\n    %cd yolo\n    !git clone https://github.com/zzh8829/yolov3-tf2.git\n    %cd yolov3-tf2\n\n    !sed -i '179s/Early/#Early/g' train.py\n    !sed -i '180s/{epoch}/x/g' train.py\n    #!sed -i '181s/verbose/#verbose/g' train.py\n\n    download_from_gcs(\"/kaggle/working/yolo/yolov3-tf2/data/yolov3.weights\", directory = \"prostate\")\n    !python convert.py --weights /kaggle/working/yolo/yolov3-tf2/data/yolov3.weights --output /kaggle/working/yolo/yolov3-tf2/checkpoints/yolov3.tf\n\n    !python train.py --dataset /kaggle/working/$train_filename --classes /kaggle/working/annotations/classes.names --num_classes 4 \\\n        --mode fit --transfer darknet --batch_size 32 --epochs 7 --weights /kaggle/working/yolo/yolov3-tf2/checkpoints/yolov3.tf \\\n        --weights_num_classes 80 --size 416 --val_dataset /kaggle/working/$valid_filename","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_image_features(filename, model, split = \"train\", patch_size = 416, max_boxes = 100):\n    img = read_image(filename, split = split, mask = False)\n    #print(img.shape)\n    #print((img.shape[0] // patch_size) * (img.shape[1] // patch_size))\n    counts = np.zeros((1,9))\n    patches = []\n    for i, patch in enumerate(generate_patches(img, patch_size = patch_size)):\n        #print(patch.shape)\n        #print(patch)\n        means = np.mean(patch, axis = 2)\n        if len(means[means > 230]) > 0.8 * (patch_size * patch_size):\n            #print(\"passing\")\n            pass\n        else:\n            patches.append(patch)\n            #print(\"staying\", len(means[means == 255]), 0.3 * (patch_size * patch_size))\n    if len(patches) > 0:\n        patches = np.stack(patches, axis = 0)\n        patches = patches/255\n        batch_size = 16\n        boxes, classes, scores, nums = [], [], [], []\n        for i in range(len(patches) // batch_size + 1):\n            batch = patches[(i*batch_size):((i+1)*batch_size)]\n            batch_boxes, batch_scores, batch_classes, batch_nums = model(batch)\n            boxes.append(batch_boxes)\n            scores.append(batch_scores)\n            classes.append(batch_classes)\n            nums.append(batch_nums)\n        boxes = np.concatenate(boxes, axis = 0)\n        scores = np.concatenate(scores, axis = 0)\n        classes = np.concatenate(classes, axis = 0)\n        nums = np.concatenate(nums, axis = 0)\n        boxes = boxes.reshape((boxes.shape[0] * boxes.shape[1], 4))\n        classes = classes.reshape((classes.shape[0]*classes.shape[1], 1))\n        boxes = np.concatenate([classes, boxes], axis = 1)\n        box_areas = (boxes[:, 3] - boxes[:,1]) * (boxes[:,4] - boxes[:,2])\n        boxes = boxes[box_areas > 0]\n        box_areas = box_areas[box_areas > 0]\n        counts = []\n        for i, grade in enumerate(['benign', 'g3', 'g4', 'g5']):\n            grade_boxes = boxes[boxes[:,0] == i]\n            grade_areas = box_areas[boxes[:,0] == i] / patches.shape[0]\n            counts.append(grade_boxes.shape[0])\n            counts.append(np.sum(grade_areas))\n    else:\n        counts = [0] * 8\n    counts.append(len(patches))\n    return counts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_features(d, model, patch_size = 416, split = \"train\", max_boxes = 100):\n    results = []\n    for row in tqdm.tqdm(range(d.shape[0])):\n        result = generate_image_features(d.iloc[row]['image_id'], model = model, split = split, patch_size = patch_size, max_boxes = max_boxes)\n        results.append(result)\n    df = pd.DataFrame.from_records(results, columns = ['benign_count', 'benign_area', 'g3_count', 'g3_area', 'g4_count', 'g4_area', 'g5_count', 'g5_area', 'patch_count'])#, \n    df = pd.concat([d.reset_index(drop = True), df], axis = 1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## code from https://github.com/zzh8829/yolov3-tf2/blob/master/train.py\n\nyolo_iou_threshold = 0\n\nphysical_devices = tf.config.experimental.list_physical_devices('GPU')\n#print(physical_devices)\nfor physical_device in physical_devices:\n    tf.config.experimental.set_memory_growth(physical_device, True)\n\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow.keras import Model\nfrom tensorflow.keras.layers import (\n    Add,\n    Concatenate,\n    Conv2D,\n    Input,\n    Lambda,\n    LeakyReLU,\n    MaxPool2D,\n    UpSampling2D,\n    ZeroPadding2D,\n    BatchNormalization,\n)\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.losses import (\n    binary_crossentropy,\n    sparse_categorical_crossentropy\n)\n\nYOLOV3_LAYER_LIST = [\n    'yolo_darknet',\n    'yolo_conv_0',\n    'yolo_output_0',\n    'yolo_conv_1',\n    'yolo_output_1',\n    'yolo_conv_2',\n    'yolo_output_2',\n]\n\nYOLOV3_TINY_LAYER_LIST = [\n    'yolo_darknet',\n    'yolo_conv_0',\n    'yolo_output_0',\n    'yolo_conv_1',\n    'yolo_output_1',\n]\n\n\ndef load_darknet_weights(model, weights_file, tiny=False):\n    wf = open(weights_file, 'rb')\n    major, minor, revision, seen, _ = np.fromfile(wf, dtype=np.int32, count=5)\n\n    if tiny:\n        layers = YOLOV3_TINY_LAYER_LIST\n    else:\n        layers = YOLOV3_LAYER_LIST\n\n    for layer_name in layers:\n        sub_model = model.get_layer(layer_name)\n        for i, layer in enumerate(sub_model.layers):\n            if not layer.name.startswith('conv2d'):\n                continue\n            batch_norm = None\n            if i + 1 < len(sub_model.layers) and \\\n                    sub_model.layers[i + 1].name.startswith('batch_norm'):\n                batch_norm = sub_model.layers[i + 1]\n\n            logging.info(\"{}/{} {}\".format(\n                sub_model.name, layer.name, 'bn' if batch_norm else 'bias'))\n\n            filters = layer.filters\n            size = layer.kernel_size[0]\n            in_dim = layer.input_shape[-1]\n\n            if batch_norm is None:\n                conv_bias = np.fromfile(wf, dtype=np.float32, count=filters)\n            else:\n                # darknet [beta, gamma, mean, variance]\n                bn_weights = np.fromfile(\n                    wf, dtype=np.float32, count=4 * filters)\n                # tf [gamma, beta, mean, variance]\n                bn_weights = bn_weights.reshape((4, filters))[[1, 0, 2, 3]]\n\n            # darknet shape (out_dim, in_dim, height, width)\n            conv_shape = (filters, in_dim, size, size)\n            conv_weights = np.fromfile(\n                wf, dtype=np.float32, count=np.product(conv_shape))\n            # tf shape (height, width, in_dim, out_dim)\n            conv_weights = conv_weights.reshape(\n                conv_shape).transpose([2, 3, 1, 0])\n\n            if batch_norm is None:\n                layer.set_weights([conv_weights, conv_bias])\n            else:\n                layer.set_weights([conv_weights])\n                batch_norm.set_weights(bn_weights)\n\n    assert len(wf.read()) == 0, 'failed to read all data'\n    wf.close()\n\n\ndef broadcast_iou(box_1, box_2):\n    # box_1: (..., (x1, y1, x2, y2))\n    # box_2: (N, (x1, y1, x2, y2))\n\n    # broadcast boxes\n    box_1 = tf.expand_dims(box_1, -2)\n    box_2 = tf.expand_dims(box_2, 0)\n    # new_shape: (..., N, (x1, y1, x2, y2))\n    new_shape = tf.broadcast_dynamic_shape(tf.shape(box_1), tf.shape(box_2))\n    box_1 = tf.broadcast_to(box_1, new_shape)\n    box_2 = tf.broadcast_to(box_2, new_shape)\n\n    int_w = tf.maximum(tf.minimum(box_1[..., 2], box_2[..., 2]) -\n                       tf.maximum(box_1[..., 0], box_2[..., 0]), 0)\n    int_h = tf.maximum(tf.minimum(box_1[..., 3], box_2[..., 3]) -\n                       tf.maximum(box_1[..., 1], box_2[..., 1]), 0)\n    int_area = int_w * int_h\n    box_1_area = (box_1[..., 2] - box_1[..., 0]) * \\\n        (box_1[..., 3] - box_1[..., 1])\n    box_2_area = (box_2[..., 2] - box_2[..., 0]) * \\\n        (box_2[..., 3] - box_2[..., 1])\n    return int_area / (box_1_area + box_2_area - int_area)\n\n\ndef draw_outputs(img, outputs, class_names):\n    boxes, objectness, classes, nums = outputs\n    boxes, objectness, classes, nums = boxes[0], objectness[0], classes[0], nums[0]\n    wh = np.flip(img.shape[0:2])\n    for i in range(nums):\n        x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))\n        x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))\n        img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)\n        img = cv2.putText(img, '{} {:.4f}'.format(\n            class_names[int(classes[i])], objectness[i]),\n            x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 2)\n    return img\n\n\ndef draw_labels(x, y, class_names):\n    img = x.numpy()\n    boxes, classes = tf.split(y, (4, 1), axis=-1)\n    classes = classes[..., 0]\n    wh = np.flip(img.shape[0:2])\n    for i in range(len(boxes)):\n        x1y1 = tuple((np.array(boxes[i][0:2]) * wh).astype(np.int32))\n        x2y2 = tuple((np.array(boxes[i][2:4]) * wh).astype(np.int32))\n        img = cv2.rectangle(img, x1y1, x2y2, (255, 0, 0), 2)\n        img = cv2.putText(img, class_names[classes[i]],\n                          x1y1, cv2.FONT_HERSHEY_COMPLEX_SMALL,\n                          1, (0, 0, 255), 2)\n    return img\n\n\ndef freeze_all(model, frozen=True):\n    model.trainable = not frozen\n    if isinstance(model, tf.keras.Model):\n        for l in model.layers:\n            freeze_all(l, frozen)\n\n\nyolo_anchors = np.array([(10, 13), (16, 30), (33, 23), (30, 61), (62, 45),\n                         (59, 119), (116, 90), (156, 198), (373, 326)],\n                        np.float32) / 416\nyolo_anchor_masks = np.array([[6, 7, 8], [3, 4, 5], [0, 1, 2]])\n\nyolo_tiny_anchors = np.array([(10, 14), (23, 27), (37, 58),\n                              (81, 82), (135, 169),  (344, 319)],\n                             np.float32) / 416\nyolo_tiny_anchor_masks = np.array([[3, 4, 5], [0, 1, 2]])\n\n\ndef DarknetConv(x, filters, size, strides=1, batch_norm=True):\n    if strides == 1:\n        padding = 'same'\n    else:\n        x = ZeroPadding2D(((1, 0), (1, 0)))(x)  # top left half-padding\n        padding = 'valid'\n    x = Conv2D(filters=filters, kernel_size=size,\n               strides=strides, padding=padding,\n               use_bias=not batch_norm, kernel_regularizer=l2(0.0005))(x)\n    if batch_norm:\n        x = BatchNormalization()(x)\n        x = LeakyReLU(alpha=0.1)(x)\n    return x\n\n\ndef DarknetResidual(x, filters):\n    prev = x\n    x = DarknetConv(x, filters // 2, 1)\n    x = DarknetConv(x, filters, 3)\n    x = Add()([prev, x])\n    return x\n\n\ndef DarknetBlock(x, filters, blocks):\n    x = DarknetConv(x, filters, 3, strides=2)\n    for _ in range(blocks):\n        x = DarknetResidual(x, filters)\n    return x\n\n\ndef Darknet(name=None):\n    x = inputs = Input([None, None, 3])\n    x = DarknetConv(x, 32, 3)\n    x = DarknetBlock(x, 64, 1)\n    x = DarknetBlock(x, 128, 2)  # skip connection\n    x = x_36 = DarknetBlock(x, 256, 8)  # skip connection\n    x = x_61 = DarknetBlock(x, 512, 8)\n    x = DarknetBlock(x, 1024, 4)\n    return tf.keras.Model(inputs, (x_36, x_61, x), name=name)\n\n\ndef DarknetTiny(name=None):\n    x = inputs = Input([None, None, 3])\n    x = DarknetConv(x, 16, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 32, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 64, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 128, 3)\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = x_8 = DarknetConv(x, 256, 3)  # skip connection\n    x = MaxPool2D(2, 2, 'same')(x)\n    x = DarknetConv(x, 512, 3)\n    x = MaxPool2D(2, 1, 'same')(x)\n    x = DarknetConv(x, 1024, 3)\n    return tf.keras.Model(inputs, (x_8, x), name=name)\n\n\ndef YoloConv(filters, name=None):\n    def yolo_conv(x_in):\n        if isinstance(x_in, tuple):\n            inputs = Input(x_in[0].shape[1:]), Input(x_in[1].shape[1:])\n            x, x_skip = inputs\n\n            # concat with skip connection\n            x = DarknetConv(x, filters, 1)\n            x = UpSampling2D(2)(x)\n            x = Concatenate()([x, x_skip])\n        else:\n            x = inputs = Input(x_in.shape[1:])\n\n        x = DarknetConv(x, filters, 1)\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, filters, 1)\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, filters, 1)\n        return Model(inputs, x, name=name)(x_in)\n    return yolo_conv\n\n\ndef YoloConvTiny(filters, name=None):\n    def yolo_conv(x_in):\n        if isinstance(x_in, tuple):\n            inputs = Input(x_in[0].shape[1:]), Input(x_in[1].shape[1:])\n            x, x_skip = inputs\n\n            # concat with skip connection\n            x = DarknetConv(x, filters, 1)\n            x = UpSampling2D(2)(x)\n            x = Concatenate()([x, x_skip])\n        else:\n            x = inputs = Input(x_in.shape[1:])\n            x = DarknetConv(x, filters, 1)\n\n        return Model(inputs, x, name=name)(x_in)\n    return yolo_conv\n\n\ndef YoloOutput(filters, anchors, classes, name=None):\n    def yolo_output(x_in):\n        x = inputs = Input(x_in.shape[1:])\n        x = DarknetConv(x, filters * 2, 3)\n        x = DarknetConv(x, anchors * (classes + 5), 1, batch_norm=False)\n        x = Lambda(lambda x: tf.reshape(x, (-1, tf.shape(x)[1], tf.shape(x)[2],\n                                            anchors, classes + 5)))(x)\n        return tf.keras.Model(inputs, x, name=name)(x_in)\n    return yolo_output\n\n\ndef yolo_boxes(pred, anchors, classes):\n    # pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...classes))\n    grid_size = tf.shape(pred)[1]\n    box_xy, box_wh, objectness, class_probs = tf.split(\n        pred, (2, 2, 1, classes), axis=-1)\n\n    box_xy = tf.sigmoid(box_xy)\n    objectness = tf.sigmoid(objectness)\n    class_probs = tf.sigmoid(class_probs)\n    pred_box = tf.concat((box_xy, box_wh), axis=-1)  # original xywh for loss\n\n    # !!! grid[x][y] == (y, x)\n    grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))\n    grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)  # [gx, gy, 1, 2]\n\n    box_xy = (box_xy + tf.cast(grid, tf.float32)) / \\\n        tf.cast(grid_size, tf.float32)\n    box_wh = tf.exp(box_wh) * anchors\n\n    box_x1y1 = box_xy - box_wh / 2\n    box_x2y2 = box_xy + box_wh / 2\n    bbox = tf.concat([box_x1y1, box_x2y2], axis=-1)\n\n    return bbox, objectness, class_probs, pred_box\n\n\ndef yolo_nms(outputs, anchors, masks, classes):\n    # boxes, conf, type\n    b, c, t = [], [], []\n\n    for o in outputs:\n        b.append(tf.reshape(o[0], (tf.shape(o[0])[0], -1, tf.shape(o[0])[-1])))\n        c.append(tf.reshape(o[1], (tf.shape(o[1])[0], -1, tf.shape(o[1])[-1])))\n        t.append(tf.reshape(o[2], (tf.shape(o[2])[0], -1, tf.shape(o[2])[-1])))\n\n    bbox = tf.concat(b, axis=1)\n    confidence = tf.concat(c, axis=1)\n    class_probs = tf.concat(t, axis=1)\n\n    scores = confidence * class_probs\n    boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(\n        boxes=tf.reshape(bbox, (tf.shape(bbox)[0], -1, 1, 4)),\n        scores=tf.reshape(\n            scores, (tf.shape(scores)[0], -1, tf.shape(scores)[-1])),\n        max_output_size_per_class=yolo_max_boxes,\n        max_total_size=yolo_max_boxes,\n        iou_threshold=yolo_iou_threshold,\n        score_threshold=yolo_score_threshold\n    )\n\n    return boxes, scores, classes, valid_detections\n\n\ndef YoloV3(size=None, channels=3, anchors=yolo_anchors,\n           masks=yolo_anchor_masks, classes=80, training=False):\n    x = inputs = Input([size, size, channels], name='input')\n\n    x_36, x_61, x = Darknet(name='yolo_darknet')(x)\n\n    x = YoloConv(512, name='yolo_conv_0')(x)\n    output_0 = YoloOutput(512, len(masks[0]), classes, name='yolo_output_0')(x)\n\n    x = YoloConv(256, name='yolo_conv_1')((x, x_61))\n    output_1 = YoloOutput(256, len(masks[1]), classes, name='yolo_output_1')(x)\n\n    x = YoloConv(128, name='yolo_conv_2')((x, x_36))\n    output_2 = YoloOutput(128, len(masks[2]), classes, name='yolo_output_2')(x)\n\n    if training:\n        return Model(inputs, (output_0, output_1, output_2), name='yolov3')\n\n    boxes_0 = Lambda(lambda x: yolo_boxes(x, anchors[masks[0]], classes),\n                     name='yolo_boxes_0')(output_0)\n    boxes_1 = Lambda(lambda x: yolo_boxes(x, anchors[masks[1]], classes),\n                     name='yolo_boxes_1')(output_1)\n    boxes_2 = Lambda(lambda x: yolo_boxes(x, anchors[masks[2]], classes),\n                     name='yolo_boxes_2')(output_2)\n\n    outputs = Lambda(lambda x: yolo_nms(x, anchors, masks, classes),\n                     name='yolo_nms')((boxes_0[:3], boxes_1[:3], boxes_2[:3]))\n\n    return Model(inputs, outputs, name='yolov3')\n\n\ndef YoloV3Tiny(size=None, channels=3, anchors=yolo_tiny_anchors,\n               masks=yolo_tiny_anchor_masks, classes=80, training=False):\n    x = inputs = Input([size, size, channels], name='input')\n\n    x_8, x = DarknetTiny(name='yolo_darknet')(x)\n\n    x = YoloConvTiny(256, name='yolo_conv_0')(x)\n    output_0 = YoloOutput(256, len(masks[0]), classes, name='yolo_output_0')(x)\n\n    x = YoloConvTiny(128, name='yolo_conv_1')((x, x_8))\n    output_1 = YoloOutput(128, len(masks[1]), classes, name='yolo_output_1')(x)\n\n    if training:\n        return Model(inputs, (output_0, output_1), name='yolov3')\n\n    boxes_0 = Lambda(lambda x: yolo_boxes(x, anchors[masks[0]], classes),\n                     name='yolo_boxes_0')(output_0)\n    boxes_1 = Lambda(lambda x: yolo_boxes(x, anchors[masks[1]], classes),\n                     name='yolo_boxes_1')(output_1)\n    outputs = Lambda(lambda x: yolo_nms(x, anchors, masks, classes),\n                     name='yolo_nms')((boxes_0[:3], boxes_1[:3]))\n    return Model(inputs, outputs, name='yolov3_tiny')\n\n\ndef YoloLoss(anchors, classes=80, ignore_thresh=0.5):\n    def yolo_loss(y_true, y_pred):\n        # 1. transform all pred outputs\n        # y_pred: (batch_size, grid, grid, anchors, (x, y, w, h, obj, ...cls))\n        pred_box, pred_obj, pred_class, pred_xywh = yolo_boxes(\n            y_pred, anchors, classes)\n        pred_xy = pred_xywh[..., 0:2]\n        pred_wh = pred_xywh[..., 2:4]\n\n        # 2. transform all true outputs\n        # y_true: (batch_size, grid, grid, anchors, (x1, y1, x2, y2, obj, cls))\n        true_box, true_obj, true_class_idx = tf.split(\n            y_true, (4, 1, 1), axis=-1)\n        true_xy = (true_box[..., 0:2] + true_box[..., 2:4]) / 2\n        true_wh = true_box[..., 2:4] - true_box[..., 0:2]\n\n        # give higher weights to small boxes\n        box_loss_scale = 2 - true_wh[..., 0] * true_wh[..., 1]\n\n        # 3. inverting the pred box equations\n        grid_size = tf.shape(y_true)[1]\n        grid = tf.meshgrid(tf.range(grid_size), tf.range(grid_size))\n        grid = tf.expand_dims(tf.stack(grid, axis=-1), axis=2)\n        true_xy = true_xy * tf.cast(grid_size, tf.float32) - \\\n            tf.cast(grid, tf.float32)\n        true_wh = tf.math.log(true_wh / anchors)\n        true_wh = tf.where(tf.math.is_inf(true_wh),\n                           tf.zeros_like(true_wh), true_wh)\n\n        # 4. calculate all masks\n        obj_mask = tf.squeeze(true_obj, -1)\n        # ignore false positive when iou is over threshold\n        best_iou = tf.map_fn(\n            lambda x: tf.reduce_max(broadcast_iou(x[0], tf.boolean_mask(\n                x[1], tf.cast(x[2], tf.bool))), axis=-1),\n            (pred_box, true_box, obj_mask),\n            tf.float32)\n        ignore_mask = tf.cast(best_iou < ignore_thresh, tf.float32)\n\n        # 5. calculate all losses\n        xy_loss = obj_mask * box_loss_scale * \\\n            tf.reduce_sum(tf.square(true_xy - pred_xy), axis=-1)\n        wh_loss = obj_mask * box_loss_scale * \\\n            tf.reduce_sum(tf.square(true_wh - pred_wh), axis=-1)\n        obj_loss = binary_crossentropy(true_obj, pred_obj)\n        obj_loss = obj_mask * obj_loss + \\\n            (1 - obj_mask) * ignore_mask * obj_loss\n        # TODO: use binary_crossentropy instead\n        class_loss = obj_mask * sparse_categorical_crossentropy(\n            true_class_idx, pred_class)\n\n        # 6. sum over (batch, gridx, gridy, anchors) => (batch, 1)\n        xy_loss = tf.reduce_sum(xy_loss, axis=(1, 2, 3))\n        wh_loss = tf.reduce_sum(wh_loss, axis=(1, 2, 3))\n        obj_loss = tf.reduce_sum(obj_loss, axis=(1, 2, 3))\n        class_loss = tf.reduce_sum(class_loss, axis=(1, 2, 3))\n\n        return xy_loss + wh_loss + obj_loss + class_loss\n    return yolo_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_precomputed_yolo_model:\n    yolo = YoloV3(classes=4, size= patch_size)\n    yolo.load_weights(os.path.join(\"/kaggle\", \"input\", \"precomputedprostate\", \"prostate_checkpoints_yolov3_train_x.tf\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_masked = get_masked_dataframe()\nd_masked = d_masked[d_masked['isup_grade'] > 1]\nprinted = False\nwhile (not printed):\n    sample_img=random.randint(0, d_masked.shape[0] - 1)\n    examples = process_image(d_masked.iloc[sample_img]['image_id'], classes = classes, patch_size = patch_size, split = \"train\", \n                          box_to_image_min_ratio = box_to_image_min_ratio, \n                          level_to_box_min_ratio = level_to_box_min_ratio, \n                          min_box_area = min_box_area, writer = None)\n    if len(examples) > 1:\n        rands = np.random.choice(range(len(examples)), min(4, len(examples)), replace = False)\n        nfig = len(rands)\n        fig, ax = plt.subplots(2, nfig, figsize = (20, 10))\n        for i in range(nfig):\n            boxes, class_nums, img = parse_example(examples[rands[i]])\n            for j in range(len(boxes)):\n                box = boxes[j]\n                xmin, ymin = box[0], box[1]\n                xmax, ymax = box[2], box[3]\n                w, h = xmax - xmin, ymax - ymin\n                rect = matplotlib.patches.Rectangle((xmin, ymin), w, h, fill = None)\n                #plt.imshow(img)\n                ax[0,i].add_patch(rect)\n                ax[0,i].text(xmin, ymin, class_nums[j])\n            ax[0,i].imshow(img)\n            boxes, scores, class_preds, nums = yolo(tf.expand_dims(img/255, 0))\n            img = draw_outputs(img, (boxes, scores, class_preds, nums), classes)\n            ax[1,i].imshow(img)\n        printed = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}