{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import math, re, os, time, gc\nimport skimage.io\nimport PIL\nimport time\nimport math\nimport warnings\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom collections import namedtuple\nimport tensorflow as tf\nimport albumentations as albu\nimport matplotlib.pyplot as plt\nimport tensorflow.keras.backend as K\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import StratifiedKFold\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\n\nSEED = 2020\nwarnings.filterwarnings('ignore')\nprint('Tensorflow version : {}'.format(tf.__version__))\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU or GPU detection","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Mixed Precision and/or XLA\nMixed Precision and XLA are not being used in this notebook but you can experiment using them. Change the following booleans to enable mixed precision and/or XLA on GPU/TPU. By default TPU already uses some mixed precision but we can add more (and it already uses XLA). These allow the GPU/TPU memory to handle larger batch sizes and can speed up the training process. The Nvidia V100 GPU has special Tensor Cores which get utilized when mixed precision is enabled. Unfortunately Kaggle's Nvidia P100 GPU does not have Tensor Cores to receive speed up.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MIXED_PRECISION = False\nXLA_ACCELERATE = False\n\nif MIXED_PRECISION:\n    from tensorflow.keras.mixed_precision import experimental as mixed_precision\n    if tpu: policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n    mixed_precision.set_policy(policy)\n    print('Mixed precision enabled')\n\nif XLA_ACCELERATE:\n    tf.config.optimizer.set_jit(True)\n    print('Accelerated Linear Algebra enabled')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Competition data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /kaggle/input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAIN_DIR = '../input/prostate-cancer-grade-assessment'\nTRAIN_IMG_DIR = '../input/panda-2020-level-1-2/train_images/train_images'\nTEST_IMG_DIR = '../input/prostate-cancer-grade-assessment/test_images'\nSAMPLE = '../input/prostate-cancer-grade-assessment/sample_submission.csv'\nsub_df  = pd.read_csv(os.path.join(MAIN_DIR, 'sample_submission.csv'))\ntest_df = pd.read_csv(os.path.join(MAIN_DIR, 'test.csv')).set_index('image_id')\ndf = pd.read_csv(os.path.join(MAIN_DIR, 'train.csv')).set_index('image_id')\n\nfiles = sorted(set([p[:32] for p in os.listdir(TRAIN_IMG_DIR)]))\ndf = df.loc[files]\ntrain_csv = df.reset_index()\n\n# Wrongly labeled data\nwrong_label = train_csv[(train_csv['isup_grade'] == 2) & (train_csv['gleason_score'] == '4+3')]\nprint(wrong_label)\ntrain_csv.drop([wrong_label.index[0]],inplace=True)\ntrain_csv = train_csv.reset_index()\n\n# incosistency with \"0\" and \"negative\"\ntrain_csv['gleason_score'] = train_csv['gleason_score'].apply(lambda x: \"0+0\" if x==\"negative\" else x)\nradboud_csv = train_csv[train_csv['data_provider'] == 'radboud']\nkarolinska_csv = train_csv[train_csv['data_provider'] != 'radboud']\n\n# GCS_DS_PATH = KaggleDatasets().get_gcs_path('prostate-cancer-grade-assessment')\n# GCS_DS_PATH = KaggleDatasets().get_gcs_path('panda-15x256x256-tiles-merged')\n# GCS_DS_PATH = KaggleDatasets().get_gcs_path('panda-4x512x512-merged-tiles')\n# GCS_DS_PATH = 'gs://kds-10bdb17716a14253285e22f42916ffb63fb2b65e918871e28846bed6' # For panda-15x256x256-tiles-merged\n# GCS_DS_PATH = 'gs://kds-eecf7646d11ffc661be4348df45875cb14d6dddf516071572ea7cc12' # For panda-4x512x512-merged-tiles\n# GCS_DS_PATH = 'gs://kds-ada7d35cbdd41095c77a3724f32409454cb4258b266b089616a4cc5a'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# GCS_DS_PATH = KaggleDatasets().get_gcs_path('panda-2020-level-1-2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = 'gs://kds-cc0d0c404acc1bbeeb5e049026f19daad1ba110ea6a6ea26434c78af'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !gsutil ls $GCS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"splits = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True)\nsplits = list(splits.split(radboud_csv, radboud_csv.isup_grade))\nfold_splits = np.zeros(len(radboud_csv)).astype(np.int)\nfor i in range(5): \n    fold_splits[splits[i][1]]=i\nradboud_csv['fold'] = fold_splits\nradboud_csv.head(5)\n\nsplits = StratifiedKFold(n_splits=5, random_state=SEED, shuffle=True)\nsplits = list(splits.split(karolinska_csv, karolinska_csv.isup_grade))\nfold_splits = np.zeros(len(karolinska_csv)).astype(np.int)\nfor i in range(5): \n    fold_splits[splits[i][1]]=i\nkarolinska_csv['fold'] = fold_splits\nkarolinska_csv.head(5)\n\ntrain_csv = pd.concat([radboud_csv, karolinska_csv])\ntrain_csv.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_FOLD = 0\ntrain_df = train_csv[train_csv['fold'] != TRAIN_FOLD]\nvalid_df = train_csv[train_csv['fold'] == TRAIN_FOLD]\n\nprint(train_df.shape)\nprint(valid_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = train_df[\"image_id\"].apply(lambda x: GCS_DS_PATH + '/train_images/train_images/' + x + '_2.jpeg').values\nvalid_paths = valid_df[\"image_id\"].apply(lambda x: GCS_DS_PATH + '/train_images/train_images/' + x + '_2.jpeg').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.get_dummies(train_df['isup_grade']).astype('int32').values\nvalid_labels = pd.get_dummies(valid_df['isup_grade']).astype('int32').values\n\nprint(train_labels.shape) \nprint(valid_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Further target pre-processing\n\n# Instead of predicting a single label, we will change our target to be a multilabel problem; \n# i.e., if the target is a certain class, then it encompasses all the classes before it. \n# E.g. encoding a class 4 retinopathy would usually be [0, 0, 0, 1], \n# but in our case we will predict [1, 1, 1, 1]. For more details, \n# please check out Lex's kernel.\n\ntrain_labels_multi = np.empty(train_labels.shape, dtype=train_labels.dtype)\ntrain_labels_multi[:, 5] = train_labels[:, 5]\n\nfor i in range(4, -1, -1):\n    train_labels_multi[:, i] = np.logical_or(train_labels[:, i], train_labels_multi[:, i+1])\n\nprint(\"Original y_train:\", train_labels.sum(axis=0))\nprint(\"Multilabel version:\", train_labels_multi.sum(axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Further target pre-processing\n\n# Instead of predicting a single label, we will change our target to be a multilabel problem; \n# i.e., if the target is a certain class, then it encompasses all the classes before it. \n# E.g. encoding a class 4 retinopathy would usually be [0, 0, 0, 1], \n# but in our case we will predict [1, 1, 1, 1]. For more details, \n# please check out Lex's kernel.\n\nvalid_labels_multi = np.empty(valid_labels.shape, dtype=valid_labels.dtype)\nvalid_labels_multi[:, 5] = valid_labels[:, 5]\n\nfor i in range(4, -1, -1):\n    valid_labels_multi[:, i] = np.logical_or(valid_labels[:, i], valid_labels_multi[:, i+1])\n\nprint(\"Original y_train:\", valid_labels.sum(axis=0))\nprint(\"Multilabel version:\", valid_labels_multi.sum(axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_paths1 = train_df[\"image_id\"].apply(lambda x: TRAIN_IMG_DIR + '/' + x + '_2.jpeg').values\n# valid_paths1 = valid_df[\"image_id\"].apply(lambda x: TRAIN_IMG_DIR + '/' + x + '_2.jpeg').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_MAX = 0\n# Y_MAX = 0\n# for i in range(len(train_paths1)):\n#     img = cv2.imread(train_paths1[i])\n#     if (X_MAX<img.shape[0]):\n#         X_MAX = img.shape[0]\n#     if (Y_MAX<img.shape[1]):\n#         Y_MAX = img.shape[1]\n\n        \n# for i in range(len(valid_paths1)):\n#     img = cv2.imread(valid_paths1[i])\n#     if (X_MAX<img.shape[0]):\n#         X_MAX = img.shape[0]\n#     if (Y_MAX<img.shape[1]):\n#         Y_MAX = img.shape[1]\n\n# print(X_MAX)\n# print(Y_MAX)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For Medium resolution\n# X_MAX = 11768\n# Y_MAX = 24152","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For Low resolution\nX_MAX = 2944\nY_MAX = 6040","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = (256, 256) # at this size, a GPU will run out of memory. Use the TPU\nEPOCHS = 15\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\nFOLDS = 1\nSEED = 777\nCHANNELS = 3\nHEIGHT = IMAGE_SIZE[0]\nWIDTH = IMAGE_SIZE[1]\nVALIDATION = True\nCLASSES = 6\nN = 36\nN_2 = 6\nSZ = 128\nMODELNAME = 'EfficientNetB3'\n# For Low Images 36x128x128 are enough\n# For Medium Images 64x256x256/36x512x512 are enough\nFOLDED_NUM_TRAIN_IMAGES = train_df.shape[0]\nFOLDED_NUM_VALID_IMAGES = valid_df.shape[0]\nSTEPS_PER_EPOCH = FOLDED_NUM_TRAIN_IMAGES // BATCH_SIZE\nVALIDATION_STEPS = FOLDED_NUM_VALID_IMAGES // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('*'*20)\nprint('Notebook info')\nprint('Training data : {}'.format(FOLDED_NUM_TRAIN_IMAGES))\nprint('Validing data : {}'.format(FOLDED_NUM_VALID_IMAGES))\nprint('Categorical classes : {}'.format(CLASSES))\nprint('Training image size : {}'.format(IMAGE_SIZE))\nprint('Training epochs : {}'.format(EPOCHS))\nprint('*'*20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization utilities\ndata -> pixels, nothing of much interest for the machine learning practitioner in this section.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# numpy and matplotlib defaults\nnp.set_printoptions(threshold=15, linewidth=80)\n\ndef batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef display_one_image(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n    \ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n   \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = label\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_image(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()\n    \ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.set_facecolor('#F8F8F8')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    #ax.set_ylim(0.28,1.05)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def augment_images(images):\n    images = tf.image.transpose(images)\n    images = tf.image.random_flip_up_down(images)\n    images = tf.image.random_flip_left_right(images)\n    return images   \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# @tf.function\ndef decode_image_n(filename, label=None):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.int32) - 255\n    image = tf.image.resize_with_pad(image, X_MAX, Y_MAX, method=tf.image.ResizeMethod.BILINEAR, antialias=False)\n    image = tf.expand_dims(image, 0)\n\n    tiles = tf.image.extract_patches(image, sizes = [1, SZ, SZ, 1], strides = [1, SZ, SZ, 1], rates=[1, 1, 1, 1], padding = 'VALID')\n    tiles = tf.reshape(tiles, [-1, SZ, SZ, 3])\n    \n    sort = tf.math.reduce_sum(tf.reshape(tiles, [tiles.shape[0], -1]), axis = 1)\n    sort = tf.argsort(sort, axis=0, direction='ASCENDING', stable=False, name=None)\n\n    tiles = tf.gather(tiles, sort[:N])\n    tiles = augment_images(tiles)\n\n    H_CONCAT = []\n    index = 0\n    for j in range(N_2):\n        V_CONCAT = []\n        for i in range(N_2):\n            V_CONCAT.append(tiles[index])\n            index+=1\n        H_CONCAT.append(tf.concat(V_CONCAT, axis = 0))\n        \n    tiles = tf.concat(H_CONCAT, axis = 1)\n    image = tf.cast(tiles, tf.int32) + 255\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, IMAGE_SIZE)\n\n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, IMAGE_SIZE)\n    if label is None:\n        return image\n    else:   \n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dataset(filenames, filelabels, labeled=True, ordered=False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # disregarding data order. Order does not matter since we will be shuffling the data anyway.\n\n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n\n    dataset = tf.data.Dataset.from_tensor_slices((filenames, filelabels)) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # uses data as soon as it streams in, rather than in its original order\n#     dataset = dataset.map(decode_image, num_parallel_calls=AUTO)\n    dataset = dataset.map(decode_image_n, num_parallel_calls=AUTO)\n    # returns a dataset of (image, label) pairs if labeled=True or (image, id) pairs if labeled=False\n    return dataset\n\ndef get_training_dataset(dataset = None, do_aug=True, file_names = None, file_labels = None):\n    if dataset == None: dataset = load_dataset(filenames = file_names, filelabels = file_labels, labeled=True)\n    dataset = dataset.shuffle(count_data_items(file_names))\n    dataset = dataset.repeat() # the training dataset must repeat for several epochs\n    \n    if do_aug:\n        dataset = dataset.map(data_augment_basic, num_parallel_calls=AUTO)\n#         dataset = dataset.map(transform_rotate, num_parallel_calls=AUTO)\n    \n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef get_validation_dataset(dataset = None, ordered=False, file_names = None, file_labels = None):\n    if dataset == None: dataset = load_dataset(filenames = file_names, filelabels = file_labels, labeled=True, ordered=ordered)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO) # prefetch next batch while training (autotune prefetch buffer size)\n    return dataset\n\ndef count_data_items(filenames):\n    return len(filenames)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Initial Augmentation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment_basic(image, label):\n    # data augmentation. Thanks to the dataset.prefetch(AUTO) statement in the next function (below),\n    # this happens essentially for free on TPU. Data pipeline code is executed on the \"CPU\" part\n    # of the TPU while the TPU itself is computing gradients.\n    image = tf.image.transpose(image)\n    image = tf.image.random_flip_up_down(image)\n    image = tf.image.random_flip_left_right(image)\n    #image = tf.image.random_saturation(image, 0, 2)\n    return image, label   \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Rotation Augmentation(https://www.kaggle.com/c/flower-classification-with-tpus/discussion/132191)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear = math.pi * shear / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n        \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n    \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n    \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), K.dot(zoom_matrix, shift_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def transform_rotate(image, label):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    DIM = IMAGE_SIZE[0]\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 15. * tf.random.normal([1],dtype='float32')\n    shr = 5. * tf.random.normal([1],dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    w_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    h_shift = 16. * tf.random.normal([1],dtype='float32') \n    w_shift = 16. * tf.random.normal([1],dtype='float32') \n  \n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot,shr,h_zoom,w_zoom,h_shift,w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image,tf.transpose(idx3))\n    \n    return tf.reshape(d,[DIM,DIM,3]), label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dataset visualizations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # data dump\n# print(\"Training data shapes:\")\n# for image, label in get_training_dataset(do_aug=False, file_names=TRAINING_FILENAMES).take(3):\n#     print(image.numpy().shape, label.numpy().shape)\n# print(\"Training data label examples:\", label.numpy())\n# print(\"Validation data shapes:\")\n# for image, label in get_validation_dataset().take(3):\n#     print(image.numpy().shape, label.numpy().shape)\n# print(\"Validation data label examples:\", label.numpy())\n# print(\"Test data shapes:\")\n# for image, idnum in get_test_dataset().take(3):\n#     print(image.numpy().shape, idnum.numpy().shape)\n# print(\"Test data IDs:\", idnum.numpy().astype('U')) # U=unicode string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Peek at training data\ntraining_dataset = get_training_dataset(do_aug=False, file_names = train_paths[:2], file_labels = train_labels[:2])\ntraining_dataset = training_dataset.unbatch().batch(2)\ntrain_batch = iter(training_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# run this cell again for next set of images\ndisplay_batch_of_images(next(train_batch))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # peer at test data\n# test_dataset = get_test_dataset()\n# test_dataset = test_dataset.unbatch().batch(20)\n# test_batch = iter(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # run this cell again for next set of images\n# display_batch_of_images(next(test_batch))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Learning rate schedule for TPU, GPU and CPU.\n# # Using an LR ramp up because fine-tuning a pre-trained model.\n# # Starting with a high LR would break the pre-trained weights.\n\n# LR_START = 0.00001\n# LR_MAX = 0.001 * strategy.num_replicas_in_sync\n# LR_MIN = 0.00001\n# LR_RAMPUP_EPOCHS = 3\n# LR_SUSTAIN_EPOCHS = 1\n# LR_EXP_DECAY = .8\n\n# # in custom training loop training you need an object to hold the epoch value\n# class LRSchedule():\n#     def __init__(self):\n#         self.epoch = 0\n        \n#     def set_epoch(self, epoch):\n#         self.epoch = epoch\n        \n#     @staticmethod\n#     def lrfn(epoch):\n#         if epoch < LR_RAMPUP_EPOCHS:\n#             lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n#         elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n#             lr = LR_MAX\n#         else:\n#             lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n#         return lr\n    \n#     def lr(self):\n#         return self.lrfn(self.epoch)\n    \n#     # LR scaled by 8 for CTL\n#     # not quite sure yet why LR must be scaled up by 8 (otherwise, does not converge the same)\n#     def lr_scaled(self):\n#         return self.lrfn(self.epoch) * 8\n    \n\n# lr_schedule = tf.keras.callbacks.LearningRateScheduler(LRSchedule.lrfn, verbose=True)\n\n# rng = [i for i in range(EPOCHS)]\n# y = [LRSchedule.lrfn(x) for x in rng]\n# plt.plot(rng, y)\n# print(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def get_model():\n#     with strategy.scope():\n#         pretrained_model = efn.EfficientNetB3(weights='noisy-student', include_top=False ,input_shape=[*IMAGE_SIZE, 3])\n# #         pretrained_model = tf.keras.applications.ResNet152V2(input_shape=[*IMAGE_SIZE, 3], include_top=False, weights='imagenet')\n#         pretrained_model.trainable = True # transfer learning\n        \n#         model = tf.keras.Sequential([\n#             pretrained_model,\n#             tf.keras.layers.GlobalAveragePooling2D(),\n#             tf.keras.layers.Flatten(),\n#             tf.keras.layers.BatchNormalization(),\n#             tf.keras.layers.Dropout(0.25),\n#             tf.keras.layers.Dense(512, activation='elu'),\n#             tf.keras.layers.BatchNormalization(),\n#             tf.keras.layers.Dropout(0.25),\n#             tf.keras.layers.Dense(CLASSES, activation='softmax')\n#         ])\n        \n#         model.compile(\n#             optimizer=tf.keras.optimizers.Adam(lr=1e-3),\n#             loss = 'categorical_crossentropy',\n#             metrics=['categorical_accuracy']\n#         )\n\n#     return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.00005 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 5\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .8\n        \n@tf.function\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    pretrained_model = efn.EfficientNetB3(weights='noisy-student', include_top=False ,input_shape=[*IMAGE_SIZE, 3])\n#         pretrained_model = tf.keras.applications.ResNet152V2(input_shape=[*IMAGE_SIZE, 3], include_top=False, weights='imagenet')\n    pretrained_model.trainable = True # transfer learning\n        \n    model = tf.keras.Sequential([\n        pretrained_model,\n        tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.25),\n        tf.keras.layers.Dense(512, activation='elu'),\n        tf.keras.layers.BatchNormalization(),\n        tf.keras.layers.Dropout(0.25),\n        tf.keras.layers.Dense(CLASSES, activation='softmax')\n    ])\n    \n    # Instiate optimizer with learning rate schedule\n    class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n        def __call__(self, step):\n            return lrfn(epoch=step//STEPS_PER_EPOCH)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=LRSchedule())\n\n    train_loss = tf.keras.metrics.Sum()\n    valid_loss = tf.keras.metrics.Sum()\n    train_accuracy = tf.keras.metrics.CategoricalAccuracy()\n    valid_accuracy = tf.keras.metrics.CategoricalAccuracy()\n    \n#     loss_fn = tf.keras.losses.categorical_crossentropy\n    loss_fn = tf.keras.losses.MSLE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training And Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_file = '{}/model_{}.h5'.format('.', 'V0')\n# earlystopper = tf.keras.callbacks.EarlyStopping(\n#     monitor='val_loss', \n#     patience=5, \n#     verbose=1,\n#     mode='min'\n# )\n# modelsaver = tf.keras.callbacks.ModelCheckpoint(\n#     model_file, \n#     monitor='val_loss', \n#     verbose=1, \n#     save_best_only=True,\n#     mode='min'\n# )\n# lrreducer = tf.keras.callbacks.ReduceLROnPlateau(\n#     monitor='val_loss',\n#     factor=.1,\n#     patience=3,\n#     verbose=1,\n#     min_lr=1e-7\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef qw_kappa_score(y_true, y_pred):     \n#     y_true=tf.math.argmax(y_true, axis=1)\n#     y_pred=tf.math.argmax(y_pred, axis=1)\n    def sklearn_qwk(y_true, y_pred) -> np.float64:\n        return cohen_kappa_score(y_true, y_pred, weights='quadratic')\n    return tf.compat.v1.py_func(sklearn_qwk, (y_true, y_pred), tf.double)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef train_step(images, labels):\n    with tf.GradientTape() as tape:\n        probabilities = model(images, training=True)\n        loss = loss_fn(labels, probabilities)\n    grads = tape.gradient(loss, model.trainable_variables)\n    optimizer.apply_gradients(zip(grads, model.trainable_variables))\n    \n    # update metrics\n    train_accuracy.update_state(labels, probabilities)\n    train_loss.update_state(loss)\n\n@tf.function\ndef valid_step(images, labels):\n    probabilities = model(images, training=False)\n    loss = loss_fn(labels, probabilities)\n    \n    # update metrics\n    valid_accuracy.update_state(labels, probabilities)\n    valid_loss.update_state(loss)\n    \n    return probabilities, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = epoch_start_time = time.time()\n\n# distribute the datset according to the strategy\ntrain_dist_ds = strategy.experimental_distribute_dataset(get_training_dataset(do_aug=True, file_names = train_paths, file_labels = train_labels_multi))\nvalid_dist_ds = strategy.experimental_distribute_dataset(get_validation_dataset(file_names = valid_paths, file_labels = valid_labels_multi))\n\nprint(\"Steps per epoch:\", STEPS_PER_EPOCH)\nHistory = namedtuple('History', 'history')\nhistory = History(history={'loss': [], 'val_loss': [], 'categorical_accuracy': [], 'val_categorical_accuracy': [], 'qwk':[]})\n\nepoch = 0\nfor step, (images, labels) in enumerate(train_dist_ds):\n    \n    # run training step\n    strategy.experimental_run_v2(train_step, args=(images, labels))\n    print('=', end='', flush=True)\n\n    # validation run at the end of each epoch\n    if ((step+1) // STEPS_PER_EPOCH) > epoch:\n        print('|', end='', flush=True)\n        \n        # validation run\n        for image, labels in valid_dist_ds:\n            probabilities, labels = strategy.experimental_run_v2(valid_step, args=(image, labels))\n            print('=', end='', flush=True)\n\n        # compute metrics\n#         qwk = qw_kappa_score(groundtruths, prediction)\n        history.history['categorical_accuracy'].append(train_accuracy.result().numpy())\n        history.history['val_categorical_accuracy'].append(valid_accuracy.result().numpy())\n        history.history['loss'].append(train_loss.result().numpy() / STEPS_PER_EPOCH)\n        history.history['val_loss'].append(valid_loss.result().numpy() / VALIDATION_STEPS)\n#         history.history['qwk'].append(qwk)\n        \n        # report metrics\n        epoch_time = time.time() - epoch_start_time\n        print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n        print('time: {:0.1f}s'.format(epoch_time),\n              'loss: {:0.4f}'.format(history.history['loss'][-1]),\n              'accuracy: {:0.4f}'.format(history.history['categorical_accuracy'][-1]),\n              'val_loss: {:0.4f}'.format(history.history['val_loss'][-1]),\n              'val_acc: {:0.4f}'.format(history.history['val_categorical_accuracy'][-1]),\n              'lr: {:0.4g}'.format(lrfn(epoch)), flush=True)\n        \n        # set up next epoch\n        epoch = (step+1) // STEPS_PER_EPOCH\n        epoch_start_time = time.time()\n        train_accuracy.reset_states()\n        valid_accuracy.reset_states()\n        valid_loss.reset_states()\n        train_loss.reset_states()\n        \n        if epoch >= EPOCHS:\n            break\n\nsimple_ctl_training_time = time.time() - start_time\nprint(\"SIMPLE CTL TRAINING TIME: {:0.1f}s\".format(simple_ctl_training_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(MODELNAME + \"_model.h5\")\ndisplay_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(history.history['categorical_accuracy'], history.history['val_categorical_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def train():\n        \n#     model = get_model()\n#     history = model.fit(\n#         get_training_dataset(do_aug=True, file_names = train_paths, file_labels = train_labels_multi), \n#         steps_per_epoch=STEPS_PER_EPOCH,\n#         epochs=EPOCHS, \n#         validation_data=get_validation_dataset(file_names = valid_paths, file_labels = valid_labels_mul),\n#         validation_steps=VALIDATION_STEPS,\n#         callbacks=[lrreducer,modelsaver,earlystopper]\n#     )\n#     model.save(MODELNAME + \"_model.h5\")\n#     return history, model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# start_time = time.time()\n\n# # run train and predict\n# history, model = train()\n# display_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\n# display_training_curves(history.history['accuracy'], history.history['val_accuracy'], 'accuracy', 212)\n\n# keras_fit_training_time = time.time() - start_time\n# print(\"KERAS FIT TRAINING TIME: {:0.1f}s\".format(keras_fit_training_time))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# cmdataset = get_validation_dataset(ordered=True) # since we are splitting the dataset and iterating separately on images and labels, order matters.\n# images_ds = cmdataset.map(lambda image, label: image)\n# labels_ds = cmdataset.map(lambda image, label: label).unbatch()\n# cm_correct_labels = next(iter(labels_ds.batch(NUM_VALIDATION_IMAGES))).numpy() # get everything as one batch\n# cm_probabilities = model.predict(images_ds)\n# cm_predictions = np.argmax(cm_probabilities, axis=-1)\n# print(\"Correct   labels: \", cm_correct_labels.shape, cm_correct_labels)\n# print(\"Predicted labels: \", cm_predictions.shape, cm_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cmat = confusion_matrix(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)))\n# score = f1_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# precision = precision_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# recall = recall_score(cm_correct_labels, cm_predictions, labels=range(len(CLASSES)), average='macro')\n# #cmat = (cmat.T / cmat.sum(axis=1)).T # normalized\n# display_confusion_matrix(cmat, score, precision, recall)\n# print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score, precision, recall))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visual validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = get_validation_dataset(file_names = valid_paths, file_labels = valid_labels_multi)\ndataset = dataset.unbatch().batch(20)\nbatch = iter(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # run this cell again for next set of images\nimages, labels = next(batch)\nprint(labels)\nprobabilities = model.predict(images)\nprint(probabilities)\npredictions = np.argmax(probabilities, axis=-1)\nprint(predictions)\n# display_batch_of_images((images, labels), predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = probabilities > 0.37757874193797547\npreds = preds.astype(int).sum(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}