{"cells":[{"metadata":{},"cell_type":"markdown","source":"# About\nKernel contains image augmentation function which can \n+ rotate\n+ crop\n+ shear\n+ shift\n+ zoom\n+ change saturation\n+ change contrast\n+ change brightness \n+ adjust gamma.\n\nSince, the functions are implemented using tf and hence can be added to computational graph and prevents data augmentation or input pipeline from being the bottleneck.\n\nIt also has [custom training loop](https://www.kaggle.com/c/flower-classification-with-tpus/discussion/135443) making it much faster than **keras.fit** method. This reduces time taken for an epoch from 3 hrs to few seconds.\n\n![image](https://drive.google.com/uc?export=view&id=14lXZOqN49chDLhqDnSH1AkLNiilO7zFE)\n\nAlong with that, it has [concat tile dataset](https://www.kaggle.com/iafoss/panda-16x128x128-tiles-data) to improve scores. [This kernel](https://www.kaggle.com/iafoss/panda-16x128x128-tiles) has more information about the pre-processing pipeline of the dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport math\nimport os\nfrom tqdm import tqdm\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.layers import Dense\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom matplotlib import pyplot as plt\n\nprint(tf.__version__)\nprint(tf.keras.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet\nimport efficientnet.tfkeras as efn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n\n# Data access\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('panda-16x128x128-tiles-data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\nprint(train_df.shape)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"msk = np.random.rand(len(train_df)) < 0.85\ntrain = train_df[msk]\nvalid = train_df[~msk]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"S = set(train.image_id.values)\nD = dict(zip(train_df.image_id.values, train_df.isup_grade.values))\ntrain_imgs = []\nval_imgs = []\ntrain_labels = []\nval_labels = []\nfor img in tqdm(os.listdir('../input/panda-16x128x128-tiles-data/train')):\n    I = img.split('_')[0]\n    if I in S:\n        train_imgs.append(img)\n        train_labels.append(D[I])\n    else:\n        val_imgs.append(img)\n        val_labels.append(D[I])\ntrain = 0\ntrain = pd.DataFrame()\ntrain['image_id'] = train_imgs\ntrain['isup_grade'] = train_labels\nvalid = 0\nvalid = pd.DataFrame()\nvalid['image_id'] = val_imgs\nvalid['isup_grade'] = val_labels\nS,D,train_imgs,val_imgs,train_labels,val_labels = [0]*6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = train.head(512)\n# valid = valid.head(128)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape) \nprint(valid.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_paths = train[\"image_id\"].apply(lambda x: GCS_DS_PATH + '/train/' + x).values\nvalid_paths = valid[\"image_id\"].apply(lambda x: GCS_DS_PATH + '/train/' + x).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = pd.get_dummies(train['isup_grade']).astype('int32').values\nvalid_labels = pd.get_dummies(valid['isup_grade']).astype('int32').values\n\nprint(train_labels.shape) \nprint(valid_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE= 128 * strategy.num_replicas_in_sync\nimg_size = 128\nEPOCHS = 7\nnb_classes = 6\nHEIGHT = img_size\nWIDTH = img_size\nCHANNELS = 3\nSTEPS_PER_EPOCH = len(train)//BATCH_SIZE\nNUM_VALIDATION_IMAGES = len(valid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Augmentation function"},{"metadata":{"trusted":true},"cell_type":"code","source":"# data augmentation @cdeotte kernel: https://www.kaggle.com/cdeotte/rotation-augmentation-gpu-tpu-0-96 and @dimitreoliveira https://www.kaggle.com/dimitreoliveira/flower-with-tpus-advanced-augmentation\ndef transform_rotation(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated\n    DIM = HEIGHT\n    XDIM = DIM%2 #fix for size 331\n    \n    rotation = 15. * tf.random.normal([1],dtype='float32')\n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    \n    # ROTATION MATRIX\n    c1 = tf.math.cos(rotation)\n    s1 = tf.math.sin(rotation)\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    rotation_matrix = tf.reshape( tf.concat([c1,s1,zero, -s1,c1,zero, zero,zero,one],axis=0),[3,3] )\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(rotation_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shear(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly sheared\n    DIM = HEIGHT\n    XDIM = DIM%2 #fix for size 331\n    \n    shear = 5. * tf.random.normal([1],dtype='float32')\n    shear = math.pi * shear / 180.\n        \n    # SHEAR MATRIX\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)\n    shear_matrix = tf.reshape( tf.concat([one,s2,zero, zero,c2,zero, zero,zero,one],axis=0),[3,3] )    \n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shear_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_shift(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly shifted\n    DIM = HEIGHT\n    XDIM = DIM%2 #fix for size 331\n    \n    height_shift = 16. * tf.random.normal([1],dtype='float32') \n    width_shift = 16. * tf.random.normal([1],dtype='float32') \n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # SHIFT MATRIX\n    shift_matrix = tf.reshape( tf.concat([one,zero,height_shift, zero,one,width_shift, zero,zero,one],axis=0),[3,3] )\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(shift_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])\n\ndef transform_zoom(image):\n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly zoomed\n    DIM = HEIGHT\n    XDIM = DIM%2 #fix for size 331\n    \n    height_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    width_zoom = 1.0 + tf.random.normal([1],dtype='float32')/10.\n    one = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n        \n    # ZOOM MATRIX\n    zoom_matrix = tf.reshape( tf.concat([one/height_zoom,zero,zero, zero,one/width_zoom,zero, zero,zero,one],axis=0),[3,3] )\n\n    # LIST DESTINATION PIXEL INDICES\n    x = tf.repeat( tf.range(DIM//2,-DIM//2,-1), DIM )\n    y = tf.tile( tf.range(-DIM//2,DIM//2),[DIM] )\n    z = tf.ones([DIM*DIM],dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(zoom_matrix,tf.cast(idx,dtype='float32'))\n    idx2 = K.cast(idx2,dtype='int32')\n    idx2 = K.clip(idx2,-DIM//2+XDIM+1,DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES \n    idx3 = tf.stack( [DIM//2-idx2[0,], DIM//2-1+idx2[1,]] )\n    d = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM,DIM,3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(filename, label):\n    image = decode_image(filename)\n    p_spatial = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')\n    p_spatial2 = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')\n    p_pixel = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')\n    p_crop = tf.random.uniform([1], minval=0, maxval=1, dtype='float32')\n    \n    ### Spatial-level transforms\n    if p_spatial >= .2:\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_flip_up_down(image)\n        \n    if p_crop >= .7:\n        if p_crop >= .95:\n            image = tf.image.random_crop(image, size=[int(HEIGHT*.6), int(WIDTH*.6), CHANNELS])\n        elif p_crop >= .85:\n            image = tf.image.random_crop(image, size=[int(HEIGHT*.7), int(WIDTH*.7), CHANNELS])\n        elif p_crop >= .8:\n            image = tf.image.random_crop(image, size=[int(HEIGHT*.8), int(WIDTH*.8), CHANNELS])\n        else:\n            image = tf.image.random_crop(image, size=[int(HEIGHT*.9), int(WIDTH*.9), CHANNELS])\n        image = tf.image.resize(image, size=[HEIGHT, WIDTH])\n\n    if p_spatial2 >= .6:\n        if p_spatial2 >= .9:\n            image = transform_rotation(image)\n        elif p_spatial2 >= .8:\n            image = transform_zoom(image)\n        elif p_spatial2 >= .7:\n            image = transform_shift(image)\n        else:\n            image = transform_shear(image)\n        \n    ## Pixel-level transforms\n    if p_pixel >= .4:\n        if p_pixel >= .85:\n            image = tf.image.random_saturation(image, lower=0, upper=2)\n        elif p_pixel >= .65:\n            image = tf.image.random_contrast(image, lower=.8, upper=2)\n        elif p_pixel >= .5:\n            image = tf.image.random_brightness(image, max_delta=.2)\n        else:\n            image = tf.image.adjust_gamma(image, gamma=.6)\n\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None, image_size=(img_size, img_size)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TF datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(data_augment, num_parallel_calls=AUTO)\n    .repeat()\n    .shuffle(BATCH_SIZE*5)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_paths, valid_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Display a batch"},{"metadata":{"trusted":true},"cell_type":"code","source":"def batch_to_numpy_images_and_labels(data):\n    images, labels = data\n    numpy_images = images.numpy()\n    numpy_labels = labels.numpy()\n    if numpy_labels.dtype == object: # binary string in this case, these are image ID strings\n        numpy_labels = [None for _ in enumerate(numpy_images)]\n    # If no labels, only image IDs, return None for labels (this is the case for test data)\n    return numpy_images, numpy_labels\n\ndef display_one_image(image, title, subplot, red=False, titlesize=16):\n    plt.subplot(*subplot)\n    plt.axis('off')\n    plt.imshow(image)\n    if len(title) > 0:\n        plt.title(title, fontsize=int(titlesize) if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))\n    return (subplot[0], subplot[1], subplot[2]+1)\n\ndef display_batch_of_images(databatch, predictions=None):\n    \"\"\"This will work with:\n    display_batch_of_images(images)\n    display_batch_of_images(images, predictions)\n    display_batch_of_images((images, labels))\n    display_batch_of_images((images, labels), predictions)\n    \"\"\"\n    # data\n    images, labels = batch_to_numpy_images_and_labels(databatch)\n    if labels is None:\n        labels = [None for _ in enumerate(images)]\n        \n    # auto-squaring: this will drop data that does not fit into square or square-ish rectangle\n    rows = int(math.sqrt(len(images)))\n    cols = len(images)//rows\n        \n    # size and spacing\n    FIGSIZE = 13.0\n    SPACING = 0.1\n    subplot=(rows,cols,1)\n    if rows < cols:\n        plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))\n    else:\n        plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))\n    \n    # display\n    labels = np.array(labels).argmax(axis = 1)\n    CLASSES = {0:'isup_grade - 0',1:'isup_grade - 1',2:'isup_grade - 2',3:'isup_grade - 3',4:'isup_grade - 4',5:'isup_grade - 5'}\n    for i, (image, label) in enumerate(zip(images[:rows*cols], labels[:rows*cols])):\n        title = '' if label is None else CLASSES[label]\n        correct = True\n        if predictions is not None:\n            title, correct = title_from_label_and_target_(predictions[i], label)\n        dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3 # magic formula tested to work from 1x1 to 10x10 images\n        subplot = display_one_image(image, title, subplot, not correct, titlesize=dynamic_titlesize)\n    \n    #layout\n    plt.tight_layout()\n    if label is None and predictions is None:\n        plt.subplots_adjust(wspace=0, hspace=0)\n    else:\n        plt.subplots_adjust(wspace=SPACING, hspace=SPACING)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_batch_of_images(next(iter(train_dataset.unbatch().batch(20))))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"LR_START = 0.00001\nLR_MAX = 0.0001 * strategy.num_replicas_in_sync\nLR_MIN = 0.00001\nLR_RAMPUP_EPOCHS = 3\nLR_SUSTAIN_EPOCHS = 1\nLR_EXP_DECAY = .8\n\n@tf.function\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nlr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model():\n    base_model =  efn.EfficientNetB7(weights=None, include_top=False, pooling='avg', input_shape=(img_size, img_size, 3))\n    x = base_model.output\n    predictions = Dense(nb_classes, activation=\"softmax\")(x)\n    return Model(inputs=base_model.input, outputs=predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    \n    model = get_model()\n\n    # Instiate optimizer with learning rate schedule\n    class LRSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n        def __call__(self, step):\n            return lrfn(epoch=step//STEPS_PER_EPOCH)\n    optimizer = tf.keras.optimizers.Adam(learning_rate=LRSchedule())\n\n    # this also works but is not very readable\n    #optimizer = tf.keras.optimizers.Adam(learning_rate=lambda: lrfn(tf.cast(optimizer.iterations, tf.float32)//STEPS_PER_EPOCH))\n    \n    # Instantiate metrics\n    train_accuracy = tf.keras.metrics.CategoricalAccuracy()\n    valid_accuracy = tf.keras.metrics.CategoricalAccuracy()\n    train_loss = tf.keras.metrics.Sum()\n    valid_loss = tf.keras.metrics.Sum()\n    train_accuracy.reset_states()\n    valid_accuracy.reset_states()\n    # Loss\n    # The recommendation from the Tensorflow custom training loop  documentation is:\n    # loss_fn = lambda a,b: tf.nn.compute_average_loss(tf.keras.losses.sparse_categorical_crossentropy(a,b), global_batch_size=BATCH_SIZE)\n    # https://www.tensorflow.org/tutorials/distribute/custom_training#define_the_loss_function\n    # This works too and shifts all the averaging to the training loop which is easier:\n    loss_fn = tf.keras.losses.categorical_crossentropy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training Step function"},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_TPU_CALL = 1\nVALIDATION_STEPS_PER_TPU_CALL = 1\n\n@tf.function\ndef train_step(data_iter):\n    def train_step_fn(images, labels):\n        with tf.GradientTape() as tape:\n            probabilities = model(images, training=True)\n            loss = loss_fn(labels, probabilities)\n        grads = tape.gradient(loss, model.trainable_variables)\n        optimizer.apply_gradients(zip(grads, model.trainable_variables))\n        \n        #update metrics\n        train_accuracy.update_state(labels, probabilities)\n        train_loss.update_state(loss)\n        \n    # this loop runs on the TPU\n    for _ in tf.range(STEPS_PER_TPU_CALL):\n        strategy.experimental_run_v2(train_step_fn, next(data_iter))\n\n@tf.function\ndef valid_step(data_iter):\n    def valid_step_fn(images, labels):\n        probabilities = model(images, training=False)\n        loss = loss_fn(labels, probabilities)\n        \n        # update metrics\n        valid_accuracy.update_state(labels, probabilities)\n        valid_loss.update_state(loss)\n\n    # this loop runs on the TPU\n    for _ in tf.range(VALIDATION_STEPS_PER_TPU_CALL):\n        strategy.experimental_run_v2(valid_step_fn, next(data_iter))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def int_div_round_up(a, b):\n    return (a + b - 1) // b","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nfrom collections import namedtuple\nstart_time = epoch_start_time = time.time()\n\n# distribute the datset according to the strategy\ntrain_dist_ds = strategy.experimental_distribute_dataset(train_dataset)\n# Hitting End Of Dataset exceptions is a problem in this setup. Using a repeated validation set instead.\n# This will introduce a slight inaccuracy because the validation dataset now has some repeated elements.\nvalid_dist_ds = strategy.experimental_distribute_dataset(valid_dataset)\n\nprint(\"Training steps per epoch:\", STEPS_PER_EPOCH, \"in increments of\", STEPS_PER_TPU_CALL)\nprint(\"Validation images:\", NUM_VALIDATION_IMAGES,\n      \"Batch size:\", BATCH_SIZE,\n      \"Validation steps:\", NUM_VALIDATION_IMAGES//BATCH_SIZE, \"in increments of\", VALIDATION_STEPS_PER_TPU_CALL)\nprint(\"Repeated validation images:\", int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE*VALIDATION_STEPS_PER_TPU_CALL)*VALIDATION_STEPS_PER_TPU_CALL*BATCH_SIZE-NUM_VALIDATION_IMAGES)\nHistory = namedtuple('History', 'history')\nhistory = History(history={'loss': [], 'val_loss': [], 'sparse_categorical_accuracy': [], 'val_sparse_categorical_accuracy': []})\n\nepoch = 0\ntrain_data_iter = iter(train_dist_ds) # the training data iterator is repeated and it is not reset\n                                      # for each validation run (same as model.fit)\nvalid_data_iter = iter(valid_dist_ds) # the validation data iterator is repeated and it is not reset\n                                      # for each validation run (different from model.fit whre the\n                                      # recommendation is to use a non-repeating validation dataset)\n\nstep = 0\nepoch_steps = 0\nepoch_start_time_1 = time.time()\nwhile True:\n    \n    # run training step\n    train_step(train_data_iter)\n    epoch_steps += STEPS_PER_TPU_CALL\n    step += STEPS_PER_TPU_CALL\n    print('=', end='', flush=True)\n    \n    # validation run at the end of each epoch\n    if (step // STEPS_PER_EPOCH) > epoch:\n        print('>|', end='\\n', flush=True)\n        \n        # validation run\n        valid_epoch_steps = 0\n        for _ in range(int_div_round_up(NUM_VALIDATION_IMAGES, BATCH_SIZE*VALIDATION_STEPS_PER_TPU_CALL)):\n            valid_step(valid_data_iter)\n            valid_epoch_steps += VALIDATION_STEPS_PER_TPU_CALL\n            print('=', end='', flush=True)\n\n        # compute metrics\n#         history.history['sparse_categorical_accuracy'].append(train_accuracy.result().numpy())\n#         history.history['sparse_categorical_accuracy'].append('Not Calculated')\n#         history.history['val_sparse_categorical_accuracy'].append(valid_accuracy.result().numpy())\n#         history.history['loss'].append(train_loss.result().numpy() / (BATCH_SIZE*epoch_steps))\n#         history.history['val_loss'].append(valid_loss.result().numpy() / (BATCH_SIZE*valid_epoch_steps))\n#         # report metrics\n#         epoch_time = time.time() - epoch_start_time\n        print('\\nEPOCH {:d}/{:d}'.format(epoch+1, EPOCHS))\n#         print('time: {:0.1f}s'.format(epoch_time),\n#               'loss: {:0.4f}'.format(history.history['loss'][-1]),\n#               'accuracy: {:0.4f}'.format(history.history['sparse_categorical_accuracy'][-1]),\n#               'val_loss: {:0.4f}'.format(history.history['val_loss'][-1]),\n#               'val_acc: {:0.4f}'.format(history.history['val_sparse_categorical_accuracy'][-1]),\n#               'lr: {:0.4g}'.format(lrfn(epoch)),\n#               'steps/val_steps: {:d}/{:d}'.format(epoch_steps, valid_epoch_steps), flush=True)\n        \n        # set up next epoch\n        epoch = step // STEPS_PER_EPOCH\n        epoch_steps = 0\n        epoch_start_time = time.time()\n        train_accuracy.reset_states()\n        valid_accuracy.reset_states()\n        valid_loss.reset_states()\n        train_loss.reset_states()\n        if epoch >= EPOCHS:\n            break\n        print('Time taken: ',time.time()-epoch_start_time_1,'sec')\n        epoch_start_time_1 = time.time()\n        print()\n        \noptimized_ctl_training_time = time.time() - start_time\nprint(\"OPTIMIZED CTL TRAINING TIME: {:0.1f}s\".format(optimized_ctl_training_time))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thanks for reading. Please upvote if you found it helpful.\n\nFor QWK metric, TTA, image augmentations, Transfer learning.. visit my kernels [1](https://www.kaggle.com/prateekagnihotri/efficientnet-keras-train-qwk-loss-augmentation), [2](https://www.kaggle.com/prateekagnihotri/efficientnet-keras-infernce-tta), and [3](https://www.kaggle.com/prateekagnihotri/transfer-learning-starter-kernel)."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}