{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":7222067,"sourceType":"datasetVersion","datasetId":4180266}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Breast Cancer TFRecord Dataset Decoding\n\n- Each train TFRecord contain roughly 548 images.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nprint(\"Tensorflow version \\t\\t:\", tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:08:50.113249Z","iopub.execute_input":"2023-12-17T11:08:50.113742Z","iopub.status.idle":"2023-12-17T11:09:02.715353Z","shell.execute_reply.started":"2023-12-17T11:08:50.113696Z","shell.execute_reply":"2023-12-17T11:09:02.714227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Image Format and Config\nIMAGE_FORMAT        = 'JPG'\nIMAGE_QUALITY       = 95\nN_SAMPLES_TFRECORDS = 548\nTARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS = ( 1280, 768, 1 )","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:09:05.966261Z","iopub.execute_input":"2023-12-17T11:09:05.96694Z","iopub.status.idle":"2023-12-17T11:09:05.972353Z","shell.execute_reply.started":"2023-12-17T11:09:05.966902Z","shell.execute_reply":"2023-12-17T11:09:05.971239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TPU VMs which can directly use Kaggle datasets\nDATA_DIR = \"/kaggle/input/rsna-breast-cancer-detection\"\nDS_PATH  = \"/kaggle/input/rsna-mammography-breast-cancer-tfrecord-dataset\"\nprint('DATA_DIR :', DATA_DIR)\nprint('DS_PATH  :', DS_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:10:32.902128Z","iopub.execute_input":"2023-12-17T11:10:32.903055Z","iopub.status.idle":"2023-12-17T11:10:32.907747Z","shell.execute_reply.started":"2023-12-17T11:10:32.903017Z","shell.execute_reply":"2023-12-17T11:10:32.906712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(f'{DATA_DIR}/train.csv')\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:14:19.863439Z","iopub.execute_input":"2023-12-17T11:14:19.863876Z","iopub.status.idle":"2023-12-17T11:14:20.022131Z","shell.execute_reply.started":"2023-12-17T11:14:19.863842Z","shell.execute_reply":"2023-12-17T11:14:20.021252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TFRecord file paths use GCS_PATH\nTFRECORDS_FILE_PATHS_TRAIN = sorted(tf.io.gfile.glob(f'{DS_PATH}/train*.tfrecords'))\nprint(f'Found {len(TFRECORDS_FILE_PATHS_TRAIN)} TFRecords')","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:14:32.971651Z","iopub.execute_input":"2023-12-17T11:14:32.972055Z","iopub.status.idle":"2023-12-17T11:14:33.006737Z","shell.execute_reply.started":"2023-12-17T11:14:32.972021Z","shell.execute_reply":"2023-12-17T11:14:33.0058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decode TFRecords","metadata":{}},{"cell_type":"code","source":"# Function to decode the TFRecords\ndef preprocess_decoder(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image'     : tf.io.FixedLenFeature([], tf.string),\n        'label'     : tf.io.FixedLenFeature([], tf.int64),\n        'patient_id': tf.io.FixedLenFeature([], tf.int64),\n        'image_id'  : tf.io.FixedLenFeature([], tf.int64),\n        'laterality': tf.io.FixedLenFeature([], tf.string),\n    })\n    patient_id = features['patient_id']\n    image_id   = features['image_id']    \n    # Convert strings to ASCII values for model\n    laterality = tf.strings.unicode_decode(features['laterality'], input_encoding='UTF-8')\n    \n    # Decode JPG Image\n    if IMAGE_FORMAT == 'PNG':\n        img_data = tf.io.decode_png(features['image'], channels=N_CHANNELS)\n    else:\n        img_data = tf.io.decode_jpeg(features['image'], channels=N_CHANNELS)\n    # Explicit reshape needed for TPU\n    img_data = tf.reshape(img_data, [TARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS])\n    label    = features['label']\n    \n    return {'image': img_data, 'patient_id': patient_id, 'image_id': image_id, 'laterality': laterality}, label","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:15:06.371589Z","iopub.execute_input":"2023-12-17T11:15:06.371972Z","iopub.status.idle":"2023-12-17T11:15:06.380529Z","shell.execute_reply.started":"2023-12-17T11:15:06.371945Z","shell.execute_reply":"2023-12-17T11:15:06.37929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset(tfrecords, isTrain=False, AUTO=tf.data.AUTOTUNE, batch_size=128):\n    # Load the dataset from the TFRecords files and apply parallel reads and compression\n    dataset = tf.data.TFRecordDataset(tfrecords, num_parallel_reads=AUTO, compression_type='GZIP')\n    # Define a function to decode the samples from the TFRecords\n    dataset = dataset.map(preprocess_decoder, num_parallel_calls=AUTO)\n    \n    # Optimize Model Training Dataset performance\n    if isTrain:\n        pass\n    \n    # Batch the data into batches of size batch_size\n    dataset = dataset.batch(batch_size, drop_remainder=isTrain)\n    # Prefetch the next batch of data while training on the current batch\n    dataset = dataset.prefetch(AUTO)    \n    return dataset ","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:25:38.821061Z","iopub.execute_input":"2023-12-17T11:25:38.821475Z","iopub.status.idle":"2023-12-17T11:25:38.827668Z","shell.execute_reply.started":"2023-12-17T11:25:38.821446Z","shell.execute_reply":"2023-12-17T11:25:38.826496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Sanity Check Dataset","metadata":{}},{"cell_type":"code","source":"# Sanity check, image and label statistics\nX_batch, y_batch = next(iter(get_dataset(TFRECORDS_FILE_PATHS_TRAIN, isTrain=True)))\nimage = X_batch['image'].numpy()\n\nprint(f'image shape  : {image.shape}, image dtype: {image.dtype}')\nprint(f'image min    : {image.min():.2f}, max: {image.max():.2f}')\nprint(f'y_batch shape: {y_batch.shape}, y_batch dtype: {y_batch.dtype}')","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:25:40.936257Z","iopub.execute_input":"2023-12-17T11:25:40.93688Z","iopub.status.idle":"2023-12-17T11:25:41.510732Z","shell.execute_reply.started":"2023-12-17T11:25:40.936849Z","shell.execute_reply":"2023-12-17T11:25:41.509656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Benchmark the dataset","metadata":{}},{"cell_type":"code","source":"import time \n\n# Function to benchmark the dataset\ndef benchmark_dataset(dataset, num_epochs=5, n_steps_per_epoch=15, batch=128):\n    start_time = time.perf_counter()\n    for epoch_num in range(num_epochs):\n        for idx, (inputs, labels) in enumerate(dataset.take(n_steps_per_epoch + 1)):\n            if idx == 0:\n                epoch_start = time.perf_counter()\n            elif idx == 1 and epoch_num == 0:\n                image = inputs['image']\n                print(f'image shape : {image.shape}, image dtype: {image.dtype},\\nlabels shape: {labels.shape}, labels dtype: {labels.dtype}\\n')\n            else:\n                pass\n        \n        epoch_t = time.perf_counter() - epoch_start\n        mean_step_t = round(epoch_t / n_steps_per_epoch * 1000, 1)\n        n_imgs_per_s = int(1 / (mean_step_t / 1000) * batch)\n        print(f'epoch {epoch_num} took: {round(epoch_t, 2)} sec, mean step duration: {mean_step_t}ms, images/s: {n_imgs_per_s}')\n        \n        \n# Benchmark Dataset\nbenchmark_dataset(get_dataset(TFRECORDS_FILE_PATHS_TRAIN, isTrain=True))","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:25:44.012245Z","iopub.execute_input":"2023-12-17T11:25:44.012646Z","iopub.status.idle":"2023-12-17T11:26:31.105169Z","shell.execute_reply.started":"2023-12-17T11:25:44.012611Z","shell.execute_reply":"2023-12-17T11:26:31.104108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explore tfrecord images","metadata":{}},{"cell_type":"code","source":"def show_batch(X_batch, label_batch=None, n_images=[25, 5], batch_size=128):\n    patient_id_batch = X_batch['patient_id'].numpy()\n    image_id_batch   = X_batch['image_id'].numpy()\n    image_batch      = X_batch['image'].numpy()\n    # Convert ASCII values back to strings\n    laterality       = tf.strings.unicode_encode(X_batch['laterality'], output_encoding='UTF-8').numpy()\n    \n    if label_batch is not None:\n        # Use np.unique to get unique values and their counts\n        values, counts = np.unique(label_batch, return_counts=True)\n        for value, count in zip(values, counts):\n            print(f\"y ratio Value: {value}, Count: {count}\")\n    \n    fig, axes = plt.subplots(\n        nrows=n_images[0], ncols=n_images[1], \n        figsize=(n_images[1]*5, n_images[0]*5)\n    )\n    fig.subplots_adjust(hspace=0.1, wspace=0.01)\n\n    for n, ax in enumerate(axes.flat):        \n        # Explicitly remove overlapping axes  \n        if n >= batch_size:   \n            for i in range(batch_size, np.prod(n_images)):\n                axes.flat[i].remove()\n            break\n        else:             \n            ax.imshow(image_batch[n]) # binary, gray, bone\n            ax.axis('off') \n            ax.set_title(f\"test\")\n            if label_batch is not None:\n                ax.set_title(f\"\"\"\n    label: {label_batch[n]}\n    laterality: {laterality[n].decode('utf-8')}\n    patient_id: {patient_id_batch[n]}\n    image_id: {image_id_batch[n]}\n    size: {image_batch[n].shape}\n    \"\"\")                \n    fig.tight_layout()\n    fig.show();\n    \n        \n# Show Example Batch\nX_batch, label_batch = next(iter(get_dataset(TFRECORDS_FILE_PATHS_TRAIN, isTrain=True).take(1)))\nshow_batch(X_batch, label_batch)\nplt.suptitle(\"AUGMENTED: Training Dataset from TFRecords (With Voi Lut)\", y=1.01, fontsize=28, family='monospace');","metadata":{"execution":{"iopub.status.busy":"2023-12-17T11:26:39.701804Z","iopub.execute_input":"2023-12-17T11:26:39.703005Z","iopub.status.idle":"2023-12-17T11:27:10.102351Z","shell.execute_reply.started":"2023-12-17T11:26:39.702961Z","shell.execute_reply":"2023-12-17T11:27:10.101459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## End of the Project","metadata":{}}]}