{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nprint(\"\\n... IMPORTS STARTING ...\\n\")\n\nprint(\"\\n\\tVERSION INFORMATION\")\n# Machine Learning and Data Science Imports\nimport tensorflow as tf; print(f\"\\t\\t– TENSORFLOW VERSION: {tf.__version__}\");\nimport tensorflow_hub as tfhub; print(f\"\\t\\t– TENSORFLOW HUB VERSION: {tfhub.__version__}\");\nimport tensorflow_addons as tfa; print(f\"\\t\\t– TENSORFLOW ADDONS VERSION: {tfa.__version__}\");\nimport tensorflow_io as tfio; print(f\"\\t\\t– TENSORFLOW I/O VERSION: {tfio.__version__}\");\nimport pandas as pd; pd.options.mode.chained_assignment = None;\nimport numpy as np; print(f\"\\t\\t– NUMPY VERSION: {np.__version__}\");\nimport sklearn; print(f\"\\t\\t– SKLEARN VERSION: {sklearn.__version__}\");\nfrom sklearn.preprocessing import RobustScaler, PolynomialFeatures\nfrom pandarallel import pandarallel; pandarallel.initialize();\nfrom sklearn.model_selection import GroupKFold, StratifiedKFold\nfrom scipy.spatial import cKDTree\n\n# # RAPIDS\n# import cudf, cupy, cuml\n# from cuml.neighbors import NearestNeighbors\n# from cuml.manifold import TSNE, UMAP\n\n# Built In Imports\nfrom kaggle_datasets import KaggleDatasets\nfrom collections import Counter\nfrom datetime import datetime\nfrom glob import glob\nimport openslide\nimport warnings\nimport requests\nimport hashlib\nimport imageio\nimport IPython\nimport sklearn\nimport urllib\nimport zipfile\nimport pickle\nimport random\nimport shutil\nimport string\nimport json\nimport math\nimport time\nimport gzip\nimport ast\nimport sys\nimport io\nimport os\nimport gc\nimport re\n\n# Visualization Imports\nfrom matplotlib.colors import ListedColormap\nfrom matplotlib.patches import Rectangle\nimport matplotlib.patches as patches\nimport plotly.graph_objects as go\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tqdm; tqdm.pandas();\nimport plotly.express as px\nimport tifffile as tif\nimport seaborn as sns\nfrom PIL import Image, ImageEnhance; Image.MAX_IMAGE_PIXELS = 5_000_000_000;\nimport matplotlib; print(f\"\\t\\t– MATPLOTLIB VERSION: {matplotlib.__version__}\");\nfrom matplotlib import animation, rc; rc('animation', html='jshtml')\nimport plotly\nimport PIL\nimport cv2 as cv\n\nimport plotly.io as pio\nprint(pio.renderers)\n\n\n\ndef seed_it_all(seed=7):\n    \"\"\" Attempt to be Reproducible \"\"\"\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n\n\n    \nprint(\"\\n\\n... IMPORTS COMPLETE ...\\n\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-21T07:15:36.168254Z","iopub.execute_input":"2022-07-21T07:15:36.169039Z","iopub.status.idle":"2022-07-21T07:15:49.784667Z","shell.execute_reply.started":"2022-07-21T07:15:36.168899Z","shell.execute_reply":"2022-07-21T07:15:49.783381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Different interpolcasefoldation based on version of Kaggle Kernel\ntry:\n    DEFAULT_INTERPOLATION = Image.resampling.BILINEAR\nexcept Exception as AttributeError:\n    DEFAULT_INTERPOLATION = Image.BILINEAR","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:15:49.787176Z","iopub.execute_input":"2022-07-21T07:15:49.787648Z","iopub.status.idle":"2022-07-21T07:15:49.793479Z","shell.execute_reply.started":"2022-07-21T07:15:49.787603Z","shell.execute_reply":"2022-07-21T07:15:49.792359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Local path to training and validation images\nDATA_DIR = \"/kaggle/input/mayo-clinic-strip-ai\"\nsave_locally = None\nload_locally = None\n\nprint(f\"\\n... DATA DIRECTORY PATH IS:\\n\\t--> {DATA_DIR}\")\n\nprint(f\"\\n... IMMEDIATE CONTENTS OF DATA DIRECTORY IS:\")\nfor file in tf.io.gfile.glob(os.path.join(DATA_DIR, \"*\")): print(f\"\\t--> {file}\")\n\nprint(\"\\n\\n... DATA ACCESS SETUP COMPLETED ...\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:15:49.795387Z","iopub.execute_input":"2022-07-21T07:15:49.798091Z","iopub.status.idle":"2022-07-21T07:15:49.869495Z","shell.execute_reply.started":"2022-07-21T07:15:49.797982Z","shell.execute_reply":"2022-07-21T07:15:49.868134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\n... BASIC DATA SETUP STARTING ...\\n\\n\")\n\n# Open the training dataframe and display the initial dataframe\nTRAIN_DIR = os.path.join(DATA_DIR, \"train\")\nTRAIN_CSV = os.path.join(DATA_DIR, \"train.csv\")\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_df[\"image_path\"] = train_df.image_id.apply(lambda x: os.path.join(TRAIN_DIR, x+\".tif\"))\n\n# Add size information to train dataframe\ntrain_df[\"image_size\"] = train_df.image_path.progress_apply(lambda x: Image.open(x).size)\ntrain_df[\"image_width\"] = train_df.image_size.apply(lambda x: int(x[0]))\ntrain_df[\"image_height\"] = train_df.image_size.apply(lambda x: int(x[1]))\ntrain_df[\"image_area\"] = train_df.image_width*train_df.image_height\n\nprint(\"\\n... TRAINING DATAFRAME... \\n\")\ndisplay(train_df)\n\n# Open the testing dataframe and display the initial dataframe\nTEST_DIR = os.path.join(DATA_DIR, \"test\")\nTEST_CSV = os.path.join(DATA_DIR, \"test.csv\")\ntest_df = pd.read_csv(TEST_CSV)\ntest_df[\"image_path\"] = test_df.image_id.apply(lambda x: os.path.join(TEST_DIR, x+\".tif\"))\n\n# Add size information to test dataframe\ntest_df[\"image_size\"] = test_df.image_path.progress_apply(lambda x: Image.open(x).size)\ntest_df[\"image_width\"] = test_df.image_size.apply(lambda x: int(x[0]))\ntest_df[\"image_height\"] = test_df.image_size.apply(lambda x: int(x[1]))\ntest_df[\"image_area\"] = test_df.image_width*test_df[\"image_height\"]\n\nprint(\"\\n... TESTING DATAFRAME... \\n\")\ndisplay(test_df)\n\n# Open the other dataframe and display the initial dataframe\nOTHER_DIR = os.path.join(DATA_DIR, \"other\")\nOTHER_CSV = os.path.join(DATA_DIR, \"other.csv\")\nother_df = pd.read_csv(OTHER_CSV)\n\nprint(\"\\n... OTHER DATAFRAME... \\n\")\ndisplay(other_df)\n\n# Open the sample submission dataframe\nSS_CSV   = os.path.join(DATA_DIR, \"sample_submission.csv\")\nss_df = pd.read_csv(SS_CSV)\n\nprint(\"\\n... SAMPLE SUBMISSION DATAFRAME... \\n\")\ndisplay(ss_df)\n\n# For debugging purposes when the test set hasn't been substituted we will know\nDEBUG=len(ss_df)==4\nprint(f\"\\n\\n\\n... ARE WE DEBUGGING: {DEBUG}... \\n\")\n\n# Capture examples that span smallest, median, and large image size examples\nSMALLEST_IMAGE_ID = train_df[train_df.image_area==train_df.image_area.min()].image_id.values[0]\nSMALLEST_IMAGE_ROW = train_df[train_df.image_id==SMALLEST_IMAGE_ID]\nMEDIAN_IMAGE_ID = train_df.sort_values(by=\"image_area\").reset_index().iloc[len(train_df)//2].image_id\nMEDIAN_IMAGE_ROW = train_df[train_df.image_id==MEDIAN_IMAGE_ID]\nLARGEST_IMAGE_ID = train_df[train_df.image_area==train_df.image_area.max()].image_id.values[0]\nLARGEST_IMAGE_ROW = train_df[train_df.image_id==LARGEST_IMAGE_ID]\n\nprint(\"\\n... BASIC DATA SETUP FINISHED ...\\n\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:15:49.87218Z","iopub.execute_input":"2022-07-21T07:15:49.872477Z","iopub.status.idle":"2022-07-21T07:16:06.669785Z","shell.execute_reply.started":"2022-07-21T07:15:49.87245Z","shell.execute_reply":"2022-07-21T07:16:06.668723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3  HELPER FUNCTION & CLASSES","metadata":{}},{"cell_type":"code","source":"# Different interpolcasefoldation based on version of Kaggle Kernel\ntry:\n    DEFAULT_INTERPOLATION = Image.resampling.BILINEAR\nexcept Exception as AttributeError:\n    DEFAULT_INTERPOLATION = Image.BILINEAR","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.671342Z","iopub.execute_input":"2022-07-21T07:16:06.671674Z","iopub.status.idle":"2022-07-21T07:16:06.677634Z","shell.execute_reply.started":"2022-07-21T07:16:06.671644Z","shell.execute_reply":"2022-07-21T07:16:06.676324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def flatten_l_o_l(nested_list):\n    \"\"\" Flatten a list of lists \"\"\"\n    return [item for sublist in nested_list for item in sublist]","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.679212Z","iopub.execute_input":"2022-07-21T07:16:06.679642Z","iopub.status.idle":"2022-07-21T07:16:06.695932Z","shell.execute_reply.started":"2022-07-21T07:16:06.679578Z","shell.execute_reply":"2022-07-21T07:16:06.694875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pil_open_downsampled_slide(img_path, downsample_by=8, interpolation=None, reducing_gap=3.0, as_numpy=True):\n    \"\"\"\n    \n    Helper function to convert WSI into smaller downscaled version using PIL\n    \n    Timing details for MAYO CLINIC STRIP AI dataset:\n        SMALLEST IMAGE BY AREA (4417, 5314)\n            * Function takes ~0.5 seconds to run\n        MEDIAN IMAGE BY AREA   (17573, 38743)\n            * Function takes ~16 seconds to run\n        LARGEST IMAGE BY AREA  (48282, 101406)(~208X LARGER THAN SMALLEST IMAGE)(~7X LARGER THAN MEDIAN IMAGE)\n            * Function will CRASH!\n    \n    Args:\n        img_path (str): Path to .tif file to be downsampled\n        downsample_by (int): How many times smaller should resultant \n            image be. i.e. image_size*(1/downsample_by) = new_size\n        interpolation (int, optional): The value that enumerates\n            the type of interpolation to be used in downsampling.\n            This can be one\n               * `Image.NEAREST`  --> ENUM=0\n               * `Image.BOX`      --> ENUM=1\n               * `Image.BILINEAR` --> ENUM=2\n               * `Image.HAMMING`  --> ENUM=3\n               * `Image.BICUBIC`  --> ENUM=4\n               * `Image.LANCZOS`  --> ENUM=5\n           If omitted, it defaults to `Image.BILINEAR`\n       reducing_gap (float, optional): How many steps to use when interpolating/resampling\n           the image after integer downscaling is performed. \n           ELI5: \n               * Higher (Approaching 5) means it takes longer but is better quality\n               * Lower  (Approaching 0) means it is much faster but  lower  quality\n        as_numpy (bool, optional): Whether to return image as numpy array (default)\n           or leave as PIL.Image object for further manipulation\n    \n    Returns:\n        Downsampled image as a numpy array of type uint8 with only 3 channels\n    \"\"\"\n    \n    # Catches and warning\n    if downsample_by<8: print(\"\\n... WARNING – DUE TO LOW DOWNSCALE_BY VALUE THE RETURND IMAGE MAY OCCUPY A LARGE AMOUNT OF RAM – WARNING ...\\n\")\n    if interpolation is None: interpolation=DEFAULT_INTERPOLATION\n        \n    # Open the image with PIL\n    tmp_img = Image.open(img_path)\n    \n    # Get original dimensions and calculate new dimensions\n    original_dimensions = tmp_img.size\n    downsample_dimensions = (int(original_dimensions[0]/downsample_by), int(original_dimensions[1]/downsample_by))\n    \n    # Resize the image\n    tmp_img.thumbnail(size=downsample_dimensions, \n                      resample=interpolation,\n                      reducing_gap=reducing_gap)\n    \n    # Return either np.ndarray or PIL.Image object\n    return np.asarray(tmp_img) if as_numpy else tmp_img","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.697532Z","iopub.execute_input":"2022-07-21T07:16:06.698152Z","iopub.status.idle":"2022-07-21T07:16:06.709368Z","shell.execute_reply.started":"2022-07-21T07:16:06.698108Z","shell.execute_reply":"2022-07-21T07:16:06.708135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_slide_tiles(img_path, tile_size=(384,384), stride=(256, 256), downsample_by=4, drop_empty=True, drop_near_empty=True, near_empty_sum_thresh=2500, near_empty_bg_thresh=170, return_downsampled_image=False):\n    \"\"\" Tile a WSI\n    \n    Args:\n        img_path (str): Path to .tif file to be downsampled \n        tile_size (tuple of ints, optional): The dimensions of the tile/patches to be extracted\n        stride (tuple of ints, optional): The stride between tile/patches. \n            i.e. a stride of two: xx____, __xx__, ____xx\n        downsample_by (int, optional): How many times smaller should resultant \n            image be. i.e. image_size*(1/downsample_by) = new_size\n        drop_empty (bool, optional): Whether or not to drop images that are determined to be empty\n            i.e. ALL the pixels in the tile are determined to be background pixels\n        drop_near_empty (bool, optional): Whether or not to drop images that are determined to be \"mostly\" empty\n            i.e. MOST of the pixels in the tile are determined to be background pixels\n        near_empty_sum_thresh (int, optional): The threshold to use to determine what \"MOST\"\n            means in the context of background vs forgeround.\n            i.e. How many pixels are allowed to be background pixels before we drop?\n        near_empty_bg_thresh (int, optional): What qualifies as the threshold at which if the pixel\n            is brighter than this it is considered background... wheras lower than this is foreground\n            (I AM AWARE FOREGROUND WILL YIELD SOME PERCENTAGE OF \"BACKGROUND\" PIXELS)\n        return_downsampled_image (bool, optional): Whether we return the downsampled numpy array\n    \n    Returns:\n        A list of tiles – np.ndarray of shape `tile_size`\n    \"\"\"\n    \n    o_img = pil_open_downsampled_slide(img_path, downsample_by)\n    (orig_h, orig_w), (tile_h, tile_w), (stride_w, stride_h) = o_img.shape[:-1], tile_size, stride\n    \n    tile_map, non_bg_sum = {}, near_empty_sum_thresh\n    \n    row_steps = range(0, orig_h+stride_h, stride_h)\n    col_steps = range(0, orig_w+stride_w, stride_w)\n    n_steps = len(row_steps)*len(col_steps)\n    for j, y in enumerate(row_steps):\n        for i, x in enumerate(col_steps):\n            tile = o_img[y:(y+tile_h), x:(x+tile_w)]\n            if drop_empty or drop_near_empty: \n                non_bg_sum = np.where(tile>near_empty_bg_thresh, 0, 1).sum()\n            if (drop_empty and non_bg_sum==0) or (drop_near_empty and non_bg_sum<near_empty_sum_thresh): \n                if n_steps<=999: print(\" XXX \", end=\"\")\n            else:\n                loc_str = f\" {j*len(col_steps)+i+1:>03} \"\n                tile_map[loc_str]=tile\n                if n_steps<=999: print(loc_str, end=\"\")\n        if n_steps<=999: print(\"\\n\")\n        \n    if return_downsampled_image:\n        return tile_map, o_img\n    else:\n        return tile_map\n","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.710756Z","iopub.execute_input":"2022-07-21T07:16:06.711106Z","iopub.status.idle":"2022-07-21T07:16:06.727022Z","shell.execute_reply.started":"2022-07-21T07:16:06.711079Z","shell.execute_reply":"2022-07-21T07:16:06.725659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_and_plot(pd_row, draw_grid=True, grid_stride=(256,256), grid_size=(384,384), draw_text=True):\n    def _add_text(img, annot, color, pt1, pt2, **kwargs):\n            text_width, text_height = cv.getTextSize(annot, FONT, FONT_SCALE, FONT_THICKNESS)[0]\n            annot_loc = (int(round(pt1[0]+((pt2[0]-pt1[0])/2)-(text_width/2))), int(round(pt1[1]+((pt2[1]-pt1[1])/2)+(text_height/2))))\n            img = cv.putText(img, annot, annot_loc, FONT, FONT_SCALE, color, FONT_THICKNESS, FONT_LINE_TYPE)\n            return img\n        \n    print(f\"\\n... PROCESSING IMAGE {pd_row.image_id.values[0]} - {pd_row.image_size.values[0]} ...\\n\")\n    t1 = time.time()\n    image_tile_map, downsampled_image = get_slide_tiles(pd_row.image_path.values[0], return_downsampled_image=True)\n    \n    n_tiles = len(image_tile_map)\n    print(f\"\\nIMAGE PROCESSING ELAPSED TIME: {time.time()-t1:.4f} SECONDS\")\n    print(\"RESULTANT NUMBER OF TILES: \", n_tiles)\n    \n    print(\"\\n\\n... ORIGINAL IMAGE - DOWNSAMPLED BY A FACTOR OF 4 ...\\n\\n\")\n    w_to_h_ratio = pd_row.image_width.values[0]/pd_row.image_height.values[0]\n    plt.figure(figsize=(int(20*w_to_h_ratio), 20) if w_to_h_ratio>1 else (20, int(20/w_to_h_ratio)))\n    plt.axis(False)\n    plt.title(f\"Complete Downsampled Image – {pd_row.image_id.values[0]}\", fontweight=\"bold\")\n    if draw_grid:\n        clr_cnt, c_map, clrs, draw_map_fg, draw_map_bg = 0, plt.get_cmap(\"rainbow\"), [], [], []\n        row_steps = range(0, downsampled_image.shape[0]+grid_stride[1], grid_stride[1])\n        col_steps = range(0, downsampled_image.shape[1]+grid_stride[0], grid_stride[0])\n        clrs = [[int(x*255) for x in c_map((i+1)/len(image_tile_map.keys()))] for i in range(len(image_tile_map.keys()))]\n        draw_image = downsampled_image.copy()\n        if draw_text: \n            draw_annots, FONT, FONT_SCALE, FONT_THICKNESS, FONT_LINE_TYPE = [], cv.FONT_HERSHEY_SIMPLEX, 1.56789, 3, cv.LINE_AA\n            \n        for j, y in enumerate(row_steps):\n            for i, x in enumerate(col_steps):\n                if f\" {j*len(col_steps)+i+1:>03} \" in image_tile_map.keys():\n                    draw_map_fg.append(dict(pt1=(x,y), pt2=(x+grid_size[0], y+grid_size[1]), color=clrs[clr_cnt], thickness=3, annot=f\" {j*len(col_steps)+i+1:>03} \"))\n                    clr_cnt +=1\n                else:\n                    draw_map_bg.append(dict(pt1=(x,y), pt2=(x+grid_size[0], y+grid_size[1]), color=(10,10,10), thickness=2, annot=f\" {j*len(col_steps)+i+1:>03} \"))                \n                    \n        for draw_map in draw_map_bg+draw_map_fg:\n            draw_image = cv.rectangle(draw_image, draw_map[\"pt1\"], draw_map[\"pt2\"], draw_map[\"color\"], draw_map[\"thickness\"])\n            if draw_text:\n                _add_text(draw_image, draw_map[\"annot\"], draw_map[\"color\"], draw_map[\"pt1\"], draw_map[\"pt2\"])\n    plt.imshow(downsampled_image if not draw_grid else draw_image)\n    plt.tight_layout()\n    plt.show()\n\n    print(f\"\\n\\n\\n... PLOTTING FIRST {min(16, n_tiles)} OF {n_tiles} TILES FROM IMAGE ...\\n\")\n    plt.figure(figsize=(20,20))\n    for i, (loc_str, tile) in enumerate(image_tile_map.items()):\n        if i==16: break\n        plt.subplot(4,4,i+1)\n        plt.title(f\"Tile Location: {loc_str}\", fontweight=\"bold\")\n        plt.axis(False)\n        plt.imshow(tile)\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.728755Z","iopub.execute_input":"2022-07-21T07:16:06.729238Z","iopub.status.idle":"2022-07-21T07:16:06.756338Z","shell.execute_reply.started":"2022-07-21T07:16:06.729194Z","shell.execute_reply":"2022-07-21T07:16:06.755179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# IMAGE_IDS\nprint(f\"\\n... THERE ARE {train_df.image_id.nunique()} UNIQUE VALUES FOR {len(train_df)} EXAMPLES IN THE TRAINING DATA ...\")\nprint(f\"... THERE ARE {test_df.image_id.nunique()} UNIQUE VALUES FOR {len(test_df)} EXAMPLES IN THE TEST DATA ...\\n\")\n\nif train_df.image_id.nunique() == len(train_df): print(f\"... NO DUPLICATE IDS IN TRAINING DATA ...\") \nif test_df.image_id.nunique() == len(test_df): print(f\"... NO DUPLICATE IDS IN TRAINING DATA ...\") ","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.759706Z","iopub.execute_input":"2022-07-21T07:16:06.760413Z","iopub.status.idle":"2022-07-21T07:16:06.776483Z","shell.execute_reply.started":"2022-07-21T07:16:06.760376Z","shell.execute_reply":"2022-07-21T07:16:06.775659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CENTER_ID\nprint(f\"\\n... `center_id` HAS {train_df['center_id'].nunique()} UNIQUE VALUES FOR {len(train_df)} EXAMPLES IN THE TRAINING DATA ...\\n\\n\")\ntrain_df[\"center_id_count\"] = train_df.groupby(\"center_id\")[\"image_id\"].transform('count')\nfig = px.histogram(train_df, \"center_id\", \n             color=\"center_id_count\", \n             color_discrete_sequence=px.colors.sequential.Inferno_r,\n             title=\"<b>Histogram Showing Frequency of `center_id` Values In Training Dataset</b>\",\n             labels={\"color\":\"<b>Center ID Legend</b>\"},\n             )\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:06.778308Z","iopub.execute_input":"2022-07-21T07:16:06.779258Z","iopub.status.idle":"2022-07-21T07:16:08.016327Z","shell.execute_reply.started":"2022-07-21T07:16:06.779211Z","shell.execute_reply":"2022-07-21T07:16:08.015109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PATIENT_ID\nprint(f\"\\n... `patient_id` HAS {train_df['patient_id'].nunique()} UNIQUE VALUES FOR {len(train_df)} EXAMPLES IN THE TRAINING DATA ...\\n\\n\")\ntrain_df[\"patient_id_count\"] = train_df.groupby(\"patient_id\")[\"image_id\"].transform('count')\nfig = px.histogram(train_df, \"patient_id_count\", color=\"center_id\", \n             title=\"<b>Histogram Showing Frequency of `patient_id` Values In Training Dataset Coloured By Center ID</b>\",\n             labels={\"center_id\":\"<b>Center ID Legend</b>\"})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:08.018499Z","iopub.execute_input":"2022-07-21T07:16:08.018869Z","iopub.status.idle":"2022-07-21T07:16:08.141326Z","shell.execute_reply.started":"2022-07-21T07:16:08.018836Z","shell.execute_reply":"2022-07-21T07:16:08.140092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LABEL\nfig = px.pie(train_df, \"label\", title=\"<b>Pie Chart Showing Class Imbalance In Provided Label Clot Etiology</b>\", hole=0.30901699, )\nfig.show()\n\nfig = px.histogram(train_df, \"image_area\", color=\"label\",\n                title=\"<b>Scatter Plot Showing Class Distribution According to Slide Size</b>\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:08.143202Z","iopub.execute_input":"2022-07-21T07:16:08.143989Z","iopub.status.idle":"2022-07-21T07:16:08.282308Z","shell.execute_reply.started":"2022-07-21T07:16:08.143945Z","shell.execute_reply":"2022-07-21T07:16:08.280722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# IMAGE_SIZE\ndisplay(train_df[(train_df.image_area==train_df.image_area.min()) | (train_df.image_area==train_df.image_area.max())])\n\nfig = px.scatter(train_df, \"image_width\", \"image_height\", color=\"label\", size=\"image_area\",\n                 title=\"<b>Scatter Plot Showing Class Distribution According to Slide Size</b>\",\n                 width=1200, height=1200)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:08.283992Z","iopub.execute_input":"2022-07-21T07:16:08.284328Z","iopub.status.idle":"2022-07-21T07:16:08.392626Z","shell.execute_reply.started":"2022-07-21T07:16:08.284293Z","shell.execute_reply":"2022-07-21T07:16:08.391301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SAMPLES","metadata":{}},{"cell_type":"code","source":"process_and_plot(SMALLEST_IMAGE_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:08.394038Z","iopub.execute_input":"2022-07-21T07:16:08.39434Z","iopub.status.idle":"2022-07-21T07:16:10.903137Z","shell.execute_reply.started":"2022-07-21T07:16:08.394313Z","shell.execute_reply":"2022-07-21T07:16:10.902307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# process_and_plot(MEDIAN_IMAGE_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:10.904237Z","iopub.execute_input":"2022-07-21T07:16:10.905109Z","iopub.status.idle":"2022-07-21T07:16:10.909705Z","shell.execute_reply.started":"2022-07-21T07:16:10.905071Z","shell.execute_reply":"2022-07-21T07:16:10.908256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# process_and_plot(LARGEST_IMAGE_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-21T07:16:10.911173Z","iopub.execute_input":"2022-07-21T07:16:10.911527Z","iopub.status.idle":"2022-07-21T07:16:10.92182Z","shell.execute_reply.started":"2022-07-21T07:16:10.911486Z","shell.execute_reply":"2022-07-21T07:16:10.920946Z"},"trusted":true},"execution_count":null,"outputs":[]}]}