{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Import Libraries**","metadata":{}},{"cell_type":"code","source":"!pip install imutils","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:03.040534Z","iopub.execute_input":"2022-09-27T10:47:03.040911Z","iopub.status.idle":"2022-09-27T10:47:13.462277Z","shell.execute_reply.started":"2022-09-27T10:47:03.040879Z","shell.execute_reply":"2022-09-27T10:47:13.460956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport keras\nimport matplotlib.pyplot as plt\nfrom matplotlib.patches import Rectangle\nfrom imutils import paths\nimport skimage\nfrom skimage.filters import sobel\nfrom skimage import segmentation\nfrom skimage.color import label2rgb\nfrom skimage.color import rgb2hed , hed2rgb\nfrom skimage.exposure import rescale_intensity\nfrom skimage.measure import regionprops , regionprops_table\nfrom scipy import ndimage as ndi\nimport tifffile as tifi\nfrom sklearn.preprocessing import StandardScaler\nimport seaborn as sn\nimport os\nimport cv2 as cv\n","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:13.467799Z","iopub.execute_input":"2022-09-27T10:47:13.470257Z","iopub.status.idle":"2022-09-27T10:47:13.480522Z","shell.execute_reply.started":"2022-09-27T10:47:13.470217Z","shell.execute_reply":"2022-09-27T10:47:13.479569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom pathlib import Path\nfrom datetime import datetime, timedelta\nimport time\nfrom dateutil.relativedelta import relativedelta\nimport gc\nimport copy\nimport pyarrow.parquet as pq\nimport pyarrow as pa\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport PIL\nfrom PIL import Image\n\npd.options.display.max_rows = 100\npd.options.display.max_columns = 100\n\nImage.MAX_IMAGE_PIXELS = None\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pytorch_lightning as pl\nrandom_seed=1234\npl.seed_everything(random_seed)","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:13.485195Z","iopub.execute_input":"2022-09-27T10:47:13.488224Z","iopub.status.idle":"2022-09-27T10:47:13.506652Z","shell.execute_reply.started":"2022-09-27T10:47:13.488188Z","shell.execute_reply":"2022-09-27T10:47:13.505548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Set Location/Path**","metadata":{}},{"cell_type":"code","source":"DATASET_FOLDER = \"/kaggle/input/mayo-clinic-strip-ai/\"","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:13.511462Z","iopub.execute_input":"2022-09-27T10:47:13.513509Z","iopub.status.idle":"2022-09-27T10:47:13.519092Z","shell.execute_reply.started":"2022-09-27T10:47:13.513474Z","shell.execute_reply":"2022-09-27T10:47:13.518301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_csv = os.path.join(DATASET_FOLDER, \"train.csv\")\ndf_train = pd.read_csv(path_csv)\ndisplay(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:13.523628Z","iopub.execute_input":"2022-09-27T10:47:13.526317Z","iopub.status.idle":"2022-09-27T10:47:13.551457Z","shell.execute_reply.started":"2022-09-27T10:47:13.526278Z","shell.execute_reply":"2022-09-27T10:47:13.550604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loading Images into DataFrame**","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nfrom tqdm.auto import tqdm\n\nImage.MAX_IMAGE_PIXELS = 5_000_000_000\n\nsizes = []\nfor name in tqdm(df_train[\"image_id\"]):\n    img = Image.open(os.path.join(DATASET_FOLDER, \"train\", f\"{name}.tif\"))\n    sizes.append({\"img_height\": img.height, \"img_width\": img.width})\n\ndf_sizes = pd.DataFrame(sizes)\nfor col in df_sizes.columns:\n    df_train[col] = df_sizes[col]","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:13.555477Z","iopub.execute_input":"2022-09-27T10:47:13.557686Z","iopub.status.idle":"2022-09-27T10:47:15.322984Z","shell.execute_reply.started":"2022-09-27T10:47:13.557649Z","shell.execute_reply":"2022-09-27T10:47:15.322145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\ndf_sizes = df_train[[\"img_width\", \"img_height\", \"label\"]]\nfor col in (\"img_width\", \"img_height\"):\n    df_sizes[col] = [round(i / 2000) * 2000 for i in df_sizes[col]]\n\ndf_sizes = df_sizes.groupby([\"img_width\", \"img_height\", \"label\"], as_index=False).size()\n# display(df_sizes.head())\nfig = px.scatter(df_sizes, x=\"img_width\", y=\"img_height\", size=\"size\", color=\"label\", height=600, width=600)\nfig.update_xaxes(range=[1_000, 120_000])\nfig.update_yaxes(scaleanchor=\"x\", scaleratio=1, range=[1_000, 120_000])\nfig.show() ","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:15.324439Z","iopub.execute_input":"2022-09-27T10:47:15.325019Z","iopub.status.idle":"2022-09-27T10:47:15.49128Z","shell.execute_reply.started":"2022-09-27T10:47:15.324972Z","shell.execute_reply":"2022-09-27T10:47:15.490154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train[\"pixels\"] = df_train[\"img_width\"] * df_train[\"img_height\"]\ndf_train.sort_values(\"pixels\", inplace=True)\ndisplay(df_train.head())","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:15.495758Z","iopub.execute_input":"2022-09-27T10:47:15.49622Z","iopub.status.idle":"2022-09-27T10:47:15.527443Z","shell.execute_reply.started":"2022-09-27T10:47:15.496182Z","shell.execute_reply":"2022-09-27T10:47:15.526232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Cropping, Resizing, Masking, Padding**\n","metadata":{}},{"cell_type":"code","source":"def prune_image_rows_cols(im, mask, thr=0.990):\n    # delete empty columns\n    for l in reversed(range(im.shape[1])):\n        if (np.sum(mask[:, l]) / float(mask.shape[0])) > thr:\n            im = np.delete(im, l, 1)\n    # delete empty rows\n    for l in reversed(range(im.shape[0])):\n        if (np.sum(mask[l, :]) / float(mask.shape[1])) > thr:\n            im = np.delete(im, l, 0)\n    return im\n\n\ndef mask_median(im, val=255):\n    masks = [None] * 3\n    for c in range(3):\n        masks[c] = im[..., c] >= np.median(im[:, :, c]) - 5\n    mask = np.logical_and(*masks)\n    im[mask, :] = val\n    return im, mask","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:15.53152Z","iopub.execute_input":"2022-09-27T10:47:15.531863Z","iopub.status.idle":"2022-09-27T10:47:15.544543Z","shell.execute_reply.started":"2022-09-27T10:47:15.531828Z","shell.execute_reply":"2022-09-27T10:47:15.543596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p train_images\n\ndef image_load_scale_norm(img_path, prune_thr=0.990, bg_val=255):\n    img = Image.open(img_path)\n    if (img.width * img.height) > 4_100_000_000:\n        print(f\"width: {img.width}, height: {img.height}, pixels: {img.width * img.height}\")\n        return None\n    scale = min(img.height / 3e2, img.width / 3e2)\n    if scale > 1:\n        tmp_size = int(img.width / scale), int(img.height / scale)\n        img.thumbnail(tmp_size, resample=Image.Resampling.BILINEAR, reducing_gap=scale)\n    im, mask = mask_median(np.array(img), val=bg_val)\n    im = prune_image_rows_cols(im, mask, thr=prune_thr)\n    img = Image.fromarray(im)\n    scale = min(img.height / 3e4, img.width / 3e4)\n    if scale > 1:\n        img = img.resize((int(img.width / scale), int(img.height / scale)), Image.ANTIALIAS)\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:15.550361Z","iopub.execute_input":"2022-09-27T10:47:15.552554Z","iopub.status.idle":"2022-09-27T10:47:16.685639Z","shell.execute_reply.started":"2022-09-27T10:47:15.552517Z","shell.execute_reply":"2022-09-27T10:47:16.683938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nfrom PIL import Image \nfrom tqdm.auto import tqdm\nfor name in tqdm(df_train[\"image_id\"]):\n    img_path = os.path.join(DATASET_FOLDER, \"train\", f\"{name}.tif\")\n    img = image_load_scale_norm(img_path)\n    if not img:\n        print(\"img\")\n        continue\n    old_size = img.size\n    new_size = (600,600)\n    new_im = Image.new(\"RGB\", new_size,\"White\")   ## luckily, this is already black!\n    box = tuple((n - o) // 2 for n, o in zip(new_size, old_size))\n    new_im.paste(img, box)\n    new_im.save(os.path.join(\"train_images\", f\"{name}.png\"))\n    del img\n    del new_im\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-09-27T10:47:16.687541Z","iopub.execute_input":"2022-09-27T10:47:16.687921Z","iopub.status.idle":"2022-09-27T10:49:06.753558Z","shell.execute_reply.started":"2022-09-27T10:47:16.687884Z","shell.execute_reply":"2022-09-27T10:49:06.74989Z"},"trusted":true},"execution_count":null,"outputs":[]}]}