{"cells":[{"metadata":{},"cell_type":"markdown","source":"# DATASET:\n\n* tiles of slides with overlay mask\n* files removed : Missing mask, Mask only contains background, Mask does not contain tissue marked as cancerous although ISUP grade > 0\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# General packages\nimport os\nimport pandas as pd\nimport numpy as np\n\nimport cv2\nimport openslide\nimport PIL\nfrom PIL import Image\nimport zipfile\n\nimport matplotlib\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom IPython.display import Image, display","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Location of the training images\npath = '../input/prostate-cancer-grade-assessment'\n\n# Location of training labels\ntrain = pd.read_csv(f'{path}/train.csv')#.set_index('image_id')\ntest = pd.read_csv(f'{path}/test.csv')#.set_index('image_id')\nsubmission = pd.read_csv(f'{path}/sample_submission.csv')\n\n#load suspicious cases ( no mask on images)\nsuspicious = pd.read_csv(f'../input/suspicious-data-panda/suspicious_test_cases.csv')\n\n# image and mask directories\ndata_dir = f'{path}/train_images'\nmask_dir = f'{path}/train_label_masks'\n\n#remove all suspicious data of training set\ndf_train= train.copy().set_index('image_id')\n\nfor j in df_train.index:\n    for i in suspicious['image_id']:\n        if i == j:\n            df_train.drop([i], axis=0, inplace = True)\n            \ndf_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#overlay masks on the slide\ndef overlay_mask_on_slide(slide, mask, center='radboud', alpha=0.8, max_size=(400, 400)):\n    \"\"\"Show a mask overlayed on a slide.\n    \n    \"\"\"\n\n    if center not in ['radboud', 'karolinska']:\n        raise Exception(\"Unsupported palette, should be one of [radboud, karolinska].\")\n\n    # Load data from the highest level\n    slide_data = slide.read_region((0,0), slide.level_count - 1, slide.level_dimensions[-1])\n    mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n\n    # Mask data is present in the R channel\n    mask_data = mask_data.split()[0]\n    # Create alpha mask\n    alpha_int = int(round(255*alpha))\n    if center == 'radboud':\n        alpha_content = np.less(mask_data.split()[0], 2).astype('uint8') * alpha_int + (255 - alpha_int)\n    elif center == 'karolinska':\n        alpha_content = np.less(mask_data.split()[0], 1).astype('uint8') * alpha_int + (255 - alpha_int)\n        \n    alpha_content = PIL.Image.fromarray(alpha_content)\n    preview_palette = np.zeros(shape=768, dtype=int)\n    \n    if center == 'radboud':\n        # Mapping: {0: background, 1: stroma, 2: benign epithelium, 3: Gleason 3, 4: Gleason 4, 5: Gleason 5}\n        preview_palette[0:18] = (np.array([0, 0, 0, 0.5, 0.5, 0.5, 0, 1, 0, 1, 1, 0.7, 1, 0.5, 0, 1, 0, 0]) * 255).astype(int)\n    elif center == 'karolinska':\n        # Mapping: {0: background, 1: benign, 2: cancer}\n        preview_palette[0:9] = (np.array([0, 0, 0, 0, 1, 0, 1, 0, 0]) * 255).astype(int)\n    \n    mask_data.putpalette(data=preview_palette.tolist())\n    mask_rgb = mask_data.convert(mode='RGB')\n    \n    overlayed_image = PIL.Image.composite(image1=slide_data, image2=mask_rgb, mask=alpha_content)\n    \n    return(overlayed_image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#make tiles on a image\ndef tile(img, sz=128, N=16):\n    shape = np.array(img).shape\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                 constant_values=255)\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    if len(img) < N:\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    img = img[idxs]\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"save_dir = \"../train_overlay/\"\nos.makedirs(save_dir, exist_ok=True)\n\n#select the number of images to overlay\nfor imge in tqdm(df_train.index):\n    \n    # select the good provider corresponding at the picture\n    prov = df_train.loc[imge,'data_provider']\n    \n    #select the label for the image\n    label = df_train.loc[imge, 'isup_grade']\n    \n    #open slide and mask images\n    slide = openslide.OpenSlide(f'../input/prostate-cancer-grade-assessment/train_images/{imge}.tiff')\n    mask = openslide.OpenSlide(f'../input/prostate-cancer-grade-assessment/train_label_masks/{imge}_mask.tiff')\n    \n    #overlay mask on a slide\n    im1 = overlay_mask_on_slide(slide, mask, center= prov )\n    \n    #close slide and mask\n    slide.close()\n    mask.close()\n    \n    #create tiles \n    image= tile(im1, sz=128, N=16)\n    \n    #concatenate each tile on a picture\n    image = cv2.hconcat([cv2.vconcat([image[0], image[1], image[2], image[3]]), \n                             cv2.vconcat([image[4], image[5], image[6], image[7]]), \n                             cv2.vconcat([image[8], image[9], image[10], image[11]]), \n                             cv2.vconcat([image[12], image[13], image[14], image[15]])])\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    \n    matplotlib.image.imsave( save_dir+f'{imge}.png', image)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../train_overlay/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndisplay(Image(filename='../train_overlay/3046035f348012fdba6f7c53c4faa16e.png'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!tar -czf tiles_overlay_mask.tar.gz ../train_overlay/*.png","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}