{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport PIL\nimport time\nimport math\nimport warnings\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.preprocessing.image import load_img\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\nimport seaborn as sns\nimport openslide\n\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd \nimport json\nimport skimage.io\nimport matplotlib.pyplot as plt\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Train Data Type\",train.dtypes)\nprint(\"Test Data Type\",test.dtypes)\n\nprint(\"unique ids : \", len(train.image_id.unique()))\nprint(\"unique data provider : \", len(train.data_provider.unique()))\nprint(\"unique isup_grade(target) : \", len(train.isup_grade.unique()))\nprint(\"unique gleason_score : \", len(train.gleason_score.unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nLabels are somewhat unbalanced.\n\"\"\"\ntrain['isup_grade'].hist(figsize = (10, 5))\n\n\"\"\"\nGleason Score Patterns\n\"\"\"\ntrain[['primary Gleason', 'secondary Gleason']] = train.gleason_score.str.split('+',expand=True)\ntrain[['primary Gleason', 'secondary Gleason']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['primary Gleason'].hist(figsize = (10, 5))\ntrain['secondary Gleason'].hist(figsize = (10, 5))\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Gleason Pattern 3 is the most common","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train['gleason_score'].unique())\n\"\"\"\nLets see what does this label \"0+0\" means , whene we have \"negative\" for no cancer data . They both map to zero. In such case\nthose two labels can be mapped to one label.\n\"\"\"\nprint(train[train['gleason_score']=='0+0']['isup_grade'].unique())\nprint(train[train['gleason_score']=='negative']['isup_grade'].unique())\n\n\"\"\"\nData points haveing no cancer cells , the proportion is around 2892/10616 = 27%\n\"\"\"\nprint(len(train[train['gleason_score']=='0+0']['isup_grade']))\nprint(len(train[train['gleason_score']=='negative']['isup_grade']))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nMapping of ISUP_grade to gleason score.\n3+4 and 4+3 map to different ISUP scores while other pairs like 3-5 and 5-3 , 4-5 and 5-4 map to same ISUP \n\"\"\"\nprint(train[(train['gleason_score'] == \"3+4\") | (train['gleason_score'] == \"4+3\")]['isup_grade'].unique())\nprint(train[(train['gleason_score']=='3+5') | (train['gleason_score']=='5+3')]['isup_grade'].unique())\nprint(train[(train['gleason_score']=='5+4') | (train['gleason_score']=='4+5')]['isup_grade'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train[train['gleason_score'] ==\"3+4\"]['isup_grade'].unique())\nprint(train[train['gleason_score'] ==\"4+3\"]['isup_grade'].unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nIdentified as an anomaly record\n\"\"\"\nprint(train[(train['gleason_score'] == \"4+3\") & (train['isup_grade'] == 2)])\ntrain.drop([7273],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nMapping Negative to 0+0 gleason score\n\"\"\"\n\ntrain['gleason_score'] = train['gleason_score'].apply(lambda x: \"0+0\" if x == \"negative\" else x)\ntrain.gleason_score.values\n\n\"\"\"\nTest Data\n\"\"\"\nprint(\"shape : \", test.shape)\nprint(\"unique ids : \", len(test.image_id.unique()))\nprint(\"unique data provider : \", len(test.data_provider.unique()))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Isup_Grade Score Distribution\n#### Observation is the isup_grade with \"0 & 1\" , no cancer data has the most number of value","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data = train.groupby('isup_grade').count()['image_id'].reset_index().sort_values(by='image_id',ascending=False)\n\nfig = px.bar(data, x='isup_grade', y='image_id',\n             hover_data=['image_id', 'isup_grade'], color='image_id',height=400)\nfig.show()\n\nfig = go.Figure(go.Funnelarea(\n    text =data.isup_grade,\n    values = data.image_id,\n    title = {\"position\": \"top center\", \"text\": \"ISUP_grade Distribution\"}\n    ))\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. ## Gleason Score Data Distribution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_gleason_score = train.groupby('gleason_score').count()['image_id'].reset_index().sort_values(by='image_id',ascending=False)\n\nfig = go.Figure(go.Funnelarea(\n    text =data_gleason_score.gleason_score,\n    values = data.image_id,\n    title = {\"position\": \"top center\", \"text\": \"Gleaseon Score Distribution\"}\n    ))\nfig.show()\n\nfig = px.bar(data_gleason_score, x='gleason_score', y='image_id',\n             hover_data=['image_id', 'gleason_score'], color='image_id',height=500)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby(['data_provider','gleason_score'])['image_id'].size().reset_index()\n\n'''\nVisualizing the GLEASON_SCORE distribution wrt Data_providers\n'''\n\nfig = plt.figure(figsize=(10,6))\nax = sns.countplot(x=\"gleason_score\", hue=\"data_provider\", data=train)\nfor p in ax.patches:\n    height = p.get_height()\n    ax.text(p.get_x()+p.get_width()/2.,\n                height + 3,\n                '{:1.2f}%'.format(100*height/10616),\n                ha=\"center\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"example = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_images\", '005e66f06bce9c2e49142536caf2f6ee.tiff'))\n\npatch = example.read_region((17800,19500), 0, (256, 256))\n\n# Display the image\ndisplay(patch)\n\n# Close the opened slide after use\nexample.close()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ex_img_path = '/kaggle/input/prostate-cancer-grade-assessment/train_images/'+train['image_id'][np.random.choice(len(train))]+'.tiff'\nexample_image = openslide.OpenSlide(ex_img_path)\nexample_image.properties","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img = example_image.read_region(location=(0,0),level=2,size=(example_image.level_dimensions[2][0],example_image.level_dimensions[2][1]))\nprint(img.size)\nimg","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.set_index('image_id')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_values(image,max_size=(600,400)):\n    slide = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_images\", f'{image}.tiff'))\n    \n    # Here we compute the \"pixel spacing\": the physical size of a pixel in the image.\n    # OpenSlide gives the resolution in centimeters so we convert this to microns.\n    f,ax =  plt.subplots(2 ,figsize=(6,16))\n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    patch = slide.read_region((1780,1950), 0, (256, 256)) #ZOOMED FUGURE\n    ax[0].imshow(patch) \n    ax[0].set_title('Zoomed Image')\n    \n    \n    ax[1].imshow(slide.get_thumbnail(size=max_size)) #UNZOOMED FIGURE\n    ax[1].set_title('Full Image')\n    \n    \n    print(f\"File id: {slide}\")\n    print(f\"Dimensions: {slide.dimensions}\")\n    print(f\"Microns per pixel / pixel spacing: {spacing:.3f}\")\n    print(f\"Number of levels in the image: {slide.level_count}\")\n    print(f\"Downsample factor per level: {slide.level_downsamples}\")\n    print(f\"Dimensions of levels: {slide.level_dimensions}\\n\\n\")\n    \n    print(f\"ISUP grade: {train.loc[image, 'isup_grade']}\")\n    print(f\"Gleason score: {train.loc[image, 'gleason_score']}\")\n    \nget_values('0a4b7a7499ed55c71033cefb0765e93d')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_images(images):\n    '''\n    This function takes in input a list of images. It then iterates through the image making openslide objects , on which different functions\n    for getting out information can be called later\n    '''\n    f, ax = plt.subplots(5,3, figsize=(18,22))\n    for i, image in enumerate(images):\n        slide = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_images\", f'{image}.tiff')) # Making Openslide Object\n        #Here we compute the \"pixel spacing\": the physical size of a pixel in the image,\n        #OpenSlide gives the resolution in centimeters so we convert this to microns\n        spacing = 1/(float(slide.properties['tiff.XResolution']) / 10000)\n        patch = slide.read_region((1780,1950), 0, (256, 256)) #Reading the image as before betweeen x=1780 to y=1950 and of pixel size =256*256\n        ax[i//3, i%3].imshow(patch) #Displaying Image\n        slide.close()       \n        ax[i//3, i%3].axis('off')\n        \n        image_id = image\n        data_provider = train.loc[image, 'data_provider']\n        isup_grade = train.loc[image, 'isup_grade']\n        gleason_score = train.loc[image, 'gleason_score']\n        ax[i//3, i%3].set_title(f\"ID: {image_id}\\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score}\")\n\n    plt.show() \n    \nimages = [\n'07a7ef0ba3bb0d6564a73f4f3e1c2293',\n    '037504061b9fba71ef6e24c48c6df44d',\n    '035b1edd3d1aeeffc77ce5d248a01a53',\n    '059cbf902c5e42972587c8d17d49efed',\n    '06a0cbd8fd6320ef1aa6f19342af2e68',\n    '06eda4a6faca84e84a781fee2d5f47e1',\n    '0a4b7a7499ed55c71033cefb0765e93d',\n    '0838c82917cd9af681df249264d2769c',\n    '046b35ae95374bfb48cdca8d7c83233f',\n    '074c3e01525681a275a42282cd21cbde',\n    '05abe25c883d508ecc15b6e857e59f32',\n    '05f4e9415af9fdabc19109c980daf5ad',\n    '060121a06476ef401d8a21d6567dee6d',\n    '068b0e3be4c35ea983f77accf8351cc8',\n    '08f055372c7b8a7e1df97c6586542ac8']\n\ndisplay_images(images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Understanding Masks\nQ) What are masks?\nApart from the slide-level label (present in the csv file), almost all slides in the training set have an associated mask with additional label information. These masks directly indicate which parts of the tissue are healthy and which are cancerous.hese masks are provided to assist with the development of strategies for selecting the most useful subsamples of the images. The mask values depend on the data provider:\n\nRadboud: Prostate glands are individually labelled, Valid values are:\n\n     0: background (non tissue) or unknown\n     1: stroma (connective tissue, non-epithelium tissue)\n     2: healthy (benign) epithelium\n     3: cancerous epithelium (Gleason 3)\n     4: cancerous epithelium (Gleason 4)\n     5: cancerous epithelium (Gleason 5)\nKarolinska: Regions are labelled, Valid values are:\n\n        1: background (non tissue) or unknown\n        2: benign tissue (stroma and epithelium combined)\n        3: cancerous tissue (stroma and epithelium combined)\n        \n        \nQ)A black canvas is displayed , Surprised ?? Wondering what that means? The Masks for Train are in RGB format right as said by organizers.\nThis happens for the following two reasons :\n\nThe label information is stored in the red (R) channel, the other channels are set to zero and can be ignored.\n\nThe masks are not image data like the WSIs.They are just matrices with values based on the data provider information provided above, instead of containing a range of values from 0 to 255, they only go up to a maximum of 6, representing the different class labels (check the dataset description for details on mask labels). Therefor when you try to visualize the mask, it will appear very dark as every value is close to 0. Applying the color map fixes the problem by assigning each label between 0 and 6 a distinct color.\n\nSo what we need to do is to grab read the image file using openslide object, take out the values of Red Level and then apply cmap to it","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"slide = '0005f7aaab2800f6170c399693a96917'\nmask_slide = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_label_masks/\", f'{slide}_mask.tiff')) # Making Openslide Obje\ndisplay(mask_slide.get_thumbnail(size=(600,400)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib\ndef display_masks(slides):    \n    f, ax = plt.subplots(2,3, figsize=(18,22))\n    for i, slide in enumerate(slides):\n        \n        mask = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_label_masks/\", f'{slide}_mask.tiff'))\n        mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n        cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n\n        ax[i//3, i%3].imshow(np.asarray(mask_data)[:,:,0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5) \n        mask.close()       \n        ax[i//3, i%3].axis('off')\n        \n        image_id = slide\n        data_provider = train.loc[slide, 'data_provider']\n        isup_grade = train.loc[slide, 'isup_grade']\n        gleason_score = train.loc[slide, 'gleason_score']\n        ax[i//3, i%3].set_title(f\"ID: {image_id}\\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score}\")\n        f.tight_layout()\n        \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_masks(images[:6]) #Visualizing Only six Examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def mask_img(image,max_size=(600,400)):\n    slide = openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_images\", f'{image}.tiff'))\n    mask =  openslide.OpenSlide(os.path.join(\"/kaggle/input/prostate-cancer-grade-assessment/train_label_masks/\", f'{image}_mask.tiff'))\n    # Here we compute the \"pixel spacing\": the physical size of a pixel in the image.\n    # OpenSlide gives the resolution in centimeters so we convert this to microns.\n    f,ax =  plt.subplots(1,2 ,figsize=(18,22))\n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    img = slide.get_thumbnail(size=(600,400)) #IMAGE \n    \n    mask_data = mask.read_region((0,0), mask.level_count - 1, mask.level_dimensions[-1])\n    cmap = matplotlib.colors.ListedColormap(['black', 'gray', 'green', 'yellow', 'orange', 'red'])\n    \n    ax[0].imshow(img) \n    #ax[0].set_title('Image')\n    \n    \n    ax[1].imshow(np.asarray(mask_data)[:,:,0], cmap=cmap, interpolation='nearest', vmin=0, vmax=5) #IMAGE MASKS\n    #ax[1].set_title('Image_MASK')\n    \n    \n    image_id = image\n    data_provider = train.loc[image, 'data_provider']\n    isup_grade = train.loc[image, 'isup_grade']\n    gleason_score = train.loc[image, 'gleason_score']\n    ax[0].set_title(f\"ID: {image_id}\\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score} IMAGE\")\n    ax[1].set_title(f\"ID: {image_id}\\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score} IMAGE_MASK\")\n    \n    \nmask_img('08f055372c7b8a7e1df97c6586542ac8')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dims, spacings, level_counts = [], [], []\ndown_levels, level_dims = [], []\n\nfor i in train.reset_index().image_id:\n    slide = openslide.OpenSlide(\"/kaggle/input/prostate-cancer-grade-assessment/train_images/\"+i+\".tiff\")\n    spacing = 1 / (float(slide.properties['tiff.XResolution']) / 10000)\n    dims.append(slide.dimensions)\n    spacings.append(spacing)\n    level_counts.append(slide.level_count)\n    down_levels.append(slide.level_downsamples)\n    level_dims.append(slide.level_dimensions)\n    slide.close()\n    del slide\n\ntrain['width']  = [i[0] for i in dims]\ntrain['height'] = [i[1] for i in dims]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}