{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom pathlib import Path\nfrom skimage.io import MultiImage\nimport openslide\nimport matplotlib.pyplot as plt\n\ninput_path = Path(\"../input/prostate-cancer-grade-assessment\")\ntrain_images_path = input_path.joinpath(\"train_images\")\ntrain_label_masks_path = input_path.joinpath(\"train_label_masks\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**First of all let's do this:**\n1. Load train.csv into pandas DataFrame\n2. Check train DataFrame for NaN values "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = pd.read_csv(input_path.joinpath(\"train.csv\"), index_col=\"image_id\")\n\n# Use the same mark for 'negative'/'0+0' Gleason score\ntrain_dataset.loc[train_dataset['gleason_score'] == 'negative', 'gleason_score'] = '0+0'\n\ntrain_dataset.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Non-Null Count for each column is equal to total number of entries.\n\n**It means that train_dataset has no empty values.**"},{"metadata":{},"cell_type":"markdown","source":"Now let's check if train_dataset match contents of ../input/prostate-cancer-grade-assessment/train_images."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images_names = [path.stem for path in train_images_path.glob(\"*.tiff\")]\ntrain_images = pd.Series(train_images_names, index=train_images_names, name=\"image_id\")\nmerge = train_dataset.merge(train_images, how='outer', left_index=True, right_index=True, indicator=True)\n\ntrain_dataset_only = merge[merge['_merge'] == 'left_only']\nprint(f\"There are {len(train_dataset_only)} records in train.csv which don't match files in train_images...\")\nprint(train_dataset_only.head())\nprint()\ntrain_images_only = merge[merge['_merge'] == 'right_only']\nprint(f\"There are {len(train_images_only)} files in train_images missing in train.csv...\")\nprint(train_images_only.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So train.csv records and train_images contents match."},{"metadata":{},"cell_type":"markdown","source":"Let's explore the shapes of train images."},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_shape(train_dataset_row: pd.Series) -> tuple:\n    try:\n        with openslide.OpenSlide(train_images_path.joinpath(f\"{train_dataset_row.name}.tiff\").as_posix()) as open_slide:\n            return open_slide.level_dimensions\n    except openslide.OpenSlideError:\n        return np.nan\n    \ntrain_dataset['level_dimensions'] = train_dataset.apply(image_shape, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset[train_dataset['level_dimensions'].isna()]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There's no invalid train images."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset['level_dimensions'].apply(len).unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Each train image has exactly 3 levels."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Total number of unique level dimensions of train images is {len(train_dataset['level_dimensions'].unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"radboud_level_dimensions = train_dataset[train_dataset['data_provider']=='radboud']['level_dimensions'].unique()\nkarolinska_level_dimensions = train_dataset[train_dataset['data_provider']=='karolinska']['level_dimensions'].unique()\nprint(f\"Number of unique level dimensions of radboud train images is {len(radboud_level_dimensions)}\")\nprint(f\"radboud level dimensions varies from {radboud_level_dimensions.min()} to {radboud_level_dimensions.max()}\")\nprint(f\"Number of unique level dimensions of karolinska train images is {len(karolinska_level_dimensions)}\")\nprint(f\"karolinska level dimensions varies from {karolinska_level_dimensions.min()} to {karolinska_level_dimensions.max()}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's explore data distribution by data provider."},{"metadata":{"trusted":true},"cell_type":"code","source":"distrib_by_data_provider = train_dataset.groupby(['data_provider', 'isup_grade', 'gleason_score']).size()\ndata_provider_count = distrib_by_data_provider.sum(level='data_provider')\ndata_provider_isup_grade_count = distrib_by_data_provider.sum(level=['data_provider', 'isup_grade'])\nisup_grade_count = distrib_by_data_provider.sum(level=['isup_grade'])\n\n# ISUP grade visualization\nfig_isup, ax_isup = plt.subplots(1, 2, figsize=(16,8))\n\n# On the left axes we visualize \n# ISUP data distribution by data provider through donut graph \nsize = 0.3\nouter_colors = ['tab:blue', 'tab:orange']\ncmap_blues = plt.get_cmap(\"Blues\")\ncmap_oranges = plt.get_cmap(\"Oranges\")\ninner_colors = np.concatenate([cmap_blues(np.linspace(0, 1, 6)), cmap_oranges(np.linspace(0, 1, 6))])\n\n# outer ring\nax_isup[0].pie(\n    data_provider_count,\n    radius=1,\n    autopct='%.2f%%',\n    pctdistance=0.85,\n    textprops={'size': 'larger'},\n    wedgeprops=dict(width=size, edgecolor='w'),\n    colors=outer_colors\n)\n\n# inner ring\nax_isup[0].pie(\n    data_provider_isup_grade_count,\n    radius=1-size,\n    labels=data_provider_isup_grade_count.index.get_level_values('isup_grade'),\n    labeldistance=0.5,\n    autopct='%.2f%%',\n    pctdistance=0.8,\n    textprops={'size': 'smaller'},\n    wedgeprops=dict(width=size, edgecolor='w'),\n    colors=inner_colors\n)\n\nax_isup[0].text(\n    0.5, 0.5,\n    'ISUP grade',\n    horizontalalignment='center',\n    verticalalignment='center',\n    transform = ax_isup[0].transAxes\n)\n\nax_isup[0].axis('equal')\nax_isup[0].set_title(\"ISUP grade distribution by data provider\")\nax_isup[0].legend(data_provider_count.index)\n\n# On the right axes we visualize \n# total ISUP data distribution through pie graph \ncmap_red = plt.get_cmap('Reds')\ncolors = cmap_red(np.linspace(0, 1, 6))\nax_isup[1].pie(\n    isup_grade_count,\n    labels = isup_grade_count.index,\n    autopct='%.2f%%',\n    textprops={'size': 'larger'},\n    wedgeprops=dict(edgecolor='w'),\n    colors=colors\n)\n\nax_isup[1].axis('equal')\nax_isup[1].set_title(\"Total ISUP grade distribution\")\n\n# Gleason score visualization\nfig_gleason, ax_gleason = plt.subplots(figsize=(16,8))\ndata_provider_gleason_score_count = distrib_by_data_provider.sum(level=['data_provider', 'gleason_score'])\n\nradboud = ax_gleason.bar(data_provider_gleason_score_count['radboud'].index.get_level_values('gleason_score'), data_provider_gleason_score_count['radboud'], color='tab:orange')\nkarolinska = ax_gleason.bar(data_provider_gleason_score_count['karolinska'].index.get_level_values('gleason_score'), data_provider_gleason_score_count['karolinska'], bottom=data_provider_gleason_score_count['radboud'], color='tab:blue')\n\n# Attach a text label above each bar, displaying its share in percentage\ntotal = data_provider_gleason_score_count.sum()\nfor rect_bottom, rect_upper in zip(radboud, karolinska):\n    height = rect_bottom.get_height() + rect_upper.get_height()\n    ax_gleason.annotate(\n        '%.2f%%' % (height * 100 / total),\n        xy=(rect_bottom.get_x() + rect_bottom.get_width() / 2, height),\n        xytext=(0, 3),  # 3 points vertical offset\n        textcoords=\"offset points\",\n        ha='center', va='bottom'\n    )\n\nax_gleason.set_title(\"Gleason score distribution by data provider\")\nax_gleason.legend((karolinska, radboud), ('karolinska', 'radboud'))\nax_gleason.grid(axis='y')\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}