{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom os import walk\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport cv2\nfrom tqdm.notebook import tqdm\nimport skimage.io\nfrom skimage.transform import resize, rescale\nimport time\nimport seaborn as sns\n\n%matplotlib inline\nimport mpld3\nmpld3.enable_notebook()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"radboud = 'radboud'\nkarolinska = 'karolinska'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_labels = pd.read_csv('../input/panda-eda-and-filtering-data/proper_test_cases.csv')\ntrain_labels_red = train_labels[train_labels['gleason_score'].isin(['0+0','3+3', '4+4', '5+5'])]\n# train_labels = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\n# train_label_red['score']\ntrain_labels_red.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print((train_labels_red['image_id']=='000920ad0b612851f8e01bcc880d9b3d').sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gleason_score_mapper = {'0+0': 0, # Healthy cells and tissues\n                        '3+3': 1, # Gleason grade 3\n                        '4+4': 2, # Gleason grade 4\n                        '5+5': 3, # Gleason grade 4\n                       }\ntrain_labels_red['score'] = train_labels_red.gleason_score.apply(lambda x: gleason_score_mapper[x])\ntrain_labels_red.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Karolinska: {len(train_labels_red[train_labels_red['data_provider']==karolinska])}\")\nprint(f\"\\nRadboud: {len(train_labels_red[train_labels_red['data_provider']==radboud])}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First trying for radboud because of smaller number of samples","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_provider = radboud\ntrain_labels_to_crop = train_labels_red[train_labels_red['data_provider']==data_provider]\ntrain_labels_to_crop.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img_dir = '/kaggle/input/prostate-cancer-grade-assessment/train_images/'\nprint(f\"Number of images in directory: {len(os.listdir(img_dir))}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask_dir = '/kaggle/input/prostate-cancer-grade-assessment/train_label_masks/'\nprint(f\"Number of images in directory: {len(os.listdir(mask_dir))}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# working with one image","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# img_id = train_labels.image_id[2]\n# mask_path = mask_dir + img_id + '_mask.tiff'\n# img_path = img_dir + img_id + '.tiff'\n# print(f\"img_id: {img_id}, path: {img_path}\")\n# print(f\"mask_path: {mask_path}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image = skimage.io.MultiImage(img_path)\n# mask = skimage.io.MultiImage(mask_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"Image sizes: \")\n# for frame in image:\n#     print(f\"frame.shape: {frame.shape}\")\n    \n# print(\"Mask sizes: \")\n# for frame in mask:\n#     print(f\"mask.shape: {frame.shape}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Consider the middle image, which is a medium size one\n# image = image[1]\n# mask = mask[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Display image\n# plt.imshow(image[1])\n\n# # Display mask\n# plt.figure()\n# plt.imshow(mask[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# path\nfor i in gleason_score_mapper.values(): \n    path = f\"{data_provider}_scores_{i}\"\n    print(path)\n    try:\n        os.mkdir(path)  \n    except OSError as error:  \n        print(error) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_slice(image, mask, x_index, y_index, target_size, display = 0):\n#   Generate indices\n    x_begin = x_index*target_size\n    x_end = (x_index+1)*target_size\n    y_begin = y_index*target_size\n    y_end = (y_index+1)*target_size\n#   Create crops\n    mask_slice = mask[slice(x_begin,x_end), slice(y_begin,y_end)]\n    image_slice = image[slice(x_begin,x_end), slice(y_begin,y_end), :]\n#   Display images\n    if display:\n#         print(f\"xindex: {x_begin}-{x_end}, yindex: {y_begin}-{y_end}\")\n        mask_slice_temp = mask_slice*30\n        plt.figure()\n        plt.subplot(1,2,1)\n        plt.imshow(mask_slice_temp, cmap = 'gray') # Mask has single channel, 2D\n        plt.title(\"Mask\")\n        plt.subplot(1,2,2)\n        plt.imshow(image_slice)\n        plt.title(\"Biopsy\")\n    return image_slice, mask_slice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# image_class:\n# 0: background\n# 1: healthy cells\n# 3: gleason_grade 3\n# 4: gleason_grade 4\n# 5: gleason_grade 5\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(image.mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a DataFrame object for storage\ndf = pd.DataFrame()\ndf.i = 0\ndf.j = 0\n## This grading is valid only for radboud\ndf[0] = 0 # White background\ndf[1] = 0 # Connecting tissue\ndf[2] = 0 # Healthy tissue\ndf[3] = 0 # Gleason grade 3\ndf[4] = 0 # Gleason grade 4\ndf[5] = 0 # Gleason grade 5\ndf['data_provider'] = \"\"\n\n# Storing mean values\nmodified_train_info = pd.DataFrame(columns = ['data_provider', 'mean', 'image_class'])\nmodified_train_info.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# This wont save any mask, just analysing threshold based on mean","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def analyse_image_mask_radboud(image_slice, mask_slice, label, i, j, df, modified_train_info, reference):\n#           Analyse image mask\n    counts = pd.Series(mask_slice.reshape(-1)).value_counts()\n    counts_np = np.zeros(6).astype('uint64')\n    for i in counts.keys():\n        counts_np[i] = counts[i]\n    counts['saved'] = 0\n#           out of 50176 pixels, selecting patches which has fewer than 20000 background pixels\n    tissue_cells = sum(counts_np[1:])\n    if tissue_cells > 20000: # To exclude white background\n        healthy_cells = sum(counts_np[1:3]) # Healthy + connecting tissue (1+2)\n#           Cancerous cells is excluding background and healthy tissue\n        cancerous_cells = sum(counts_np[3:]) # 3,4,5: cancerous tissue\n        if healthy_cells > 2*cancerous_cells:\n            label = 0 # If healthy cells is twice frequent than cencerous cells, consider as healthy slice\n#                 path = f\"{data_provider}_gleason_scores_{i}\"\n#         time.sleep(10 / 1000) # Sleep for 10ms to store results better\n#         cv2.imwrite(f\"{data_provider}_scores_{label}/{row['image_id']}_{i}_{j}.png\",image_slice)\n        counts['saved'] = 1\n#           Updating info in df, to design a classifier for distinguishing healthy and cencerous cells\n        modified_train_info = modified_train_info.append({'data_provider' : data_provider,\n                                    'mean': image_slice.mean(),\n                                    'image_class': label} , ignore_index=True)\n        counts['data_provider'] = data_provider\n        reference[label] += 1\n        df = df.append(counts, ignore_index = True)\n    else:\n        modified_train_info = modified_train_info.append({'data_provider' : data_provider,\n                                    'mean': image_slice.mean(),\n                                    'image_class': -1} , ignore_index=True)\n        \n    return df, modified_train_info\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_size = 224\nimg_size = 224*224\ndef gen_crops(row, df, modified_train_info, reference):\n#   image and mask path\n    mask_path = mask_dir + row['image_id'] + '_mask.tiff'\n    img_path = img_dir + row['image_id'] + '.tiff'\n    label = row['score']\n#   Load image and mask\n    image = skimage.io.MultiImage(img_path)[1] # Take the second image in tiff file which has medium size\n    mask = skimage.io.MultiImage(mask_path)[1] # Sly as image\n#     print(f\"img.shape: {image.shape}, mask.shape: {mask.shape}\")\n#   Check image and mask shapes\n    assert(image.shape == mask.shape)\n#   Create crops\n    mask = mask[:,:,0] # class information is present in first channel of mask\n#     print(f\"mask.shape: {mask.shape}\")\n#   Integer division of crops\n    n_x = mask.shape[0]//target_size\n    n_y = mask.shape[1]//target_size\n#     print(f\"n_x: {n_x}, n_y: {n_y}\")\n    for i in range(n_x):\n        for j in range(n_y):\n#             print(f\"i: {i}, j: {j}\")\n#           get slices for image and mask\n            image_slice, mask_slice = get_slice(image,mask,i,j,target_size)\n#           skip iteration when size is 0\n            if image_slice.size == 0 or mask_slice.size == 0:\n                continue\n            if data_provider == radboud:\n                df, modified_train_info = analyse_image_mask_radboud(image_slice, mask_slice, label, i,j, df, modified_train_info, reference)\n            else:\n                print(\"NOT YET IMPLEMENTED FOR KAROLINSKA\")\n                # To do for karolinska\n#             print(counts)\n    return df, modified_train_info\n#             mask_crop = mask\n#     counts = pd.Series(temp.reshape(-1)).value_counts()\n#     row.height = temp.shape[0]\n#     row.width = temp.shape[1]\n#     row.update(counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %time df, modified_train_info = gen_crops(train_labels.loc[2], df, modified_train_info)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(6,4))\nax = sns.countplot(x=\"score\", hue=\"data_provider\", data=train_labels_to_crop)\nplt.title(\"Score by Data Provider\", fontsize=14)\nplt.xlabel(\"Score\", fontsize=14)\nplt.ylabel(\"Count\", fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nax = sns.countplot(x=\"score\", hue=\"data_provider\", data=train_labels_to_crop.iloc[0:100])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test: 0-100","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"reference = np.zeros(len(gleason_score_mapper.values()))\ntemp_df = train_labels_to_crop.iloc[0:100]\nfor _, row in tqdm(temp_df.iterrows(),total=temp_df.shape[0]):\n    df, modified_train_info = gen_crops(row, df, modified_train_info, reference)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PDF of healthy vs cancerous","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"cells = modified_train_info[modified_train_info['image_class'].isin([0,1,2,3])]\nbackground = modified_train_info[modified_train_info['image_class'].isin([-1])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot( cells[\"mean\"] , color=\"skyblue\", label=\"Cells\")\nsns.distplot( background[\"mean\"] , color=\"red\", label=\"Background\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"kwargs = {'cumulative': True}\nsns.distplot(cells['mean'], hist_kws=kwargs, kde_kws=kwargs)\nsns.distplot(background['mean'], hist_kws=kwargs, kde_kws=kwargs)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Comparing distributions of each cell category","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class0 = modified_train_info[modified_train_info['image_class'].isin([0])]\nclass1 = modified_train_info[modified_train_info['image_class'].isin([1])]\nclass2 = modified_train_info[modified_train_info['image_class'].isin([2])]\nclass3 = modified_train_info[modified_train_info['image_class'].isin([3])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot( class0[\"mean\"] , label=\"Class-0\")\nsns.distplot( class1[\"mean\"] , label=\"Class-1\")\nsns.distplot( class2[\"mean\"] , label=\"Class-2\")\nsns.distplot( class3[\"mean\"] , label=\"Class-3\")\nsns.distplot(background['mean'], label='Background')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Observation:\nMax of cells(approx):\n* Class 0: 250\n* Class 1,2,3: 230\n\n> So, we can choose mean *[140, 240]*******, to be our region of intrest. Considering mean beyond this as background\nIn this higher classes such as 1,2,3 are somewhat far from 240. Thus we are not losing information about most important classes.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"di = {'Class 0': 39343 ,\n'Class 1': 3116,\n'Class 2': 3712,\n'Class 3': 585}\n\ntotal = sum(di.values())\nfor key in di:\n    di[key] = di[key]/total\nprint(di)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}