{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/arkajyotib/better-image-tiles-removing-white-spaces/edit\nimport os\nimport cv2\nimport PIL\nimport random\nimport openslide\nimport skimage.io\nimport matplotlib\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom IPython.display import Image, display\n\ndef compute_statistics(image):\n    \"\"\"\n    Args:\n        image                  numpy.array   multi-dimensional array of the form WxHxC\n    \n    Returns:\n        ratio_white_pixels     float         ratio of white pixels over total pixels in the image \n    \"\"\"\n    width, height = image.shape[0], image.shape[1]\n    num_pixels = width * height\n    \n    num_white_pixels = 0\n    \n    summed_matrix = np.sum(image, axis=-1)\n    # Note: A 3-channel white pixel has RGB (255, 255, 255)\n    num_white_pixels = np.count_nonzero(summed_matrix > 620)\n    ratio_white_pixels = num_white_pixels / num_pixels\n    \n    green_concentration = np.mean(image[1])\n    blue_concentration = np.mean(image[2])\n    \n    return ratio_white_pixels, green_concentration, blue_concentration\n\ndef select_k_best_regions(regions, k=20):\n    \"\"\"\n    Args:\n        regions               list           list of 2-component tuples first component the region, \n                                             second component the ratio of white pixels\n                                             \n        k                     int            number of regions to select\n    \"\"\"\n    regions = [x for x in regions if x[3] > 180 and x[4] > 180]\n    k_best_regions = sorted(regions, key=lambda tup: tup[2])[:k]\n    return k_best_regions\n\ndef get_k_best_regions(coordinates, image, window_size=512):\n    regions = {}\n    for i, tup in enumerate(coordinates):\n        x, y = tup[0], tup[1]\n        regions[i] = image[x : x+window_size, y : y+window_size, :]\n    \n    return regions\n\ndef generate_patches(image, window_size=200, stride=128, k=20):\n    \n    #image = skimage.io.MultiImage(slide_path)[-2]\n    image = np.array(image)\n    \n    max_width, max_height = image.shape[0], image.shape[1]\n    regions_container = []\n    i = 0\n    \n    while window_size + stride*i <= max_height:\n        j = 0\n        \n        while window_size + stride*j <= max_width:            \n            x_top_left_pixel = j * stride\n            y_top_left_pixel = i * stride\n            \n            patch = image[\n                x_top_left_pixel : x_top_left_pixel + window_size,\n                y_top_left_pixel : y_top_left_pixel + window_size,\n                :\n            ]\n            \n            ratio_white_pixels, green_concentration, blue_concentration = compute_statistics(patch)\n            \n            region_tuple = (x_top_left_pixel, y_top_left_pixel, ratio_white_pixels, green_concentration, blue_concentration)\n            regions_container.append(region_tuple)\n            \n            j += 1\n        \n        i += 1\n    \n    k_best_region_coordinates = select_k_best_regions(regions_container, k=k)\n    k_best_regions = get_k_best_regions(k_best_region_coordinates, image, window_size)\n    \n    return image, k_best_region_coordinates, k_best_regions\n\n\ndef display_images(regions, title):\n    fig, ax = plt.subplots(5, 4, figsize=(15, 15))\n    \n    for i, region in regions.items():\n        ax[i//4, i%4].imshow(region)\n    \n    fig.suptitle(title)\n    \ndef glue_to_one_picture(image_patches, window_size=200, k=16):\n    side = int(np.sqrt(k))\n    image = np.zeros((side*window_size, side*window_size, 3), dtype=np.int16)\n        \n    for i, patch in image_patches.items():\n        x = i // side\n        y = i % side\n        image[\n            x * window_size : (x+1) * window_size,\n            y * window_size : (y+1) * window_size,\n            :\n        ] = patch\n    \n    return image\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport skimage.io\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport gc\n\nfrom sklearn.metrics import cohen_kappa_score , confusion_matrix\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nfeatures = pd.read_csv(\"/kaggle/input/featuresv3/LBP_CCA_features_on_train_images_upsampled.csv\")\ntrain = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/test.csv\")\ntrain.head()\nsz = 128\nN=16\n\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [15, 5]\nimport numpy\nimport pandas as pd\nimport numpy as np\nimport cv2\nfrom skimage import morphology\nimport openslide\nimport time\nimport statistics\ndef otsu_filter(channel, gaussian_blur=True):\n    \"\"\"Otsu filter.\"\"\"\n    if gaussian_blur:\n        channel = cv2.GaussianBlur(channel, (5, 5), 0)\n    channel = channel.reshape((channel.shape[0], channel.shape[1]))\n\n    return cv2.threshold(\n        channel, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)[1]\n\ndef detect_tissue(wsi, sensitivity = 3000, downsampling_factor=64):\n    \n    \"\"\"\n    Find RoIs containing tissue in WSI.\n    Generate mask locating tissue in an WSI. Inspired by method used by\n    Wang et al. [1]_.\n    .. [1] Dayong Wang, Aditya Khosla, Rishab Gargeya, Humayun Irshad, Andrew\n    H. Beck, \"Deep Learning for Identifying Metastatic Breast Cancer\",\n    arXiv:1606.05718\n    \n    Parameters\n    ----------\n    wsi: OpenSlide/AnnotatedOpenSlide class instance\n        The whole-slide image (WSI) to detect tissue in.\n    downsampling_factor: int\n        The desired factor to downsample the image by, since full WSIs will\n        not fit in memory. The image's closest level downsample is found\n        and used.\n    sensitivity: int\n        The desired sensitivty of the model to detect tissue. The baseline is set\n        at 5000 and should be adjusted down to capture more potential issue and\n        adjusted up to be more agressive with trimming the slide.\n        \n    Returns\n    -------\n    -Binary mask as numpy 2D array, \n    -RGB slide image (in the used downsampling level, in case the user is visualizing output examples),\n    -Downsampling factor.\n    \"\"\"\n    #For timing\n    time_stamps = {}\n    time_stamps[\"start\"] = time.time()\n    \n    # Get a downsample of the whole slide image (to fit in memory)\n    downsampling_factor = min(\n        wsi.level_downsamples, key=lambda x: abs(x - downsampling_factor))\n    level = wsi.level_downsamples.index(downsampling_factor)\n\n    slide = wsi.read_region((0, 0), level, wsi.level_dimensions[level])\n    slide = np.array(slide)[:, :, :3]\n    time_stamps[\"1\"] = time.time()\n    # Convert from RGB to HSV color space\n    slide_hsv = cv2.cvtColor(slide, cv2.COLOR_BGR2HSV)\n    time_stamps[\"2\"] = time.time()\n    # Compute optimal threshold values in each channel using Otsu algorithm\n    _, saturation, _ = np.split(slide_hsv, 3, axis=2)\n\n    mask = otsu_filter(saturation, gaussian_blur=True)\n    time_stamps[\"3\"] = time.time()\n    # Make mask boolean\n    mask = mask != 0\n\n    mask = morphology.remove_small_holes(mask, area_threshold=sensitivity)\n    mask = morphology.remove_small_objects(mask, min_size=sensitivity)\n    time_stamps[\"4\"] = time.time()\n    mask = mask.astype(np.uint8)\n    mask_contours, tier = cv2.findContours(\n        mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n    time_stamps[\"5\"] = time.time()\n    time_stamps = {key:(value-time_stamps[\"start\"]) * 1000 for key,value in time_stamps.items()}\n    return mask_contours, tier, slide, downsampling_factor, time_stamps\n\ndef draw_tissue_polygons(mask, polygons, polygon_type,\n                              line_thickness=None):\n        \"\"\"\n        Plot as numpy array detected tissue.\n        Modeled WSIPRE github package\n        \n        Parameters\n        ----------\n        mask: numpy array \n            This is the original image represented as 0's for a starting canvas\n        polygons: numpy array \n            These are the identified tissue regions\n        polygon_type: str (\"line\" | \"area\")\n            The desired display type for the tissue regions\n        polygon_type: int\n            If the polygon_type==\"line\" then this parameter sets thickness\n\n        Returns\n        -------\n        Nunmpy array of tissue mask plotted\n        \"\"\"\n        \n        tissue_color = 1\n\n        for poly in polygons:\n            if polygon_type == 'line':\n                mask = cv2.polylines(\n                    mask, [poly], True, tissue_color, line_thickness)\n            elif polygon_type == 'area':\n                if line_thickness is not None:\n                    warnings.warn('\"line_thickness\" is only used if ' +\n                                  '\"polygon_type\" is \"line\".')\n\n                mask = cv2.fillPoly(mask, [poly], tissue_color)\n            else:\n                raise ValueError(\n                    'Accepted \"polygon_type\" values are \"line\" or \"area\".')\n\n        return mask\n\ndef tissue_cutout_old(tissue_slide, tissue_contours, slide):\n    #https://stackoverflow.com/a/28759496\n    crop_mask = np.zeros_like(tissue_slide) # Create mask where white is what we want, black otherwise\n    cv2.drawContours(crop_mask, tissue_contours, -1, 255, -1) # Draw filled contour in mask\n    tissue_only = np.zeros_like(slide) # Extract out the object and place into output image\n    tissue_only[crop_mask == 255] = slide[crop_mask == 255]\n    return tissue_only\n\ndef tissue_cutout(input_slide, tissue_contours):\n    \n    \"\"\"\n    Description\n    ----------\n    Set all parts of the in_slide to black except for those\n    within the provided tissue contours\n    Credit: https://stackoverflow.com/a/28759496\n    \n    Parameters\n    ----------\n    input_slide: numpy array\n            Slide to cut non-tissue backgound out\n    tissue_contours: numpy array \n            These are the identified tissue regions as cv2 contours\n            \n    Returns (1)\n    -------\n    - Numpy array of slide with non-tissue set to black\n    \"\"\"\n    \n    # Get intermediate slide\n    base_slide_mask = np.zeros(input_slide.shape[:2])\n    \n    # Create mask where white is what we want, black otherwise\n    crop_mask = np.zeros_like(base_slide_mask) \n    \n    # Draw filled contour in mask\n    cv2.drawContours(crop_mask, tissue_contours, -1, 255, -1) \n    \n    # Extract out the object and place into output image\n    tissue_only_slide = np.zeros_like(input_slide)  \n    tissue_only_slide[crop_mask == 255] = input_slide[crop_mask == 255]\n    \n    return tissue_only_slide\n\n\ndef getSubImage(rect, src_img):\n    width = int(rect[1][0])\n    height = int(rect[1][1])\n    box = cv2.boxPoints(rect)\n\n    src_pts = box.astype(\"float32\")\n    dst_pts = np.array([[0, height-1],\n                        [0, 0],\n                        [width-1, 0],\n                        [width-1, height-1]], dtype=\"float32\")\n    M = cv2.getPerspectiveTransform(src_pts, dst_pts)\n    warped = cv2.warpPerspective(src_img, M, (width, height))\n    return warped\n\n\n\n\ndef detect_and_crop(image_location:str, sensitivity:int=3000, \n                    downsample_rate:int=16, show_plots:str=\"simple\"):\n    \n    #For timing\n    time_stamps = {}\n    time_stamps[\"start\"] = time.time()\n    \n    #Open Slide\n    wsi = openslide.open_slide(image_location)\n    time_stamps[\"open\"] = time.time()\n    \n    #Get returns from detect_tissue()\n    (tissue_contours, tier, \n     downsampled_slide, \n     downsampling_factor,\n     time_stamps_detect) = detect_tissue(wsi,\n                                          sensitivity,downsample_rate)\n    time_stamps[\"tissue_detect\"] = time.time()\n    \n    #Get Tissue Only Slide\n    base_slide_mask = np.zeros(downsampled_slide.shape[:2])\n    tissue_slide = draw_tissue_polygons(base_slide_mask, tissue_contours,'line', 5)\n    base_size = get_disk_size(downsampled_slide)\n    tissue_only_slide = tissue_cutout(tissue_slide, tissue_contours, downsampled_slide)\n    time_stamps[\"tissue_trim\"] = time.time()\n    #Get minimal bounding rectangle for all tissue contours\n    if len(tissue_contours) == 0:\n        img_id = image_location.split(\"/\")[-1]\n        print(f\"No Tissue Contours - ID: {img_id}\")\n        #downsampling_factor = min(\n        #wsi.level_downsamples, key=lambda x: abs(x - downsample_rate))\n        #level = wsi.level_downsamples.index(downsampling_factor)\n        #slide = wsi.read_region((0, 0), level, wsi.level_dimensions[level])\n        #slide = np.array(slide)\n        return tissue_only_slide, 1.0, time_stamps\n    \n    all_bounding_rect = cv2.minAreaRect(np.concatenate(tissue_contours))\n    #Crop with getSubImage()\n    smart_bounding_crop = getSubImage(all_bounding_rect,tissue_only_slide)\n    time_stamps[\"crop\"] = time.time()\n    \n    #Crop empty space\n    #Remove by row\n    row_not_blank =  [row.all() for row in ~np.all(smart_bounding_crop == [255,0,0],\n                                                   axis=1)]\n    space_cut = smart_bounding_crop[row_not_blank,:]\n    #Remove by column\n    col_not_blank =  [col.all() for col in ~np.all(smart_bounding_crop == [255,0,0],\n                                                   axis=0)]\n    space_cut = space_cut[:,col_not_blank]\n    time_stamps[\"cut\"] = time.time()\n    \n    #Get size change\n    start_size = get_disk_size(downsampled_slide)\n    final_size = get_disk_size(space_cut)\n    pct_change = final_size / start_size\n    \n    if show_plots == \"simple\":\n        print(f\"Percent Reduced from Base Slide to Final: {(1- pct_change)*100:.2f}\")\n        plt.imshow(space_cut)\n        plt.show() \n    elif show_plots == \"verbose\":\n        #Set-up dictionary for plotting\n        verbose_plots = {}\n        #Add Base Slide to verbose print\n        verbose_plots[f\"Base Slide\\n{get_disk_size(downsampled_slide):.2f}MB\"] = downsampled_slide\n        #Add Tissue Only to verbose print\n        verbose_plots[f\"Tissue Detect\\nNo Change\"] = tissue_slide\n        #Add Bounding Boxes to verbose print\n        verbose_plots[f\"Bounding Boxes\\n{get_disk_size(smart_bounding_crop):.2f}MB\"] = smart_bounding_crop\n        #Add Space Cut Boxes to verbose print\n        verbose_plots[f\"Space Cut\\n{get_disk_size(space_cut):.2f}MB\"] = space_cut\n        print(f\"Percent Reduced from Base Slide to Final: {(1- pct_change)*100:.2f}\")\n        plt = plot_figures(verbose_plots, 1, len(verbose_plots))\n        plt.show()\n    elif show_plots == \"none\":\n        pass\n    else:\n        pass\n    time_stamps[\"all\"] = time.time()\n    time_stamps = {key:(value-time_stamps[\"start\"]) * 1000 for key,value in time_stamps.items()}\n    return space_cut, (1-pct_change), time_stamps\n\ndef get_disk_size(numpy_image):\n    \"\"\" Returns size in MB of numpy array on disk.\"\"\"\n    return (numpy_image.size * numpy_image.itemsize) / 1000000\n\ndef plot_figures(figures, nrows = 1, ncols=1):\n    #https://stackoverflow.com/a/11172032\n    \"\"\"Plot a dictionary of figures.\n\n    Parameters\n    ----------\n    figures : <title, figure> dictionary\n    ncols : number of columns of subplots wanted in the display\n    nrows : number of rows of subplots wanted in the figure\n    \"\"\"\n\n    fig, axeslist = plt.subplots(ncols=ncols, nrows=nrows)\n    for ind,title in enumerate(figures):\n        axeslist.ravel()[ind].imshow(figures[title], aspect='auto')\n        axeslist.ravel()[ind].set_title(title)\n    plt.tight_layout()\n    return plt\n\ndef remove_pen_marks(img):\n    \n    # Define elliptic kernel\n    kernel5x5 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))\n    \n    # Convert image to gray scale and mask out background\n    img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n    img_mask = np.where(img_gray > 45, 1, 0).astype(np.uint8)\n    \n    # Reshape red channel into 1-d array, aims to mask most of the pen marks\n    img_r = np.reshape(img[:, :, 0], (-1,))\n    img_r = img_r[np.where(img_r >0)[0]]\n    img_r_mask = (img[:, :, 0] > np.median(img_r)-50).astype(np.uint8)\n\n    # When computing the pen mark mask, some tissue gets masked as well,\n    # thus needing to erode the mask to get rid of it. Then some dilatation is \n    # applied to capture the \"edges\" of the \"gradient-like\"/non-uniform pen marks\n    img_r_mask = cv2.erode(img_r_mask, kernel5x5, iterations=3)\n    img_r_mask = cv2.dilate(img_r_mask, kernel5x5, iterations=5)\n    \n    # Combine the two masks\n    img_r_mask = img_r_mask\n    img_mask = img_mask * img_r_mask\n    \n    # There might still be some gaps/holes in the tissue, here's an attempt to \n    # fill those gaps/holes\n    img_mask = cv2.morphologyEx(img_mask, cv2.MORPH_CLOSE, kernel5x5, iterations=1)\n    img_mask = cv2.dilate(img_mask, kernel5x5, iterations=1)\n    contours, _ = cv2.findContours(img_mask, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)\n    for contour in contours:\n        cv2.drawContours(img_mask, [contour], 0, 1, -1)\n    \n    # Some final touch\n    img_mask = cv2.erode(img_mask, kernel5x5, iterations=3)\n    img_mask = cv2.dilate(img_mask, kernel5x5, iterations=1)\n    img_mask = cv2.erode(img_mask, kernel5x5, iterations=2)\n    \n    # Mask out pen marks from original image\n    img = img * img_mask[:, :, np.newaxis]\n    \n    return img\n\ndef hsv_level_changes(img, level1,level2,level3):\n    hsv = img.copy()\n    hsv = cv2.cvtColor(hsv, cv2.COLOR_RGB2HSV)\n    (h, s, v) = cv2.split(hsv)\n    s = s*level2\n    s = np.clip(s,0,255)\n    h = h*level1\n    h = np.clip(h,0,255)\n    v = v*level3\n    v = np.clip(v,0,255)\n    hsv = cv2.merge([h,s,v])\n    hsv_img = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)\n    shape = hsv_img.shape\n    pad0 = (sz-shape[0]%sz)%sz  #### horizontal padding\n    pad1 = (sz-shape[1]%sz)%sz  #### vartical padding\n    hsv_img = np.pad(hsv_img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=0)\n        \n    hsv_img = hsv_img.reshape(hsv_img.shape[0]//sz,sz,hsv_img.shape[1]//sz,sz,3)\n    hsv_img = hsv_img.transpose(0,2,1,3,4)\n    hsv_img = hsv_img.reshape(-1,sz,sz,3)\n    if len(hsv_img) < N:\n        hsv_img = np.pad(hsv_img,[[0,N-len(hsv_img)],[0,0],[0,0],[0,0]],constant_values=0)\n \n    idxs = np.argsort(hsv_img.reshape(hsv_img.shape[0],-1).sum(-1))[:N]\n    hsv_img = hsv_img[idxs]\n    hsv_img = (hsv_img/255.0).reshape(-1,3)\n        \n        \n    #r_g_b_sd_product.append(hsv_img.std(0)[0]*hsv_img.std(0)[1]*hsv_img.std(0)[2])\n        \n    return (hsv_img.std(0)[0]*hsv_img.std(0)[1]*hsv_img.std(0)[2]),(hsv_img.std(0)[0]+hsv_img.std(0)[1]+hsv_img.std(0)[2])\n\nfrom skimage import feature\nimport numpy as np\nclass LocalBinaryPatterns:\n\tdef __init__(self, numPoints, radius):\n\t\t# store the number of points and radius\n\t\tself.numPoints = numPoints\n\t\tself.radius = radius\n\tdef describe(self, image, eps=1e-7):\n\t\t# compute the Local Binary Pattern representation\n\t\t# of the image, and then use the LBP representation\n\t\t# to build the histogram of patterns\n\t\tlbp = feature.local_binary_pattern(image, self.numPoints,\n\t\t\tself.radius, method=\"uniform\")\n\t\t(hist, _) = np.histogram(lbp.ravel(),\n\t\t\tbins=np.arange(0, self.numPoints + 3),\n\t\t\trange=(0, self.numPoints + 2))\n\t\t# normalize the histogram\n\t\thist = hist.astype(\"float\")\n\t\thist /= (hist.sum() + eps)\n\t\t# return the histogram of Local Binary Patterns\n\t\treturn hist\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def detect_tissue_external(input_slide, sensitivity=3000):\n    \n    \"\"\"\n    Description\n    ----------\n    Find RoIs containing tissue in WSI and only return the external most.\n    Generate mask locating tissue in an WSI. Inspired by method used by\n    Wang et al. [1]_.\n    .. [1] Dayong Wang, Aditya Khosla, Rishab Gargeya, Humayun Irshad, Andrew\n    H. Beck, \"Deep Learning for Identifying Metastatic Breast Cancer\",\n    arXiv:1606.05718\n    Credit: Github-wsipre\n    \n    Parameters\n    ----------\n    input_slide: numpy array\n        Slide to detect tissue on.\n    sensitivity: int\n        The desired sensitivty of the model to detect tissue. The baseline is set\n        at 3000 and should be adjusted down to capture more potential issue and\n        adjusted up to be more agressive with trimming the slide.\n        \n    Returns (3)\n    -------\n    -Tissue binary mask as numpy 2D array, \n    -Tiers investigated,\n    -Time Stamps from running tissue detection pipeline\n    \"\"\"\n    \n    # For timing\n    time_stamps = {}\n    time_stamps[\"start\"] = time.time()\n\n    # Convert from RGB to HSV color space\n    slide_hsv = cv2.cvtColor(input_slide, cv2.COLOR_BGR2HSV)\n    time_stamps[\"re-color\"] = time.time()\n    # Compute optimal threshold values in each channel using Otsu algorithm\n    _, saturation, _ = np.split(slide_hsv, 3, axis=2)\n\n    mask = otsu_filter(saturation, gaussian_blur=True)\n    time_stamps[\"filter\"] = time.time()\n    # Make mask boolean\n    mask = mask != 0\n\n    mask = morphology.remove_small_holes(mask, area_threshold=sensitivity)\n    mask = morphology.remove_small_objects(mask, min_size=sensitivity)\n    time_stamps[\"morph\"] = time.time()\n    mask = mask.astype(np.uint8)\n    mask_contours, tier = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    time_stamps[\"contour\"] = time.time()\n    time_stamps = {\n        key: (value - time_stamps[\"start\"]) * 1000 for key, value in time_stamps.items()\n    }\n    return mask_contours, tier, time_stamps\n\ndef color_cut(in_slide, color = [255,255,255]):\n    \n    \"\"\"\n    Description\n    ----------\n    Take a input image and remove all rows or columns that\n    are only made of the input color [R,G,B]. The default color\n    to cut from image is white.\n    \n    Parameters\n    ----------\n    input_slide: numpy array \n        Slide to cut white cols/rows \n    color: list\n        List of [R,G,B] pixels to cut from the input slide\n    \n    Returns (1)\n    -------\n    - Numpy array of input_slide with white removed\n    \"\"\"\n    #Remove by row\n    row_not_blank = [row.all() for row in ~np.all(in_slide == color, axis=1)]\n    output_slide = in_slide[row_not_blank, :]\n    \n    #Remove by col\n    col_not_blank = [col.all() for col in ~np.all(output_slide == color, axis=0)]\n    output_slide = output_slide[:, col_not_blank]\n    return output_slide\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the necessary packages\nfrom skimage import feature\nimport numpy as np\nfrom scipy.stats import kurtosis\nfrom scipy.stats import skew\nfrom statistics import *\nclass LocalBinaryPatterns:\n\tdef __init__(self, numPoints, radius):\n\t\t# store the number of points and radius\n\t\tself.numPoints = numPoints\n\t\tself.radius = radius\n\tdef describe(self, image, eps=1e-7):\n\t\t# compute the Local Binary Pattern representation\n\t\t# of the image, and then use the LBP representation\n\t\t# to build the histogram of patterns\n\t\tlbp = feature.local_binary_pattern(image, self.numPoints,\n\t\t\tself.radius, method=\"uniform\")\n\t\t(hist, _) = np.histogram(lbp.ravel(),\n\t\t\tbins=np.arange(0, self.numPoints + 3),\n\t\t\trange=(0, self.numPoints + 2))\n\t\t# normalize the histogram\n\t\thist = hist.astype(\"float\")\n\t\thist /= (hist.sum() + eps)\n\t\thist[hist == 0] = 1\n\t\tentropy = -sum(hist*np.log(hist))\n\t\tskewness_val = skew(lbp.ravel())\n\t\tkurtosis_val = kurtosis(lbp.ravel())\n\t\tmean_val = mean(lbp.ravel())\n\t\tsd_val = np.std(lbp.ravel())\n\t\t# return the histogram of Local Binary Patterns\n\t\treturn entropy, skewness_val, kurtosis_val, mean_val, sd_val\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_detect_and_crop(image_location=\"\",sensitivity: int = 3000, downsample_lvl = -1,\n                        show_plots= \"simple\", out_lvl=-2, shape=(512,512)):\n    \"\"\"\n    Description\n    ----------\n    This method performs the pipeline as described in the notebook:\n    https://www.kaggle.com/dannellyz/panda-tissue-detect-scaling-bounding-boxes-fast\n    \n    Parameters\n    ----------\n    image_location:str\n        Location of the slide image to process\n    sensitivity:int\n        The desired sensitivty of the model to detect tissue. The baseline is set\n        at 3000 and should be adjusted down to capture more potential issue and\n        adjusted up to be more agressive with trimming the slide.\n    downsample_lvl: int\n        The level at which to downsample the slide. This can be referenced in\n        reverse order to access the lowest resoltuion items first.\n        [-1] = lowest resolution\n        [0] = highest resolution\n    show_plots: str (verbose|simple|none)\n        The types of plots to display:\n            - verbose - show all steps of process\n            - simple - show only last step\n            - none - show none of the plots\n    out_lvl: int\n        The level at which the final slide should sample at. This can be referenced in\n        reverse order to access the lowest resoltuion items first.\n        [-1] = lowest resolution\n        [0] = highest resolution\n    shape: touple\n        (height, width) of the desired produciton(prod) image\n        \n    Returns (4)\n    -------\n    - Numpy array of final produciton(prod) slide\n    - Percent memory reduciton from original slide\n    - Time stamps from stages of the pipeline\n    - Time stamps from the Tissue Detect pipeline\n    \"\"\"\n    # For timing\n    time_stamps = {}\n    time_stamps[\"start\"] = time.time()\n\n    # Open Small Slide\n    wsi_small = skimage.io.MultiImage(image_location)[downsample_lvl]\n    time_stamps[\"open_small\"] = time.time()\n\n    # Get returns from detect_tissue() ons mall image\n    (   tissue_contours,\n        tier,\n        time_stamps_detect,\n    ) = detect_tissue_external(wsi_small, sensitivity)\n    \n    base_slide_mask = np.zeros(wsi_small.shape[:2])\n    \n    # Get minimal bounding rectangle for all tissue contours\n    if len(tissue_contours) == 0:\n        tissue_slide = draw_tissue_polygons(base_slide_mask, tissue_contours,'line', 5)\n        base_size = get_disk_size(wsi_small)\n        tissue_only_slide = tissue_cutout_old(tissue_slide, tissue_contours, wsi_small)\n    \n        img_id = image_location.split(\"/\")[-1]\n        print(f\"No Tissue Contours - ID: {img_id}\")\n        return tissue_only_slide, 1.0, time_stamps,time_stamps_detect\n        #return None, 0, None, None\n    \n    # Open Big Slide\n    wsi_big = skimage.io.MultiImage(image_location)[out_lvl]\n    time_stamps[\"open_big\"] = time.time()\n    \n    #Get small boudning rect and scale\n    bounding_rect_small = cv2.minAreaRect(np.concatenate(tissue_contours))\n    #print(bounding_rect_small)\n    \n    # Scale Rectagle to larger image\n    scale = int(wsi_big.shape[0] / wsi_small.shape[0])\n    #print(scale)\n    scaled_rect = (\n        (bounding_rect_small[0][0] * scale, bounding_rect_small[0][1] * scale),\n        (bounding_rect_small[1][0] * scale, bounding_rect_small[1][1] * scale),\n        bounding_rect_small[2],\n    )\n    # Crop bigger image with getSubImage()\n    #print(scaled_rect)\n    \n    scaled_crop = getSubImage(scaled_rect, wsi_big)\n    time_stamps[\"scale_bounding\"] = time.time()\n    \n    #Cut out white\n    white_cut = color_cut(scaled_crop)\n    time_stamps[\"white_cut_big\"] = time.time()\n    \n    #Scale\n    scaled_slide = cv2.resize(white_cut, shape)\n    time_stamps[\"resize_big\"] = time.time()\n    \n    # Get returns from detect_tissue() on small image\n    (   tissue_contours_big,\n        tier_big,\n        time_stamps_detect,\n    ) = detect_tissue_external(scaled_slide, sensitivity)\n    prod_slide = tissue_cutout(scaled_slide, tissue_contours_big)\n    time_stamps[\"remove_tissue\"] = time.time()\n\n    # Get size change\n    base_size_high = get_disk_size(wsi_big)\n    final_size = get_disk_size(prod_slide)\n    pct_change = final_size / base_size_high\n    \n    if show_plots == \"simple\":\n        print(f\"Percent Reduced from Base Slide to Final: {(1- pct_change)*100:.2f}\")\n        plt.imshow(smart_bounding_crop)\n        plt.show()\n    elif show_plots == \"verbose\":\n        # Set-up dictionary for plotting\n        verbose_plots = {}\n        # Add Base Slide to verbose print\n        verbose_plots[f\"Smaller Slide\\n{get_disk_size(wsi_small):.2f}MB\"] = wsi_small\n        # Add Tissue Only to verbose print\n        verbose_plots[f\"Tissue Detect Low\\nNo Change\"] = wsi_big\n        # Add Larger Plot cut with bounding boxes\n        verbose_plots[f\"Larger scaled\\n{get_disk_size(scaled_crop):.2f}MB\"] = scaled_crop\n        # Add Bounding Boxes to verbose print\n        verbose_plots[\n            f\"Final Produciton\\n{get_disk_size(prod_slide):.2f}MB\"\n        ] = prod_slide\n        print(f\"Percent Reduced from Base Slide to Final: {(1- pct_change)*100:.2f}\")\n        plt = plot_figures(verbose_plots, 2, 2)\n    elif show_plots == \"none\":\n        pass\n    else:\n        pass\n    time_stamps = {\n        key: (value - time_stamps[\"start\"]) * 1000 for key, value in time_stamps.items()\n    }\n    return prod_slide, (1 - pct_change), time_stamps, time_stamps_detect\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import the necessary packages\nimport sys\nimport numpy as np\nimport skimage.color\nimport skimage.filters\nimport skimage.io\nimport skimage.viewer\nimport skimage.measure\nimport skimage.color\n\n\ndef connected_components(img, sigma, threshold):\n    # blur and grayscale before thresholding\n    blur = skimage.filters.gaussian(img, sigma=sigma)\n\n    # perform inverse binary thresholding\n    mask = blur < threshold\n\n    # Perform CCA on the mask\n    labeled_image = skimage.measure.label(mask, connectivity=2, return_num=True)\n    \n    #metric = (labeled_image[1])/(labeled_image[0].shape[0]*labeled_image[0].shape[1])\n    #metric = (labeled_image[0]==labeled_image[1]).sum()\n    metric = []\n    number_of_objects_area_L_10 = []\n    for j in list(range(0,labeled_image[1])):\n        metric.append((labeled_image[0]==j).sum())\n        \n    count = 0\n    count1 = 0\n    count2 = 0\n    count3 = 0\n    count4 = 0\n    count5 = 0\n    count6 = 0\n    count7 = 0\n    for k in metric:\n        if(k<5):\n            count = count+1\n        elif(k<10):\n            count1 = count1+1\n        elif(k<50):\n            count2 = count2+1\n        elif(k<100):\n            count3 = count3+1\n        elif(k<500):\n            count4 = count4+1\n        elif(k<1000):\n            count5 = count5+1\n        elif(k<5000):\n            count6 = count6+1\n        else:\n            count7 = count7+1\n            \n    metric.sort(reverse = True)\n    count8 = 0\n    if(count7==1):\n        if(len(metric)>1):\n            count8 = metric[1]\n        #print(\"ok\")\n    elif(count7==2):\n        if(len(metric)>2):\n            count8 = metric[2]\n        #print(\"ok2\")\n    number_of_objects_area_L_10 = count,count1,count2,count3,count4,count5,count6,count7,count8\n    #viewer = skimage.viewer.ImageViewer(labeled_image)\n    #viewer.show()\n    #return labeled_image[1]\n    return number_of_objects_area_L_10\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.svm import LinearSVC\n#from imutils import paths\nimport argparse\nimport cv2\nimport os\nfrom skimage import color\n\ndef feature_engineering_lbp_cca(data = train , dir_name = \"train_images\"):\n    desc = LocalBinaryPatterns(8, 1)\n    LBP_hist_entropy = []\n    number_of_objects_10_1 = []\n    number_of_objects_50_1 = []\n    number_of_objects_200_1 = []\n    number_of_objects_10_11 = []\n    number_of_objects_50_11 = []\n    number_of_objects_200_11 = []\n    number_of_objects_10_21 = []\n    number_of_objects_50_21 = []\n    number_of_objects_200_21 = []\n    number_of_objects_10_31 = []\n    number_of_objects_50_31 = []\n    number_of_objects_200_31 = []\n    number_of_objects_10_41 = []\n    number_of_objects_50_41 = []\n    number_of_objects_200_41 = []\n    number_of_objects_10_51 = []\n    number_of_objects_50_51 = []\n    number_of_objects_200_51 = []\n    number_of_objects_10_61 = []\n    number_of_objects_50_61 = []\n    number_of_objects_200_61 = []\n    number_of_objects_10_71 = []\n    number_of_objects_50_71 = []\n    number_of_objects_200_71 = []\n    number_of_white_dots = []\n    # credits to Rohit Singh\n    pen_marked_images = [\n        'fd6fe1a3985b17d067f2cb4d5bc1e6e1',\n        'ebb6a080d72e09f6481721ef9f88c472',\n        'ebb6d5ca45942536f78beb451ee43cc4',\n        'ea9d52d65500acc9b9d89eb6b82cdcdf',\n        'e726a8eac36c3d91c3c4f9edba8ba713',\n        'e90abe191f61b6fed6d6781c8305fe4b',\n        'fd0bb45eba479a7f7d953f41d574bf9f',\n        'ff10f937c3d52eff6ad4dd733f2bc3ac',\n        'feee2e895355a921f2b75b54debad328',\n        'feac91652a1c5accff08217d19116f1c',\n        'fb01a0a69517bb47d7f4699b6217f69d',\n        'f00ec753b5618cfb30519db0947fe724',\n        'e9a4f528b33479412ee019e155e1a197',\n        'f062f6c1128e0e9d51a76747d9018849',\n        'f39bf22d9a2f313425ee201932bac91a',\n    ]\n\n    for i in tqdm(data['image_id'].values):\n        #img = skimage.io.MultiImage(os.path.join(f\"/kaggle/input/prostate-cancer-grade-assessment/{dir_name}\"+\"/\"+str(i)+\".tiff\"))[2]\n        #Set up example slide\n        slide_dir = \"../input/prostate-cancer-grade-assessment/\"\n        #annotation_dir = \"../input/prostate-cancer-grade-assessment/train_label_masks/\"\n        #example_id = \"0032bfa835ce0f43a92ae0bbab6871cb\"\n        example_slide = f\"{slide_dir}{dir_name}\"+\"/\"+str(i)+\".tiff\"\n        #img, pct_change, time_stamps = detect_and_crop(image_location=example_slide, downsample_rate=4, show_plots=\"none\")\n        img, pct_change, time_stamps, detect_time_med = new_detect_and_crop(image_location=example_slide,show_plots=\"none\")\n        #url = data_dir + img + '.tiff'\n        #image, best_coordinates, best_regions = generate_patches(img, window_size=128, stride=64, k=16)\n        #img = glue_to_one_picture(best_regions, window_size=128, k=16)\n    \n        \n        if i in pen_marked_images:\n            img = remove_pen_marks(img)\n            #plt.imshow(img)\n        \n        shape = img.shape\n        pad0 = (sz-shape[0]%sz)%sz  #### horizontal padding\n        pad1 = (sz-shape[1]%sz)%sz  #### vartical padding\n        img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=0)\n        number_of_white_dots.append((img>250).sum()/(img.shape[0]*img.shape[1]))\n        #print(number_of_objects[1])\n        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n        LBP_hist_entropy.append(desc.describe(gray))\n        #hist = desc.describe(gray)\n        #print(hist[0])\n        #print(hist)\n        #// Set any entries in the PDF that are 0 to 1 so log calculation works\n        \n\n        # label and data lists\n        #LBP_hist_entropy.append(desc.describe(gray))\n        \n        img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n        img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n        if len(img) < N:\n            img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=0)\n        idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n        img = img[idxs]\n        \n        #objects_in_box = 0\n        #for j in list(range(0,16)):\n        #    objects_in_box = objects_in_box + connected_components(img[j], sigma=1, threshold = 0.01)\n        img1 = cv2.hconcat([cv2.vconcat([img[0], img[1], img[2], img[3]]), \n                        cv2.vconcat([img[4], img[5], img[6], img[7]]), \n                        cv2.vconcat([img[8], img[9], img[10], img[11]]), \n                        cv2.vconcat([img[12], img[13], img[14], img[15]])])\n        img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)\n        \n        #gray = color.rgb2gray(img1)\n        number_of_objects_10_1.append(connected_components(img1, sigma=0.1,threshold =0.01))\n        number_of_objects_50_1.append(connected_components(img1, sigma=0.5,threshold =0.01))\n        number_of_objects_200_1.append(connected_components(img1, sigma=2,threshold =0.01))\n        number_of_objects_10_11.append(connected_components(img1, sigma=0.1,threshold =0.11))\n        number_of_objects_50_11.append(connected_components(img1, sigma=0.5,threshold =0.11))\n        number_of_objects_200_11.append(connected_components(img1, sigma=2,threshold =0.11))\n        number_of_objects_10_21.append(connected_components(img1, sigma=0.1,threshold =0.21))\n        number_of_objects_50_21.append(connected_components(img1, sigma=0.5,threshold =0.21))\n        number_of_objects_200_21.append(connected_components(img1, sigma=2,threshold =0.21))\n        number_of_objects_10_31.append(connected_components(img1, sigma=0.1,threshold =0.31))\n        number_of_objects_50_31.append(connected_components(img1, sigma=0.5,threshold =0.31))\n        number_of_objects_200_31.append(connected_components(img1, sigma=2,threshold =0.31))\n        number_of_objects_10_41.append(connected_components(img1, sigma=0.1,threshold =0.41))\n        number_of_objects_50_41.append(connected_components(img1, sigma=0.5,threshold =0.41))\n        number_of_objects_200_41.append(connected_components(img1, sigma=2,threshold =0.41))\n        number_of_objects_10_51.append(connected_components(img1, sigma=0.1,threshold =0.51))\n        number_of_objects_50_51.append(connected_components(img1, sigma=0.5,threshold =0.51))\n        number_of_objects_200_51.append(connected_components(img1, sigma=2,threshold =0.51))\n        number_of_objects_10_61.append(connected_components(img1, sigma=0.1,threshold =0.61))\n        number_of_objects_50_61.append(connected_components(img1, sigma=0.5,threshold =0.61))\n        number_of_objects_200_61.append(connected_components(img1, sigma=2,threshold =0.61))\n        number_of_objects_10_71.append(connected_components(img1, sigma=0.1,threshold =0.71))\n        number_of_objects_50_71.append(connected_components(img1, sigma=0.5,threshold =0.71))\n        number_of_objects_200_71.append(connected_components(img1, sigma=2,threshold =0.71))\n        \n        #print(LBP_hist_entropy)\n        del img\n        gc.collect()\n    \n    data['number_of_white_dots'] = number_of_white_dots\n    data['LBP_hist_entropy'] = [a_tuple[0] for a_tuple in LBP_hist_entropy]\n    data['LBP_hist_skewness'] = [a_tuple[1] for a_tuple in LBP_hist_entropy]\n    data['LBP_hist_kurtosis'] = [a_tuple[2] for a_tuple in LBP_hist_entropy]\n    data['LBP_hist_mean'] = [a_tuple[3] for a_tuple in LBP_hist_entropy]\n    data['LBP_hist_sd'] = [a_tuple[4] for a_tuple in LBP_hist_entropy]\n    data['number_of_objects_10_1_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_5']  = [a_tuple[0] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_10']  = [a_tuple[1] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_50']  = [a_tuple[2] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_100']  = [a_tuple[3] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_500']  = [a_tuple[4] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_1000']  = [a_tuple[5] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_ALT_5000']  = [a_tuple[6] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_biggest_area']  = [a_tuple[7] for a_tuple in number_of_objects_200_71]\n    data['number_of_objects_10_1_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_1]\n    data['number_of_objects_50_1_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_1]\n    data['number_of_objects_200_1_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_1]\n    data['number_of_objects_10_11_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_11]\n    data['number_of_objects_50_11_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_11]\n    data['number_of_objects_200_11_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_11]\n    data['number_of_objects_10_21_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_21]\n    data['number_of_objects_50_21_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_21]\n    data['number_of_objects_200_21_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_21]\n    data['number_of_objects_10_31_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_31]\n    data['number_of_objects_50_31_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_31]\n    data['number_of_objects_200_31_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_31]\n    data['number_of_objects_10_41_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_41]\n    data['number_of_objects_50_41_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_41]\n    data['number_of_objects_200_41_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_41]\n    data['number_of_objects_10_51_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_51]\n    data['number_of_objects_50_51_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_51]\n    data['number_of_objects_200_51_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_51]\n    data['number_of_objects_10_61_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_61]\n    data['number_of_objects_50_61_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_61]\n    data['number_of_objects_200_61_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_61]\n    data['number_of_objects_10_71_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_10_71]\n    data['number_of_objects_50_71_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_50_71]\n    data['number_of_objects_200_71_area_of_FSO']  = [a_tuple[8] for a_tuple in number_of_objects_200_71]\n    data['data_prov_ind'] = np.where(data['data_provider'] == \"radboud\" , 1 , 0)\n        \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_v1 = train[train.image_id!='033e39459301e97e457232780a314ab7']\ntrain_v1 = train_v1[train_v1.image_id!='0b6e34bf65ee0810c1a4bf702b667c88']\ntrain_v1 = train_v1[train_v1.image_id!='3385a0f7f4f3e7e7b380325582b115c9']\ntrain_v1 = train_v1[train_v1.image_id!='3790f55cad63053e956fb73027179707']\ntrain_v1 = train_v1[train_v1.image_id!='5204134e82ce75b1109cc1913d81abc6']\ntrain_v1 = train_v1[train_v1.image_id!='a08e24cff451d628df797efc4343e13c']\n\ntrain_v1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_v2 = feature_engineering_lbp_cca(data = train_v1[0:1] , dir_name = \"train_images\")\ntrain_v2 = features\ntrain_v2.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = ['number_of_white_dots',\n'LBP_hist_entropy',\n'LBP_hist_skewness',\n'LBP_hist_kurtosis',\n'LBP_hist_mean',\n'LBP_hist_sd',\n'number_of_objects_10_1_ALT_5',\n'number_of_objects_50_1_ALT_5',\n'number_of_objects_200_1_ALT_5',\n'number_of_objects_10_11_ALT_5',\n'number_of_objects_50_11_ALT_5',\n'number_of_objects_200_11_ALT_5',\n'number_of_objects_10_21_ALT_5',\n'number_of_objects_50_21_ALT_5',\n'number_of_objects_200_21_ALT_5',\n'number_of_objects_10_31_ALT_5',\n'number_of_objects_50_31_ALT_5',\n'number_of_objects_200_31_ALT_5',\n'number_of_objects_10_41_ALT_5',\n'number_of_objects_50_41_ALT_5',\n'number_of_objects_200_41_ALT_5',\n'number_of_objects_10_51_ALT_5',\n'number_of_objects_50_51_ALT_5',\n'number_of_objects_200_51_ALT_5',\n'number_of_objects_10_61_ALT_5',\n'number_of_objects_50_61_ALT_5',\n'number_of_objects_200_61_ALT_5',\n'number_of_objects_10_71_ALT_5',\n'number_of_objects_50_71_ALT_5',\n'number_of_objects_200_71_ALT_5',\n'number_of_objects_10_1_ALT_10',\n'number_of_objects_50_1_ALT_10',\n'number_of_objects_200_1_ALT_10',\n'number_of_objects_10_11_ALT_10',\n'number_of_objects_50_11_ALT_10',\n'number_of_objects_200_11_ALT_10',\n'number_of_objects_10_21_ALT_10',\n'number_of_objects_50_21_ALT_10',\n'number_of_objects_200_21_ALT_10',\n'number_of_objects_10_31_ALT_10',\n'number_of_objects_50_31_ALT_10',\n'number_of_objects_200_31_ALT_10',\n'number_of_objects_10_41_ALT_10',\n'number_of_objects_50_41_ALT_10',\n'number_of_objects_200_41_ALT_10',\n'number_of_objects_10_51_ALT_10',\n'number_of_objects_50_51_ALT_10',\n'number_of_objects_200_51_ALT_10',\n'number_of_objects_10_61_ALT_10',\n'number_of_objects_50_61_ALT_10',\n'number_of_objects_200_61_ALT_10',\n'number_of_objects_10_71_ALT_10',\n'number_of_objects_50_71_ALT_10',\n'number_of_objects_200_71_ALT_10',\n'number_of_objects_10_1_ALT_50',\n'number_of_objects_50_1_ALT_50',\n'number_of_objects_200_1_ALT_50',\n'number_of_objects_10_11_ALT_50',\n'number_of_objects_50_11_ALT_50',\n'number_of_objects_200_11_ALT_50',\n'number_of_objects_10_21_ALT_50',\n'number_of_objects_50_21_ALT_50',\n'number_of_objects_200_21_ALT_50',\n'number_of_objects_10_31_ALT_50',\n'number_of_objects_50_31_ALT_50',\n'number_of_objects_200_31_ALT_50',\n'number_of_objects_10_41_ALT_50',\n'number_of_objects_50_41_ALT_50',\n'number_of_objects_200_41_ALT_50',\n'number_of_objects_10_51_ALT_50',\n'number_of_objects_50_51_ALT_50',\n'number_of_objects_200_51_ALT_50',\n'number_of_objects_10_61_ALT_50',\n'number_of_objects_50_61_ALT_50',\n'number_of_objects_200_61_ALT_50',\n'number_of_objects_10_71_ALT_50',\n'number_of_objects_50_71_ALT_50',\n'number_of_objects_200_71_ALT_50',\n'number_of_objects_10_1_ALT_100',\n'number_of_objects_50_1_ALT_100',\n'number_of_objects_200_1_ALT_100',\n'number_of_objects_10_11_ALT_100',\n'number_of_objects_50_11_ALT_100',\n'number_of_objects_200_11_ALT_100',\n'number_of_objects_10_21_ALT_100',\n'number_of_objects_50_21_ALT_100',\n'number_of_objects_200_21_ALT_100',\n'number_of_objects_10_31_ALT_100',\n'number_of_objects_50_31_ALT_100',\n'number_of_objects_200_31_ALT_100',\n'number_of_objects_10_41_ALT_100',\n'number_of_objects_50_41_ALT_100',\n'number_of_objects_200_41_ALT_100',\n'number_of_objects_10_51_ALT_100',\n'number_of_objects_50_51_ALT_100',\n'number_of_objects_200_51_ALT_100',\n'number_of_objects_10_61_ALT_100',\n'number_of_objects_50_61_ALT_100',\n'number_of_objects_200_61_ALT_100',\n'number_of_objects_10_71_ALT_100',\n'number_of_objects_50_71_ALT_100',\n'number_of_objects_200_71_ALT_100',\n'number_of_objects_10_1_ALT_500',\n'number_of_objects_50_1_ALT_500',\n'number_of_objects_200_1_ALT_500',\n'number_of_objects_10_11_ALT_500',\n'number_of_objects_50_11_ALT_500',\n'number_of_objects_200_11_ALT_500',\n'number_of_objects_10_21_ALT_500',\n'number_of_objects_50_21_ALT_500',\n'number_of_objects_200_21_ALT_500',\n'number_of_objects_10_31_ALT_500',\n'number_of_objects_50_31_ALT_500',\n'number_of_objects_200_31_ALT_500',\n'number_of_objects_10_41_ALT_500',\n'number_of_objects_50_41_ALT_500',\n'number_of_objects_200_41_ALT_500',\n'number_of_objects_10_51_ALT_500',\n'number_of_objects_50_51_ALT_500',\n'number_of_objects_200_51_ALT_500',\n'number_of_objects_10_61_ALT_500',\n'number_of_objects_50_61_ALT_500',\n'number_of_objects_200_61_ALT_500',\n'number_of_objects_10_71_ALT_500',\n'number_of_objects_50_71_ALT_500',\n'number_of_objects_200_71_ALT_500',\n'number_of_objects_10_1_ALT_1000',\n'number_of_objects_50_1_ALT_1000',\n'number_of_objects_200_1_ALT_1000',\n'number_of_objects_10_11_ALT_1000',\n'number_of_objects_50_11_ALT_1000',\n'number_of_objects_200_11_ALT_1000',\n'number_of_objects_10_21_ALT_1000',\n'number_of_objects_50_21_ALT_1000',\n'number_of_objects_200_21_ALT_1000',\n'number_of_objects_10_31_ALT_1000',\n'number_of_objects_50_31_ALT_1000',\n'number_of_objects_200_31_ALT_1000',\n'number_of_objects_10_41_ALT_1000',\n'number_of_objects_50_41_ALT_1000',\n'number_of_objects_200_41_ALT_1000',\n'number_of_objects_10_51_ALT_1000',\n'number_of_objects_50_51_ALT_1000',\n'number_of_objects_200_51_ALT_1000',\n'number_of_objects_10_61_ALT_1000',\n'number_of_objects_50_61_ALT_1000',\n'number_of_objects_200_61_ALT_1000',\n'number_of_objects_10_71_ALT_1000',\n'number_of_objects_50_71_ALT_1000',\n'number_of_objects_200_71_ALT_1000',\n'number_of_objects_10_1_ALT_5000',\n'number_of_objects_50_1_ALT_5000',\n'number_of_objects_200_1_ALT_5000',\n'number_of_objects_10_11_ALT_5000',\n'number_of_objects_50_11_ALT_5000',\n'number_of_objects_200_11_ALT_5000',\n'number_of_objects_10_21_ALT_5000',\n'number_of_objects_50_21_ALT_5000',\n'number_of_objects_200_21_ALT_5000',\n'number_of_objects_10_31_ALT_5000',\n'number_of_objects_50_31_ALT_5000',\n'number_of_objects_200_31_ALT_5000',\n'number_of_objects_10_41_ALT_5000',\n'number_of_objects_50_41_ALT_5000',\n'number_of_objects_200_41_ALT_5000',\n'number_of_objects_10_51_ALT_5000',\n'number_of_objects_50_51_ALT_5000',\n'number_of_objects_200_51_ALT_5000',\n'number_of_objects_10_61_ALT_5000',\n'number_of_objects_50_61_ALT_5000',\n'number_of_objects_200_61_ALT_5000',\n'number_of_objects_10_71_ALT_5000',\n'number_of_objects_50_71_ALT_5000',\n'number_of_objects_200_71_ALT_5000',\n'number_of_objects_10_1_biggest_area',\n'number_of_objects_50_1_biggest_area',\n'number_of_objects_200_1_biggest_area',\n'number_of_objects_10_11_biggest_area',\n'number_of_objects_50_11_biggest_area',\n'number_of_objects_200_11_biggest_area',\n'number_of_objects_10_21_biggest_area',\n'number_of_objects_50_21_biggest_area',\n'number_of_objects_200_21_biggest_area',\n'number_of_objects_10_31_biggest_area',\n'number_of_objects_50_31_biggest_area',\n'number_of_objects_200_31_biggest_area',\n'number_of_objects_10_41_biggest_area',\n'number_of_objects_50_41_biggest_area',\n'number_of_objects_200_41_biggest_area',\n'number_of_objects_10_51_biggest_area',\n'number_of_objects_50_51_biggest_area',\n'number_of_objects_200_51_biggest_area',\n'number_of_objects_10_61_biggest_area',\n'number_of_objects_50_61_biggest_area',\n'number_of_objects_200_61_biggest_area',\n'number_of_objects_10_71_biggest_area',\n'number_of_objects_50_71_biggest_area',\n'number_of_objects_200_71_biggest_area',\n'number_of_objects_10_1_area_of_FSO',\n'number_of_objects_50_1_area_of_FSO',\n'number_of_objects_200_1_area_of_FSO',\n'number_of_objects_10_11_area_of_FSO',\n'number_of_objects_50_11_area_of_FSO',\n'number_of_objects_200_11_area_of_FSO',\n'number_of_objects_10_21_area_of_FSO',\n'number_of_objects_50_21_area_of_FSO',\n'number_of_objects_200_21_area_of_FSO',\n'number_of_objects_10_31_area_of_FSO',\n'number_of_objects_50_31_area_of_FSO',\n'number_of_objects_200_31_area_of_FSO',\n'number_of_objects_10_41_area_of_FSO',\n'number_of_objects_50_41_area_of_FSO',\n'number_of_objects_200_41_area_of_FSO',\n'number_of_objects_10_51_area_of_FSO',\n'number_of_objects_50_51_area_of_FSO',\n'number_of_objects_200_51_area_of_FSO',\n'number_of_objects_10_61_area_of_FSO',\n'number_of_objects_50_61_area_of_FSO',\n'number_of_objects_200_61_area_of_FSO',\n'number_of_objects_10_71_area_of_FSO',\n'number_of_objects_50_71_area_of_FSO',\n'number_of_objects_200_71_area_of_FSO',\n'data_prov_ind']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#features = ['LBP_hist_sd', 'LBP_hist_skewness', 'number_of_white_dots', 'LBP_hist_entropy',  'LBP_hist_kurtosis' , 'LBP_hist_mean','number_of_objects_50_71_area_of_FSO','data_prov_ind']\n#features = ['LBP_hist_sd', 'LBP_hist_skewness', 'number_of_white_dots', 'LBP_hist_entropy',  'LBP_hist_kurtosis' , 'LBP_hist_mean','data_prov_ind','number_of_objects_10_51_ALT_5','number_of_objects_10_41_ALT_5','number_of_objects_50_71_area_of_FSO','number_of_objects_200_61_area_of_FSO',]\n#features = ['data_prov_ind','number_of_objects_10_51_ALT_5','number_of_objects_10_41_ALT_5','number_of_objects_50_71_area_of_FSO','number_of_objects_200_61_area_of_FSO']\n #'number_of_objects_10_41_ALT_5', 'number_of_objects_50_71_area_of_FSO',  'number_of_objects_200_61_area_of_FSO','number_of_objects_10_51_ALT_5','data_prov_ind'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')\n\ndef QWK(preds, dtrain):\n    labels = dtrain.get_label()\n    preds = np.rint(preds)\n    score = quadratic_weighted_kappa(preds, labels)\n    return (\"QWK\", score, True)\n\ny = train_v2[\"isup_grade\"]\ntrain = train_v2[features]\nX_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.2, random_state=0)\n\ntrain_dataset = lgb.Dataset(X_train, y_train)\nvalid_dataset = lgb.Dataset(X_test, y_test)\n\nparams = {\n            \"objective\": 'regression',\n            \"metric\": 'rmse',\n            \"seed\": 0,\n            \"learning_rate\": 0.05,\n            \"boosting\": \"gbdt\",\n            \"num_leaves\": 31,\n            \"min_data_in_leaf\": 300,\n            \"max_depth\": -1,\n            }\n        \nmodel = lgb.train(\n            params=params,\n            num_boost_round=15000,\n            early_stopping_rounds=200,\n            train_set=train_dataset,\n            valid_sets=[train_dataset, valid_dataset],\n            verbose_eval=100,\n            feval=QWK)\n\n\npreds = model.predict(X_test, num_iteration=model.best_iteration)\npreds = np.rint(preds)\npreds = np.clip(preds, 0 , 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimportance = model.feature_importance()\nprint(importance)\nprint(features)\nfold_importance_df = pd.DataFrame()\nfold_importance_df[\"feature\"] = features\nfold_importance_df[\"importance\"] =importance\nfold_importance_df.head(20)\nprint(fold_importance_df.shape)\nall_features = fold_importance_df.sort_values(by=\"importance\", ascending=False)\nall_features.reset_index(inplace=True)\nfeatures = list(all_features[0:100]['feature'])\n#print(features)\nall_features.head(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')\n\ndef QWK(preds, dtrain):\n    labels = dtrain.get_label()\n    preds = np.rint(preds)\n    score = quadratic_weighted_kappa(preds, labels)\n    return (\"QWK\", score, True)\n\ny = train_v2[\"isup_grade\"]\ntrain = train_v2[features]\nX_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.2, random_state=0)\n\ntrain_dataset = lgb.Dataset(X_train, y_train)\nvalid_dataset = lgb.Dataset(X_test, y_test)\n\nparams = {\n            \"objective\": 'regression',\n            \"metric\": 'rmse',\n            \"seed\": 0,\n            \"learning_rate\": 0.01,\n            \"boosting\": \"gbdt\",\n            \"num_leaves\": 31,\n            \"min_data_in_leaf\": 200,\n            \"max_depth\": -1,\n            }\n        \nmodel = lgb.train(\n            params=params,\n            num_boost_round=15000,\n            early_stopping_rounds=100,\n            train_set=train_dataset,\n            valid_sets=[train_dataset, valid_dataset],\n            verbose_eval=100,\n            feval=QWK)\n\n\npreds = model.predict(X_test, num_iteration=model.best_iteration)\npreds = np.rint(preds)\npreds = np.clip(preds, 0 , 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimportance = model.feature_importance()\nprint(importance)\nprint(features)\nfold_importance_df = pd.DataFrame()\nfold_importance_df[\"feature\"] = features\nfold_importance_df[\"importance\"] =importance\nfold_importance_df.head(50)\nprint(fold_importance_df.shape)\nall_features = fold_importance_df.sort_values(by=\"importance\", ascending=False)\nall_features.reset_index(inplace=True)\nfeatures = list(all_features[0:50]['feature'])\n#print(features)\nall_features.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def quadratic_weighted_kappa(y_hat, y):\n    return cohen_kappa_score(y_hat, y, weights='quadratic')\n\ndef QWK(preds, dtrain):\n    labels = dtrain.get_label()\n    preds = np.rint(preds)\n    score = quadratic_weighted_kappa(preds, labels)\n    return (\"QWK\", score, True)\n\ny = train_v2[\"isup_grade\"]\ntrain = train_v2[features]\nX_train, X_test, y_train, y_test = train_test_split(train, y, test_size=0.2, random_state=0)\n\ntrain_dataset = lgb.Dataset(X_train, y_train)\nvalid_dataset = lgb.Dataset(X_test, y_test)\n\nparams = {\n            \"objective\": 'regression',\n            \"metric\": 'rmse',\n            \"seed\": 0,\n            \"learning_rate\": 0.01,\n            \"boosting\": \"gbdt\",\n            \"num_leaves\": 31,\n            \"min_data_in_leaf\": 300,\n            \"max_depth\": 6,\n            }\n        \nmodel = lgb.train(\n            params=params,\n            num_boost_round=15000,\n            early_stopping_rounds=100,\n            train_set=train_dataset,\n            valid_sets=[train_dataset, valid_dataset],\n            verbose_eval=100,\n            feval=QWK)\n\n\npreds = model.predict(X_test, num_iteration=model.best_iteration)\npreds = np.rint(preds)\npreds = np.clip(preds, 0 , 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.best_iteration","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"our validation score is\" , quadratic_weighted_kappa(preds, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(confusion_matrix(preds,y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import print_function  \nimport sys\n\nlocal_vars = list(locals().items())\nfor var, obj in local_vars:\n    if not var.startswith('_'):\n        print(var, sys.getsizeof(obj))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train ,X_test ,y_train ,y_test \ngc.collect()\n\ndef inference (da = test , dir_path = \"test_images\"):\n    if os.path.exists(f'../input/prostate-cancer-grade-assessment/{dir_path}'):\n        print('run inference')\n        \n        preds = model.predict(da[features], num_iteration=model.best_iteration)\n        preds = np.rint(preds)\n        preds = np.clip(preds, 0 ,5)\n        da['isup_grade'] = preds.astype(int)\n        cols = [\"image_id\" , \"isup_grade\"]\n        da = da[cols]\n        \n    return da\n\ntrain = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/train.csv\")\n\nsub = inference(da = feature_engineering_lbp_cca(data = train.head(10) , dir_name = \"train_images\") , dir_path = \"train_images\")\nsub['isup_grade'] = sub['isup_grade'].astype(int)\nsub.to_csv('submission.csv', index=False)\nsub.head()\n\nif os.path.exists(f'../input/prostate-cancer-grade-assessment/test_images'):\n    print(\"still can not access the test file ?\")\n    sub = inference(da = feature_engineering_lbp_cca(data = test , dir_name = \"test_images\") , dir_path = \"test_images\")\n    sub['isup_grade'] = sub['isup_grade'].astype(int)\n    sub.to_csv('submission.csv', index=False)\n    \nelse:\n    sub = pd.read_csv(\"/kaggle/input/prostate-cancer-grade-assessment/sample_submission.csv\")\n    sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}