{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport math\nimport multiprocessing\nimport pandas as pd\nimport typing as t\nfrom PIL import Image\nimport numpy as np\nfrom skimage.io import MultiImage\nimport tqdm\nimport cv2\nimport os\nimport json\nfrom datetime import datetime\nimport itertools\nimport functools\nfrom pandas import to_timedelta, Timedelta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"input_data = Path('../input/prostate-cancer-grade-assessment/')\nTRAIN = input_data/'train_images'\nLABELS = input_data/'train.csv'\n\ntarget_dir = Path('slide_chunks/')\nos.makedirs(target_dir, exist_ok=True)\n\n\ndf = pd.read_csv(LABELS).set_index('image_id')\n\n#torch.hub.DEFAULT_CACHE_DIR = 'cache'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### general jupyter and cv2 utility helpers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" general jupyter and cv2 utility helpers\n\"\"\"\nfrom matplotlib import pyplot as plt\nfrom matplotlib.pyplot import figure, imshow, axis\nfrom matplotlib.image import imread\n\ndef where_not_none(collection):\n    return [c for c in collection if c is not None]\n    \n\ndef reject_outliers(data, m=2):\n    if not isinstance(data, np.ndarray):\n        data = np.array(data)\n    return data[abs(data - np.mean(data)) < m * np.std(data)]\n\n\ndef sum_values(dict_list):\n    \"sum values for each key in dicts within a list\"\n    result = {} \n    for d in dict_list: \n        for k in d.keys(): \n            result[k] = result.get(k, 0) + d[k] \n    return result\n\ndef count_where(dict_list, key: str, value_equals=None):\n    \"count occurrences in dict_list having a key (that matches value_equals if provided)\"\n    if value_equals != None:\n        return sum(1 for m in dict_list if m.get(key, False) == value_equals)\n    return sum(1 for m in dict_list if m.get(key, False))\n\n\ndef interleave(list_a, list_b):\n    \"\"\"[a,b,c], [1,2,3] => [a,1,b,2,c,3]\n        \n        https://stackoverflow.com/a/7947461/2234013    \n    \"\"\"\n    c = list(list_a + list_b)\n    c[::2] = list_a\n    c[1::2] = list_b\n    return c\n\ndef cv2_to_pil(img_data):\n    if isinstance(img_data, Image.Image):\n        return img_data\n    return Image.fromarray(cv2.cvtColor(img_data, cv2.COLOR_BGR2RGB), mode='RGB')\n\ndef pil_to_cv2(img_data):\n    if isinstance(img_data, Image.Image):\n        img_data = np.array(img_data)\n        return cv2.cvtColor(img_data, cv2.COLOR_RGB2BGR)\n    return img_data\n\n\ndef meaningful_pixels(*images):\n    return np.sum([np.sum(np.array(image) <= 254) for image in images])\n\n# level 0 is shape (13312, 15360),\n# level 1 is shape (3328, 3840), (/4)\n# level 2 is shape (832, 960)   (/16)\ndef cv2_tiff_frame(path, level: int):\n    frame = MultiImage(str(path), conserve_memory=True)[level]\n    return cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)\n\ndef pil_tiff_frame(path, level: int):\n    return Image.fromarray(MultiImage(str(path), conserve_memory=True)[level])\n\ndef cv2_image(image):\n    \"resolve a str or Path into an image\"\n    if isinstance(image, (str, Path)):\n        return cv2.imread(str(image))\n    return image\n\ndef pil_image(image):\n    \"resolve a str or Path into an image\"\n    if isinstance(image, (str, Path)):\n        return Image.open(str(image))\n    return image\n\ndef show_image_row(list_of_images, figsize=(25,10), log_shape=True):\n    \"dispaly a row of images\"\n    list_of_images = list(list_of_images)\n    fig = figure(figsize=figsize)\n    number_of_files = len(list_of_images)\n    total_area = 0\n    for i in range(number_of_files):\n        a=fig.add_subplot(1,number_of_files,i+1)\n        image =pil_image(list_of_images[i])\n        if log_shape:\n            width, height= image.size\n            total_area += width*height\n            print(f'{i}th image is {width}x{height}, '\n                  f'mean is {np.mean(np.array(image))}, tot area {total_area}')\n        imshow(image,cmap='Greys_r')\n        axis('off')\n        \ndef show_slides(*image_ids, figsize=(25,10), level=2, log_shape=True):\n    show_image_row([\n        pil_tiff_frame(TRAIN/f'{i_id}.tiff', level)\n        for i_id in image_ids\n     ], figsize, log_shape=log_shape)\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### cv2 contour bounding rect extraction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" cv2 contour bounding rect extraction\n\n    we use contouring to find the tissue rectangles,\n    then write them to a cache file with write_chunk_windows\n\"\"\"\n \ndef cv2_threshold(image):\n    \"\"\"get a thresholded version of image for contouring\n    \n    we use generous blur (bilateralFilter) because of how\n    acceptible crude outlines are\n    \"\"\"\n    t = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n    t = cv2.bilateralFilter(t, 5, 175 , 175)\n    t = cv2.threshold(t ,200,255,cv2.THRESH_BINARY_INV)[1]\n    t = cv2.bilateralFilter(t, 32, 175 , 175)\n    return t\n    \ndef contours_of(image: Image):\n    \"extract the top-level contours from image\"\n    (contours, hierarchy) = cv2.findContours(\n        cv2_threshold(pil_to_cv2(image)), \n        cv2.RETR_EXTERNAL, \n        cv2.CHAIN_APPROX_SIMPLE\n    )\n    return sorted(contours, key = cv2.contourArea, reverse = True)\n\ndef tissue_chunk_windows_of(image: Image):\n    return {\n        'image_shape': (image.shape[0], image.shape[1]),\n        'chunk_rects': [\n            # center: (float, float), size: (float, float), theta: float)\n            cv2.minAreaRect(contour)\n            for contour in contours_of(image)\n        ]\n    }","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### write bounding rects to a cache","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" use the logic above to actually write a json cache of bouding rects\n\"\"\"\ntimestamp = lambda :datetime.now().strftime(\"%H:%M:%S\")\n\ndef write_json(f, data):\n    if os.path.exists(f):\n        os.remove(f)\n    with open(f, 'w') as o:\n        json.dump(data, o, indent=2)\n\ndef read_json(f):\n    if not os.path.exists(f):\n        return None\n    with open(f, 'r') as o:\n        return json.load(o)\n\ndef write_chunk_windows(source_directory: Path, image_ids: t.List[str], target_directory: Path, level = 2):\n\n    os.makedirs(target_directory, exist_ok=True)\n    total_chunks = 0\n    chunk_map = {}\n    target_json = target_directory/f'chunk_windows_{level}.json'\n    \n    for file_index, image_id in enumerate(image_ids):\n        wsi = pil_tiff_frame(source_directory/f'{image_id}.tiff', level)\n        chunk_map[image_id] = tissue_chunk_windows_of(wsi)\n        total_chunks += len(chunk_map[image_id]['chunk_rects'])\n        write_json(target_json, chunk_map)\n        if file_index % 100 == 0:\n            print(\n                f'[{timestamp()} | {file_index} | {image_id}]'\n                f' {total_chunks} ({total_chunks / (file_index + 1) }/file) chunks so far'\n            )\n    \n    write_json(target_json, chunk_map)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_level = 2\ncache_file = target_dir /f'chunk_windows_{_level}.json'\n\n# am fairly salty about this https://www.kaggle.com/product-feedback/99742\n_cache_file_resaved_as_input = '../input/panda-contour-min-bounding-rects/chunk_windows_from_tiff_2.json'\nwrite_json(cache_file, read_json(_cache_file_resaved_as_input))\n\nchunk_window_cache = read_json(cache_file)\n\nif not chunk_window_cache:\n    write_chunk_windows(\n         source_directory=TRAIN,\n         target_directory=target_dir,\n         image_ids=df.index #['031f5ef5b254fbacd6fbd279ebfe5cc0', '000920ad0b612851f8e01bcc880d9b3d']\n    )\n    chunk_window_cache = read_json(cache_file)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show_image_row(chunks_of('000920ad0b612851f8e01bcc880d9b3d', 0))  #645x137\n# wsi = pil_tiff_frame(str(TRAIN/'000920ad0b612851f8e01bcc880d9b3d.tiff'), 0)\n# Image.Image.resize?\n# print(datetime.now());wsi.resize((int(15360 / 16), int(13312 / 16)), resample=Image.BILINEAR);print(datetime.now())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### rectangle cropping and scaling helpers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" rectangle cropping and scaling helpers\n\n    now that we have a cache of rects, the most computationally intensive work is done\n\"\"\"\n\n     \ndef crop_tilted_rect(image: Image, rect):\n    \"\"\" crop rect out of image, handing rotation\n    \n    rect in this case is a tuple of ((center_x, center_y), (width, height), theta),\n    which I get from opencv's cv2.minAreaRect(contour)\n    \n    adapted from sub_image https://github.com/martinjevans/OpenCV-Rotate-and-Crop/blob/master/rotate_and_crop.py#L15\n    I've seen a few other solutions but they left some weird artifacts. Hopefully this one is mathematically correct\n    \"\"\"\n    # Get center, size, and angle from rect\n    center, size, theta = rect\n    width, height = [int(d) for d in size]\n\n    if width * height == 0:\n        return None\n    \n    if 45 < theta <= 90:\n        theta = theta - 90\n        width, height = height, width\n\n    theta *= math.pi / 180 # convert to rad\n    v_x = (math.cos(theta), math.sin(theta))\n    v_y = (-math.sin(theta), math.cos(theta))\n    s_x = center[0] - v_x[0] * (width / 2) - v_y[0] * (height / 2)\n    s_y = center[1] - v_x[1] * (width / 2) - v_y[1] * (height / 2)\n    mapping = np.array([v_x[0],v_y[0], s_x, v_x[1],v_y[1], s_y])\n    return image.transform((width, height), Image.AFFINE, data=mapping, resample=0, fill=1, fillcolor=(255,255,255))\n\n\ndef scale_factor(target_shape, source_shape):\n    t_x, t_y, *_ = target_shape\n    s_x, s_y, *_ = source_shape\n    scale_x = int(t_x / s_x)\n    scale_y = int(t_y / s_y)\n    assert scale_x == scale_y, (\n           f'scale factor ({scale_x}, {scale_y}) is uneven for shapes'\n           f'target ({t_x}, {t_y}) and source ({s_x}, {s_y})')\n    return scale_x\n\ndef scaled_rect(rect, scale: float):\n    (center_x, center_y), (width, height), theta = rect \n    if (scale == 1.0):\n        return ((center_x, center_y), (width, height), theta)\n    return (\n        (scale * center_x, scale * center_y),\n        (scale * width, scale * height),\n        theta\n    )\n\ndef scaled_chunks_of(full_image: Image, chunk_source_shape, chunk_rects):\n    # chunk_source_shape is (height, width, rgb)\n    c_height, c_width, *_ = chunk_source_shape\n    scale = scale_factor(full_image.size, (c_width, c_height))\n    return [\n        crop_tilted_rect(full_image, scaled_rect(chunk, scale))\n        for chunk in chunk_rects\n    ]\n\ndef _tilted_rect_area(rect):\n    center, (width, height), theta = rect\n    return width * height\n\ndef get_chunk_loader(source_dir, chunk_windows, area_floor=0):\n    def chunks_of(image_id: str, level: int = 2):\n        info = chunk_windows[image_id]\n        wsi_frame: Image = pil_tiff_frame(Path(source_dir)/f'{image_id}.tiff', level)\n        try:\n            chunks = [\n                c for c in info['chunk_rects']\n                if _tilted_rect_area(c) > area_floor\n            ]\n            return scaled_chunks_of(\n                wsi_frame,\n                info['image_shape'],\n                chunks,\n            )\n        except AssertionError as e:\n            raise ValueError(f'{image_id} scales wrong: {e}')\n    return chunks_of\n\n\ndef cropped_contours_of(image, contour_scaling=1.0):\n    width, height = image.size\n    cropped = where_not_none([\n        crop_tilted_rect(image, scaled_rect(cv2.minAreaRect(contour), 1 / contour_scaling))\n        for contour in contours_of(\n            image if contour_scaling == 1.0 else image.resize(\n                (int(contour_scaling * width), int(contour_scaling * height))\n            )\n        )\n    ])\n    if meaningful_pixels(*cropped) < (0.75 * meaningful_pixels(image)):\n        return cropped\n    return [image]\n\n\nchunks_of = get_chunk_loader(TRAIN, chunk_window_cache, area_floor=50)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### rectangle and image packing helpers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def _reduce_bounding_boxes(a, b):\n    \"bb that countains both a and b\"\n    (a_left, a_upper, a_right, a_lower) = a\n    (b_left, b_upper, b_right, b_lower) = b\n    return (\n        min(a_left, b_left),\n        min(a_upper, b_upper),\n        max(a_right, b_right),\n        max(a_lower, b_lower)\n    )\n\ndef _max_bounds(boxes):\n    return functools.reduce(_reduce_bounding_boxes,boxes)\n\ndef center_bounds(enclosing, contained):\n    \"center bounds if contained is within enclosing size\"\n    e_width, e_height = enclosing\n    c_width, c_height = contained\n    h_padding = int((e_width - c_width) / 2)\n    v_padding = int((e_height - c_height) / 2)\n    return (\n        h_padding, v_padding, h_padding + c_width, v_padding + c_height\n    )\n  \n\n\n\n# generic rect packer\nclass PackNode(object):\n    \"\"\"\n    Creates an area which can recursively pack other areas of smaller sizes into itself.\n    \n    https://code.activestate.com/recipes/578585/\n    \"\"\"\n    def __init__(self, area):\n        #if tuple contains two elements, assume they are width and height, and origin is (0,0)\n        if len(area) == 2:\n            area = (0,0,area[0],area[1])\n        self.area = area\n        self.children = None\n        self.filled = None\n\n\n    def __repr__(self):\n        return f\"PackNode({self.area})\"\n\n    @property\n    def width(self):\n        return self.area[2] - self.area[0]\n\n    @property\n    def height(self):\n        return self.area[3] - self.area[1]\n        \n    @property\n    def filled_bounding_box(self):\n        \"area of actual filled space\"\n        if self.children is not None:\n            child_bounds = [c.filled_bounding_box for c in self.children]\n            return _max_bounds([\n                self.filled.area,\n                *child_bounds\n            ])\n            \n         # empty\n        return (0,0,0,0)\n        \n        \n    \n    def _can_contain(self, area): # : PackNode\n        return area.width <= self.width and area.height <= self.height\n    \n    def area_to_right_of(self, width, height):\n        \"\"\" get a new area that is to the right of the given width/height\n            from the corner of this area\n        \"\"\"\n        # PIL crop boxes (areas) are (left, upper,  right, lower)\n        return (self.area[0]+width, self.area[1], self.area[2], self.area[1] + height)\n    \n    def area_below(self, height):\n        \"\"\" get a new area that is below the given height\n        \"\"\"\n        # PIL crop boxes (areas) are (left, upper,  right, lower)\n        return (self.area[0], self.area[1]+height, self.area[2], self.area[3])\n\n    def area_from_upper_left(self, width, height):\n        \"\"\" get a new area with the given width and height, \n            starting from the upper left of the current area\n        \"\"\"\n        # PIL crop boxes (areas) are (left, upper,  right, lower)\n        return (self.area[0], self.area[1], self.area[0]+width, self.area[1]+height)\n\n    def insert(self, area):\n        if self.children is not None:\n            area_right, area_below = self.children\n            return (\n                area_right.insert(area) or \n                area_below.insert(area)\n            )\n\n        area = PackNode(area)\n        if self._can_contain(area):\n            self.children = (\n                PackNode(self.area_to_right_of(area.width, area.height)),\n                PackNode(self.area_below(area.height))\n            )\n            self.filled = PackNode(self.area_from_upper_left(area.width, area.height))\n            return self.filled","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### `get_image_packer` for defining packers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\" image packing helpers\n\"\"\"\n\ndef cut_empty_space(image_data, empty=(255,255,255)):\n    \"\"\"lifted from https://www.kaggle.com/dannellyz/tissue-detect-td-conv-png-512x512\n    \n    removes all completely empty rows and columns\n    \"\"\"\n    if isinstance(image_data, Image.Image):\n        image_data = np.array(image_data)\n\n    #Crop empty space\n    #Remove by row\n    row_not_blank = [\n        row.all() for row in ~np.all(\n            image_data == empty, axis=1\n        )\n    ]\n    image_data = image_data[row_not_blank,:]\n    #Remove by column\n    col_not_blank =  [\n        col.all() for col in ~np.all(\n            image_data == empty, axis=0\n        )\n    ]\n    image_data = image_data[:,col_not_blank]\n    if image_data.size == 0:\n        return None\n    return Image.fromarray(image_data)\n\n\n\ndef halve_image(image: Image):\n    \" cut a PIL image in half, halving the longest side \"\n    width, height = image.size\n\n    if width > height:\n        return (\n            # crop box is   left,   upper,    right,   lower\n            image.crop((       0,       0,  width/2,  height )),\n            image.crop(( width/2,       0,    width,  height ))\n        )\n    else:\n        half_h = height/2\n        return (\n            # crop box is   left,   upper,    right,   lower\n            image.crop((       0,       0,    width,  half_h )),\n            image.crop((       0,  half_h,    width,  height ))\n        )\n\ndef recontour_parts(images, recontour_at=1.0):\n    return [\n        part for part in\n        _flatten([\n            cropped_contours_of(\n                part,\n                contour_scaling=recontour_at\n            ) for part in images\n        ])\n        if part is not None and len(part.size)\n    ]\n\n    \ndef _longest_side_ascending(image: Image):\n    \"a sorted key for ordering from the image with the longest side to that with the shortest\"\n    return -max(image.size)\n\n_flatten = lambda l: [item for sublist in l for item in sublist]\n\ndef eager_splits(images, split_list):\n    splits = []\n    for image, split_count in itertools.zip_longest(images, split_list, fillvalue=0):\n        if image == 0:\n            break\n        parts = [image]\n        while split_count > 0:\n            parts = _flatten([halve_image(p) for p in parts])\n            split_count -= 1\n        splits.extend(parts)\n    return splits\n        \n\n\ndef get_image_packer(\n    target_size, order_by=_longest_side_ascending,\n    retry_rotated=True, retry_halved=2, \n    retry_reversed_order_by=True,\n    retry_eager_splits=[4,2],\n    recontour_at=1.0,\n    low_to_high_first=False,\n    mean_threshold = 255,\n    recontour_splits = True,\n):\n    \n    def halve_and_recontour(image: Image):\n        return recontour_parts(halve_image(image), recontour_at)\n    \n    def pack_images(images, image_id=None):\n        \"\"\" Attempt to pack PIL images into a single image of target_size\n        \"\"\"\n        input_pixels = np.sum([np.sum(np.array(image) <= 254) for image in images])\n        start = datetime.now()\n        _base_metrics = {\n            'image_id': image_id,\n            'split_strategy': 'halve_and_recontour',\n            'splits': {},\n            'total_area': 0,\n            'rotations': 0\n        }\n        metrics = {**_base_metrics}\n                \n        def _pack(images, split_strategy=halve_and_recontour if recontour_splits else halve_image):\n            tree = PackNode(target_size)\n            composit = Image.new('RGB', target_size, (255, 255, 255))\n\n            def insert(image, halve_attempts=retry_halved):\n                image = cut_empty_space(image)\n                if image is None:\n                    return\n                width, height = image.size\n                if width*height == 0 or (np.mean(np.array(image)) >= mean_threshold):\n                    # skip all-white chunks\n                    return\n                width, height = image.size\n                uv = tree.insert((width, height))\n\n                if uv is None and retry_rotated:\n                    uv = tree.insert((height, width))\n                    if uv is not None:\n                        metrics['rotations'] += 1\n                        image = image.rotate(90, expand=True)\n                        \n                if uv is None and halve_attempts > 0:\n                    depth = retry_halved - halve_attempts + 1\n                    metrics['splits'].setdefault(depth, 0)\n                    metrics['splits'][depth] += 1\n                    # abstracted out halve_image into split_strategy \n                    # so we can retry with recontouring\n                    return [\n                        insert(part, halve_attempts - 1)\n                        for part in split_strategy(image)\n                    ]\n\n                if uv is None:\n                    raise ValueError(f'Pack size {target_size} too small for rects from {image_id}. {metrics}')\n                \n                metrics['total_area'] += width * height\n                composit.paste(image, uv.area)\n                \n            def try_inplace_center():\n                filled = tree.filled_bounding_box\n                _left, _upper, rightmost, lowermost = filled\n                if rightmost == 0 or lowermost == 0:\n                    return composit, metrics\n\n                data = composit.crop(filled)\n                composit.paste(Image.new('RGB', (rightmost, lowermost), (255, 255, 255)), filled)\n                composit.paste(data, center_bounds(target_size, (rightmost, lowermost)))\n                            \n            for image in images:\n                insert(image)\n            \n            try_inplace_center()\n            metrics['time'] = str(datetime.now() - start)\n            metrics['lost_pixels'] = int(input_pixels - np.sum(np.array(composit) <= 254))\n            metrics['lost_pixel_ratio'] = float(metrics['lost_pixels'] / input_pixels)\n\n            return composit, metrics\n        \n        expection = None\n        # todo this exception handling / error checking is bad / ugly\n        try:\n           return _pack(sorted(images, key=order_by, reverse=low_to_high_first))\n        except ValueError as e:\n            exception = e\n\n        \n        if retry_reversed_order_by:\n            metrics = {**_base_metrics}\n            metrics['chunks_reversed'] = True\n            try:\n                return _pack(sorted(images, key=order_by, reverse=not low_to_high_first))\n            except ValueError as e:\n                exception = e\n                     \n        if retry_eager_splits:\n            metrics = {**_base_metrics}\n            metrics['eager_splits'] = retry_eager_splits\n            try:\n                split = eager_splits(sorted(images, key=order_by), retry_eager_splits)\n                return _pack(recontour_parts(split, recontour_at) if recontour_splits else split)\n            except ValueError as e:\n                exception = e\n\n        raise exception\n\n    return pack_images\n\n# pack_images = get_image_packer((128 * 20, 128 * 6), retry_halved=4) # get_image_packer((128 * 8,128 * 8), retry_halved=4)\n# pack_images_hi = get_image_packer((4 * 128 * 30, 4 *128 * 10))    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# s = ['a10eb69fb260132fde150bd76bd7b15c', 'a6a7146bd23b394f54a5950d2dbefa7b', '1836f6539ccc9e37d426603cc4526f8b']\n\n# pack_images = get_image_packer((128 * 20, 128 * 6), retry_halved=4)\n\n# packed = []\n# for i in range(0,3):\n#     for i in df.sample(n=4).index:\n#         chunks = chunks_of(i, level=2)\n#         p, metrics = pack_images(chunks, i)\n#         packed.append(p)\n#         print(metrics)\n    \n#     show_image_row(packed)#, *chunks])\n#     packed=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# image_id, info = list(chunk_window_cache.items())[0]\n\n# samples = ['000920ad0b612851f8e01bcc880d9b3d', '031f5ef5b254fbacd6fbd279ebfe5cc0', '004391d48d58b18156f811087cd38abf']\n# #df = pd.read_csv(LABELS).set_index('image_id')i.size\n# get_image_packer((128 * 10,128 * 8))(chunks_of('031f5ef5b254fbacd6fbd279ebfe5cc0'), '031f5ef5b254fbacd6fbd279ebfe5cc0')[0]\n\n# s = [pack_images(chunks_of(i, 1)) for i in  df.sample(n=4).index]\n# show_image_row([pack_images(chunks_of(i, 1))[0] for i in  df.sample(n=4).index])# figsize=(50,20))\n\n# p = get_image_packer(\n#         (128 * 8,128 * 8),\n#         retry_halved=4,\n#         retry_eager_splits=[6,4]\n#     )\n# bigboys = [pack_images(chunks_of(i)) for i in  failures]\n# print([b[1] for b in bigboys])\n# show_image_row([b[0] for b in bigboys])# figsize=(50,20))\n\n# show_image_row(chunks_of('000920ad0b612851f8e01bcc880d9b3d'))\n# pack_images = get_image_packer((128 * 8,128 * 8), retry_halved=4, recontour_at=1.0)\n# pack_hi =get_image_packer((4 * 128 * 8, 4 * 128 * 8), retry_halved=4)\n\n# datetime.now()\n# print(datetime.now())\n# show_image_row(_flatten([\n#     (pack_images(chunks_of(i))[0], pack_hi(chunks_of(i, 1))[0])\n#     for i in df.sample(n=3).index\n# ]))\n# d = datetime.now()\n# pack_hi =get_image_packer((16 * 128 * 8, 16 * 128 * 8), retry_halved=4, recontour_at=(1/16))\n# s = [pack_hi(chunks_of(i, 0)) for i in  df.sample(n=4).index]\n# print(datetime.now())\n# h, m = pack_hi(chunks_of('a10eb69fb260132fde150bd76bd7b15c', 1))\n\n# cv2.__version__\n# print(datetime.now())\n# h\n\n# Image.MAX_IMAGE_PIXELS  = 268435456 + 1\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def whats_up_with(*image_ids):\n     for image_id in image_ids:\n         show_slides(image_id)\n         show_image_row(chunks_of(image_id))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # samples = ['000920ad0b612851f8e01bcc880d9b3d', '031f5ef5b254fbacd6fbd279ebfe5cc0', '004391d48d58b18156f811087cd38abf']\n# #df = pd.read_csv(LABELS).set_index('image_id')\n\n# metrics = [\n#     pack_images(chunks_of(sample, 2), sample)[1]\n#     for sample in df.index\n# ]\n# i = show_image_row(chunks_of('00928370e2dfeb8a507667ef1d4efcbb'))[0]\n# i.rotate(90, expand=True)\n# i, metrics = pack_images(chunks_of('1836f6539ccc9e37d426603cc4526f8b', 2)); i[0]\n# s = '1f368e9829e850bd6b6de7a521376720'\n\n\n# def whats_up_with(*image_ids):\n#     for image_id in image_ids:\n#         show_slides(image_id)\n#         show_image_row(chunks_of(image_id))\n    \n\n# def consider_packer(pack=pack_images, n=None):\n#     metrics = []\n#     global s\n#     failures = []\n#     for index, sample in enumerate(df.sample(n=n).index if n else df.index):\n#         try:\n#             metrics.append(pack(chunks_of(sample, 2), sample)[1])\n#         except Exception as e:\n#             print(f'{sample} failed!')\n#             s = sample\n#             failures.append(sample)\n#     return metrics, failures\n\n# metrics, failures = consider_packer()\n# whats_up_with(*failures)\n\n\n# metrics, failures = consider_packer(\n#     get_image_packer(\n#         (128 * 8,128 * 8),\n#         retry_halved=4,\n#         retry_eager_splits=[6,4]\n#     )\n# )\n# len(metrics)\n#  get_image_packer(\n#         (128 * 8,128 * 8),\n#         retry_halved=4,\n#         retry_eager_splits=[6,4]\n#     )(chunks_of(failures[0])\n# whats_up_with(*failures)\n\n\n# show_image_row([\n#     get_image_packer((128 * 10,128 * 10), retry_halved=8,\n#                     retry_eager_splits=[6,4])(chunks_of(s), s)[0]\n#     for s in [\n#         'a10eb69fb260132fde150bd76bd7b15c',\n#         '1836f6539ccc9e37d426603cc4526f8b', '1f368e9829e850bd6b6de7a521376720', '000920ad0b612851f8e01bcc880d9b3d', '031f5ef5b254fbacd6fbd279ebfe5cc0', '004391d48d58b18156f811087cd38abf'\n#     ] ])\n\n# metrics = consider_packer()\n# show_slides(s)\n# show_image_row(chunks_of(s))\n# i = pack_images(chunks_of(s, 2))[0]\n# i\n# show_image_row([pack_images(chunks_of(s, 2))[0] for s in [\n#     '1836f6539ccc9e37d426603cc4526f8b', '1f368e9829e850bd6b6de7a521376720', '000920ad0b612851f8e01bcc880d9b3d', '031f5ef5b254fbacd6fbd279ebfe5cc0', '004391d48d58b18156f811087cd38abf'\n# ] ])\n# show_slides(s)\n# show_image_row(chunks_of(s))\n# show_image_row([print(np.mean(np.array(c))) or c for c in chunks_of(s)])\n\n# for sample in samples:#df.sample(n=5).index:\n#     try:\n#         show_image_row([\n#             pack_images(chunks_of(sample, 2))[0],\n#             #pack_images_hi(chunks_of(sample, 1))[0],\n#         ], figsize=(50,20))\n#     except Exception as e:\n#         print(sample)\n#         raise e\n# #2472 / 624, 1856 /472\n\n# #pack_images(chunks_of('d527d0d353eef920f47505af9fe37956'))\n\n# result, metrics = get_image_packer((128 * 6, 128 * 6), retry_halved=4, retry_eager_splits=[12])(chunks_of('1836f6539ccc9e37d426603cc4526f8b'))\n# result\n# whats_up_with('1836f6539ccc9e37d426603cc4526f8b')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reject_outlying_deltas(times):\n    seconds = reject_outliers([t.total_seconds() for t in times])\n    return [Timedelta(seconds=sec) for sec in seconds]\n\n\ndef summarize_metrics(metrics):\n    times = [to_timedelta(m['time']) for m in metrics]\n    total_time = sum(times, Timedelta(0))\n    non_outlier_times = reject_outlying_deltas(times)\n    return dict(\n        splits = sum_values([m.get('splits', {}) for m in metrics]),\n        recontoured_splits = count_where(metrics, 'split_strategy', value_equals='halve_and_recontour'),\n        reversed_chunks = count_where(metrics, 'chunks_reversed'),\n        split_eagerly = count_where(metrics, 'eager_splits'),\n        total_count = len(metrics),\n        times=dict(\n            total = str(total_time.to_pytimedelta()),\n            average = str((total_time / len(times)).to_pytimedelta()),\n            max_non_outlier = str(max(non_outlier_times or times).to_pytimedelta()),\n            max = str(max(times).to_pytimedelta()),\n            outlier_count = len(metrics) - len(non_outlier_times)\n        )\n    )\n\n\n    \n\ndef write_packed_images(pack_images, chunks_of, image_ids: t.List[str], target_directory: Path, level = 2):\n\n    os.makedirs(target_directory, exist_ok=True)\n    target_json = target_directory/f'pack_run.json'\n    \n    metrics = []\n    failures = []\n    \n    for file_index, image_id in enumerate(image_ids):\n        \n        chunks = chunks_of(image_id, level=2)\n        try:\n            packed, _metrics = pack_images(chunks, image_id)\n        except ValueError as e:\n            print(f'{image_id} failed! {e}')\n            failures.append(image_id)\n            \n        metrics.append(_metrics)\n        \n        if file_index % 100 == 0:\n            summary = summarize_metrics(metrics)\n            print(\n                f'[{timestamp()} | {file_index} | {image_id}] {len(failures)} failures! summary={json.dumps(summary, indent=2)}'\n            )\n            write_json(target_json, { 'metrics': metrics, 'failures': failures, 'summary': summary })\n            \n        packed.save(str(target_directory/f'{image_id}.png'),\"PNG\")\n\n    summary = summarize_metrics(metrics)\n    write_json(target_json, { 'metrics': metrics, 'failures': failures, 'summary': summary })\n    \n    return metrics, failures, summary\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"whats_up_with('1836f6539ccc9e37d426603cc4526f8b')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" i = pil_tiff_frame(TRAIN/f'1836f6539ccc9e37d426603cc4526f8b.tiff', 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sum(np.array(i) <= 254)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"whats_up_with('1f368e9829e850bd6b6de7a521376720')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# slender denny and lenny\npacked, stats = get_image_packer((128 * 9, 128 * 9), retry_halved=6, recontour_at=0.25, recontour_splits=True, retry_eager_splits=[4,3,2])(chunks_of('1f368e9829e850bd6b6de7a521376720'))\nprint(stats)\npacked","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# big bertha\npacked, stats = get_image_packer((128 * 8, 128 * 8), retry_halved=6, recontour_at=0.25, recontour_splits=True)(chunks_of('1836f6539ccc9e37d426603cc4526f8b'))\nprint(stats)\npacked","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"business are connecting 1 to another","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"`a37b456a2bc920630c60c2ff0c7d6325`","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I could have gotten them smaller if I had just munged outliers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# 10000 * ((128 * 20 *128 * 6) * 3 / 8) bytes to gigabytes is 7.3728 gigabytes, \n# tarred will be ~ 3.6864. This is ok for now but won't be for level 1\n\n# output_dimensions = (128 * 20, 128 * 6)\n# (128 * 13, 128 * 5) saved under run_1664x640\n\n# (128 * 12 , 128 * 4) under run_1536x512\n\noutput_dimensions = (128 * 9, 128 * 9)\npack_low = get_image_packer(output_dimensions, retry_halved=6, recontour_at=0.25, recontour_splits=True, retry_eager_splits=[4,3,2])\n#pack_hi = get_image_packer(tuple((4 * d for d in output_dimensions)), retry_halved=4, recontour_at=0.25)\n\n# whats_up_with('01571191abf6e8e209111e819823759b')\nmetrics, failures, summary = write_packed_images(\n    pack_images=pack_low,\n    chunks_of=chunks_of,\n    target_directory=Path('../level_2_packed/'),\n    image_ids=df.index, #or df.sample(n=10).index,\n    level=2\n)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"whats_up_with('a37b456a2bc920630c60c2ff0c7d6325')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loosen retry_halved for our special case\n_pack_low = get_image_packer(output_dimensions, retry_halved=8, recontour_at=0.25, recontour_splits=True, retry_eager_splits=[4,3,2])\npacked, stats = _pack_low(chunks_of('a37b456a2bc920630c60c2ff0c7d6325'))\nprint(stats)\npacked","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nf_metrics, f_failures, f_summary = write_packed_images(\n    pack_images=_pack_low,\n    chunks_of=chunks_of,\n    target_directory=Path('../level_2_packed/'),\n    image_ids=['a37b456a2bc920630c60c2ff0c7d6325'],\n    level=2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"! ls ../level_2_packed/a37b456a2bc920630c60c2ff0c7d6325.png\n! ls ../level_2_packed/* | wc -l\n\n! rm -f level_2_packed_w_recontouring.tar.gz\n!tar -czf level_2_packed_w_recontouring.tar.gz ../level_2_packed/*.png\n#!mv ../level_2_packed/pack_run.json recontouring_pack_run.json \n!ls -l .\n!ls ../level_2_packed | wc -l\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!tar -czf level_2_packed_w_recontouring.tar.gz ../level_2_packed/*.png\n!mv ../level_2_packed/pack_run.json recontouring_pack_run.json \n!ls -l .\n!ls ../level_2_packed | wc -l","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}