{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!conda install ../input/strip-ai-library-fetcher/*.tar.bz2","metadata":{"execution":{"iopub.status.busy":"2022-09-22T20:29:36.16943Z","iopub.execute_input":"2022-09-22T20:29:36.170543Z","iopub.status.idle":"2022-09-22T20:32:30.626549Z","shell.execute_reply.started":"2022-09-22T20:29:36.170446Z","shell.execute_reply":"2022-09-22T20:32:30.625096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport skimage.io as io\nimport cv2\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nimport math\nimport gc\nimport zipfile\nimport openslide\nimport os\nimport pyvips\nimport time\nimport random\nimport torch\nimport pandas as pd\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-22T20:33:39.942201Z","iopub.execute_input":"2022-09-22T20:33:39.94265Z","iopub.status.idle":"2022-09-22T20:33:39.949548Z","shell.execute_reply.started":"2022-09-22T20:33:39.942615Z","shell.execute_reply":"2022-09-22T20:33:39.948531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed_value):\n    random.seed(seed_value)\n    np.random.seed(seed_value)\n    torch.manual_seed(seed_value)\n    os.environ['PYTHONHASHSEED'] = str(seed_value)    \n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(seed_value)\n        torch.cuda.manual_seed_all(seed_value)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = True\n\nseed = 0\nseed_everything(seed)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T20:33:40.207896Z","iopub.execute_input":"2022-09-22T20:33:40.208782Z","iopub.status.idle":"2022-09-22T20:33:40.21723Z","shell.execute_reply.started":"2022-09-22T20:33:40.20873Z","shell.execute_reply":"2022-09-22T20:33:40.215746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/mayo-clinic-strip-ai/train.csv\")\ntrain['image_dir'] = \"../input/mayo-clinic-strip-ai/train\"\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-22T20:33:40.571434Z","iopub.execute_input":"2022-09-22T20:33:40.572407Z","iopub.status.idle":"2022-09-22T20:33:40.590752Z","shell.execute_reply.started":"2022-09-22T20:33:40.572364Z","shell.execute_reply":"2022-09-22T20:33:40.589611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"format_to_dtype = {\n       'uchar': np.uint8,\n       'char': np.int8,\n       'ushort': np.uint16,\n       'short': np.int16,\n       'uint': np.uint32,\n       'int': np.int32,\n       'float': np.float32,\n       'double': np.float64,\n       'complex': np.complex64,\n       'dpcomplex': np.complex128,\n    }\ndef vips2numpy(vi):\n\n        return np.ndarray(\n            buffer=vi.write_to_memory(),\n            dtype=format_to_dtype[vi.format],\n            shape=[vi.height, vi.width, vi.bands])\n# Can open at least 35kx35kx3 images\ndef load_image(img_path, max_size):\n    slide = openslide.OpenSlide(img_path)\n    x_dim,y_dim = slide.dimensions\n\n    del slide\n    gc.collect()\n    size = x_dim * y_dim\n    size_limit = max_size*max_size\n    if size > size_limit:\n        resize_factor = math.sqrt(size / size_limit)\n        width = int(x_dim / resize_factor)\n        height= int(y_dim / resize_factor)\n \n        img = pyvips.Image.thumbnail(img_path,width,height=height) \n    else:\n        img = pyvips.Image.new_from_file(img_path)\n    #img = vips2numpy(img)\n    return img.numpy()\ndef make_tiles(img, tile_size=224, fg_perc = 0.2):\n    '''\n    img: np.ndarray with dtype np.uint8 and shape (width, height, channel)\n    '''\n\n    w, h, ch = img.shape\n    num_tiles = int((w * h / (tile_size*tile_size)) * fg_perc) \n    pad0, pad1 = (tile_size - w%tile_size) % tile_size, (tile_size - h%tile_size) % tile_size\n    padding = [[pad0//2, pad0-pad0//2], [pad1//2, pad1-pad1//2], [0, 0]]\n    img = np.pad(img, padding, mode='constant', constant_values=255)\n    img = img.reshape(img.shape[0]//tile_size, tile_size, img.shape[1]//tile_size, tile_size, ch)\n    img = img.transpose(0, 2, 1, 3, 4).reshape(-1, tile_size, tile_size, ch)\n    if len(img) < num_tiles: # pad images so that the output shape be the same\n        padding = [[0, num_tiles-len(img)], [0, 0], [0, 0], [0, 0]]\n        img = np.pad(img, padding, mode='constant', constant_values=255)\n    sort = np.argsort(img.reshape(img.shape[0], -1).sum(-1))\n    idxs = sort[:num_tiles]\n    img = img[idxs]\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-09-22T20:44:19.437692Z","iopub.status.idle":"2022-09-22T20:44:19.43867Z","shell.execute_reply.started":"2022-09-22T20:44:19.438332Z","shell.execute_reply":"2022-09-22T20:44:19.438373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tile_size = 224\nmax_size = 20000\nfg_perc = 0.15 # ASsume x percentage of image is foreground\nstart = 0\ntracker = start\nlength = len(train)\nend = min(length,start+75)\nout_dir = 'train_images.zip'\nwith zipfile.ZipFile(out_dir, \"w\") as out_image:\n    for idx,row in train[start:end].iterrows():\n        image_id = row['image_id']\n        print(f\"Processing image no: {tracker}...\")\n        path = f\"{row['image_dir']}/{image_id}.tif\"\n        img = load_image(path,max_size=max_size)\n        tiles = make_tiles(img,tile_size,fg_perc)\n        for i, tile in enumerate(tiles):\n                tile = cv2.cvtColor(tile, cv2.COLOR_RGB2BGR)\n                tile = cv2.imencode(\".jpg\", tile, [cv2.IMWRITE_JPEG_QUALITY, 100])[1]\n                out_image.writestr(f\"{image_id}_{i}.jpg\", tile)\n        del img, tiles, tile; gc.collect()\n        tracker+=1\n\n    \n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T20:44:54.469386Z","iopub.execute_input":"2022-09-22T20:44:54.469911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# format_to_dtype = {\n#        'uchar': np.uint8,\n#        'char': np.int8,\n#        'ushort': np.uint16,\n#        'short': np.int16,\n#        'uint': np.uint32,\n#        'int': np.int32,\n#        'float': np.float32,\n#        'double': np.float64,\n#        'complex': np.complex64,\n#        'dpcomplex': np.complex128,\n#     }\n# def vips2numpy(vi):\n    \n#         return np.ndarray(\n#             buffer=vi.write_to_memory(),\n#             dtype=format_to_dtype[vi.format],\n#             shape=[vi.height, vi.width, vi.bands])\n# # Can open at least 35kx35kx3 images\n# def load_image(img_path, max_size=35000):\n#     slide = openslide.OpenSlide(img_path)\n#     x_dim,y_dim = slide.dimensions\n\n#     del slide\n#     gc.collect()\n#     size = x_dim * y_dim\n#     size_limit = max_size*max_size\n#     if size > size_limit:\n#         resize_factor = math.sqrt(size / size_limit)\n#         width = int(x_dim / resize_factor)\n#         height= int(y_dim / resize_factor)\n#         img = pyvips.Image.thumbnail(img_path,width,height=height,size=pyvips.enums.Size.DOWN) \n    \n#     else:\n#         img = pyvips.Image.new_from_file(img_path)\n    \n#     img = vips2numpy(img)\n#     return img\n# def make_tiles(img, tile_size=500, num_tiles=100):\n#     '''\n#     img: np.ndarray with dtype np.uint8 and shape (width, height, channel)\n#     '''\n#     w, h, ch = img.shape\n#     pad0, pad1 = (tile_size - w%tile_size) % tile_size, (tile_size - h%tile_size) % tile_size\n#     padding = [[pad0//2, pad0-pad0//2], [pad1//2, pad1-pad1//2], [0, 0]]\n#     img = np.pad(img, padding, mode='constant', constant_values=255)\n#     img = img.reshape(img.shape[0]//tile_size, tile_size, img.shape[1]//tile_size, tile_size, ch)\n#     img = img.transpose(0, 2, 1, 3, 4).reshape(-1, tile_size, tile_size, ch)\n#     if len(img) < num_tiles: # pad images so that the output shape be the same\n#         padding = [[0, num_tiles-len(img)], [0, 0], [0, 0], [0, 0]]\n#         img = np.pad(img, padding, mode='constant', constant_values=255)\n#     sort = np.argsort(img.reshape(img.shape[0], -1).sum(-1))\n#     top = len(sort) // 6 # Index for Top %15 percent \n#     if top < num_tiles:\n#         idxs = sort[:num_tiles]\n#     else:\n#         # Randomly select from top %15 rather than select top num_tiles to avoid bias towards dark parts\n#         idxs = np.random.choice(sort[:top],num_tiles,replace=False)\n#     img = img[idxs]\n#     return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_tiles = 100\n# tile_size = 500\n# start = 0\n# tracker = start\n# length = len(image_paths)\n# end = min(length,start+100)\n# with zipfile.ZipFile(f\"images_{start}_{end}.zip\", \"w\") as out_image:\n#     for image_path in image_paths[start:end]:\n#         image = load_image(image_path)\n#         tiles = make_tiles(image,tile_size,num_tiles)\n#         del image\n#         idx = 0\n#         name = image_path.name.replace('.tif','')\n#         for tile in tiles:\n#             img = cv2.imencode(\".jpg\", tile, [cv2.IMWRITE_JPEG_QUALITY, 100])[1]\n#             out_image.writestr(f\"{name}_{idx}.jpg\", img)\n#             #cv2.imwrite(f\"{name}_{idx}.jpg\",tile)\n#             idx+=1\n#         del tiles\n#         print(f\"Completed image no: {tracker}\")\n#         tracker+=1\n#         gc.collect()\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}