{"cells":[{"cell_type":"markdown","metadata":{},"source":"### First push of tiling refactor"},{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":"import os, sys\nimport cv2\nimport skimage.io\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt"},{"cell_type":"code","execution_count":6,"metadata":{},"outputs":[],"source":"get_user = os.environ.get('USER', 'KAGGLE')\n\nif get_user == 'KAGGLE':\n    my_env = 'KAGGLE'\nelif get_user == 'jupyter':\n    my_env = 'GCP'\nelif get_user == 'user':\n    my_env = 'LOCAL'\nelse:\n    my_env = None\n    \nassert my_env is not None    \n\nenv_input_fn = {\n    'KAGGLE': '../input/prostate-cancer-grade-assessment/',\n    'LOCAL':  '../data/',\n    'GCP':    '../../',\n}\n\ninput_fn = env_input_fn[my_env]"},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[],"source":"train_df = pd.read_csv(input_fn + 'train.csv')"},{"cell_type":"markdown","metadata":{},"source":"##### Functions"},{"cell_type":"code","execution_count":13,"metadata":{},"outputs":[],"source":"TRAIN = input_fn + 'train_images/'\nMASKS = input_fn + 'train_label_masks/'\nOUT_TRAIN = 'train.zip'\nOUT_MASKS = 'masks.zip'\nsz = 128\nN = 16\nx_tot,x2_tot = [],[]"},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[],"source":"def build_tiles(img, mask):\n\n    shape = img.shape\n    \n    # padding allows even modulo of SZ into WSI's\n    # no overflow smaller tiles\n    pad0,pad1 = (sz - shape[0]%sz)%sz, (sz - shape[1]%sz)%sz\n    img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=255)\n    mask = np.pad(mask,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],\n                constant_values=0)\n    \n    # tile with broadcasting\n    img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)\n    img = img.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    mask = mask.reshape(mask.shape[0]//sz,sz,mask.shape[1]//sz,sz,3)\n    mask = mask.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n    # now, img is : (N_TILES, SZ, SZ, CHANNELS )\n    \n    return img, mask\n\ndef select_tiles(img, mask):\n    \n    result = []\n    \n    # deal with case where tiles < fully titled WSI\n    if len(img) < N:\n        mask = np.pad(mask,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=0)\n        img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)\n    \n    # tile selection when image is larger than tiles selected\n    # idxs are indexes for the lowest N\n    idxs = np.argsort(img.reshape(img.shape[0],-1).sum(-1))[:N]\n    img = img[idxs]\n    mask = mask[idxs]\n    for i in range(len(img)):\n        result.append({'img':img[i], 'mask':mask[i], 'idx':i})\n    return result\n\ndef tile(img, mask):\n    \n    img_tiled, mask_tiled = build_tiles(img, mask)\n    \n    return select_tiles(img_tiled, mask_tiled)"},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[],"source":"x_tot,x2_tot = [],[]\nnames = [name[:-10] for name in os.listdir(MASKS)]"},{"cell_type":"code","execution_count":17,"metadata":{},"outputs":[],"source":"# name = names[0]\n\n# img = skimage.io.MultiImage(os.path.join(TRAIN,name+'.tiff'))[-1]\n# mask = skimage.io.MultiImage(os.path.join(MASKS,name+'_mask.tiff'))[-1]\n# tiles = tile(img,mask)"},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[],"source":"with zipfile.ZipFile(OUT_TRAIN, 'w') as img_out,\\\n zipfile.ZipFile(OUT_MASKS, 'w') as mask_out:\n    for name in tqdm(names):\n        img = skimage.io.MultiImage(os.path.join(TRAIN,name+'.tiff'))[-1]\n        mask = skimage.io.MultiImage(os.path.join(MASKS,name+'_mask.tiff'))[-1]\n        tiles = tile(img,mask)\n        for t in tiles:\n            img,mask,idx = t['img'],t['mask'],t['idx']\n            x_tot.append((img/255.0).reshape(-1,3).mean(0))\n            x2_tot.append(((img/255.0)**2).reshape(-1,3).mean(0)) \n            #if read with PIL RGB turns into BGR\n            img = cv2.imencode('.png',cv2.cvtColor(img, cv2.COLOR_RGB2BGR))[1]\n            img_out.writestr(f'{name}_{idx}.png', img)\n            mask = cv2.imencode('.png',mask[:,:,0])[1]\n            mask_out.writestr(f'{name}_{idx}.png', mask)"},{"cell_type":"code","execution_count":27,"metadata":{},"outputs":[],"source":"img_avr =  np.array(x_tot).mean(0)\nimg_std =  np.sqrt(np.array(x2_tot).mean(0) - img_avr**2)\nprint('mean:',img_avr, ', std:', np.sqrt(img_std))"},{"cell_type":"markdown","metadata":{},"source":"##### Exploratory"},{"cell_type":"code","execution_count":30,"metadata":{},"outputs":[],"source":"# !ls -lh1 "},{"cell_type":"code","execution_count":29,"metadata":{},"outputs":[],"source":"# tz = zipfile.ZipFile('train.zip')"},{"cell_type":"code","execution_count":28,"metadata":{},"outputs":[],"source":"# fns = [e.filename for e in tz.filelist]\n# len(fns)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":""}],"metadata":{"kernelspec":{"display_name":"Python [conda env:fastai2]","language":"python","name":"conda-env-fastai2-py"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.6"}},"nbformat":4,"nbformat_minor":2}