{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport sys\nsys.setrecursionlimit(16130)\nimport shutil\nimport json\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom skimage import io,color\n\nfrom glob import glob\nfrom pprint import pprint\nfrom collections import defaultdict\nfrom random import sample\nimport gc","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:18:51.903693Z","iopub.execute_input":"2023-01-30T19:18:51.9041Z","iopub.status.idle":"2023-01-30T19:18:52.891364Z","shell.execute_reply.started":"2023-01-30T19:18:51.904058Z","shell.execute_reply":"2023-01-30T19:18:52.890281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions for selecting first-level boxes \nFirst get binary (bw) 256*256 image, apply simple convolution to define boxes around non-bkg regions\nUse filter size that divides N and do half-filter-size steps","metadata":{}},{"cell_type":"code","source":"## Not really used\ndef get_closest_filter_size(n):\n    diffs=[abs(k-n) for k in filter_sizes]\n    i=diffs.index(min(diffs))\n    return fss[i]\n\n## Simple conv for seeding and expanding boxes\ndef myconvsum2d(img,filt_size,steps,cs=[]):\n    out=np.zeros((steps,steps))\n    step=int(filt_size/2)\n    for i in range(steps):\n        for j in range(steps):\n            if cs and (i*step<cs[0] or (i+2)*step>cs[1] \\\n                         or j*step<cs[2] or (j+2)*step>cs[3]):\n                out[i,j]=0\n            else:\n                out[i,j]=np.sum(img[i*step:(i+2)*step,j*step:(j+2)*step])\n    return out\n\n## Expand box from seed\ndef get_boxes_expand(raw,m): ##get_boxes_expand(raw_img,filter_size)\n    r=np.copy(raw)\n    def check(i,j,corners):\n#        print(\" \",i,j)\n        r[i,j]=0\n        if i-1>=0 and r[i-1,j]:\n            corners[0]=min(corners[0],i-1)\n            corners=check(i-1,j,corners)\n        if i+1<m and r[i+1,j]:\n            corners[1]=max(corners[1],i+1)\n            corners=check(i+1,j,corners)\n        if j-1>=0 and r[i,j-1]:\n            corners[2]=min(corners[2],j-1)\n            corners=check(i,j-1,corners)\n        if j+1<m and r[i,j+1]:\n            corners[3]=max(corners[3],j+1)\n            corners=check(i,j+1,corners)\n        if i-1>=0 and j-1>=0 and r[i-1,j-1]:\n            corners[0]=min(corners[0],i-1)\n            corners[2]=min(corners[2],j-1)\n            corners=check(i-1,j-1,corners)\n        if i-1>=0 and j+1<m and r[i-1,j+1]:\n            corners[0]=min(corners[0],i-1)\n            corners[3]=max(corners[3],j+1)\n            corners=check(i-1,j+1,corners)\n        if i+1<m and j-1>=0 and r[i+1,j-1]:\n            corners[1]=max(corners[1],i+1)\n            corners[2]=min(corners[2],j-1)\n            corners=check(i+1,j-1,corners)\n        if i+1<m and j+1<m and r[i+1,j+1]:\n            corners[1]=max(corners[1],i+1)\n            corners[3]=max(corners[3],j+1)\n            corners=check(i+1,j+1,corners)\n        return corners\n    #####################\n    allcorners=[]\n    for i in range(m):\n        for j in range(m):\n            if r[i,j]:\n#                print(i,j)\n                allcorners.append(check(i,j,[i,i,j,j]))\n    return allcorners\n\n## Translate indexes before/after conv\ndef idx_expand(allcorners,step):\n    for c in allcorners:\n        c[0]=step*c[0]\n        c[1]=step*(c[1]+2)\n        c[2]=step*c[2]\n        c[3]=step*(c[3]+2)\n    return allcorners\ndef idx_reduce(allcorners,step):\n    for c in allcorners:\n        c[0]=c[0]//step\n        c[1]=c[1]//step-2\n        c[2]=c[2]//step\n        c[3]=c[3]//step-2\n    return allcorners","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:18:55.584137Z","iopub.execute_input":"2023-01-30T19:18:55.584505Z","iopub.status.idle":"2023-01-30T19:18:55.606655Z","shell.execute_reply.started":"2023-01-30T19:18:55.584474Z","shell.execute_reply":"2023-01-30T19:18:55.60558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Return box corners per conv filter size (in the end I use only filter_size==4)\ndef get_box_corners(mask,fs_start=0,fs_end=0):\n    if not fs_start:\n        C=(np.sum(mask))**0.5 ## If all '1' bins in a square, this is its side size\n        fs_start=get_closest_filter_size(C)//2\n    if not fs_end:\n        fs_end=filter_sizes[-1]\n    k_start=filter_sizes.index(fs_start)\n    k_end=filter_sizes.index(fs_end)\n    cornerdct={}\n    first=True\n    for k in range(k_start,k_end+1):\n        fs=filter_sizes[k]\n        step=fs//2 ## step size\n        m=2*n//fs-1 ## number of steps == size of myconvsum2d output (N=256)\n        th=box_thdct[fs] ## density threshold per filter size\n        if first:\n            out=myconvsum2d(mask,fs,m) ## myconvsum2d output\n            first=False\n        else:\n            out=np.zeros((m,m))\n            allpre=[cs for k1 in range(k_start,k) for cs in cornerdct[filter_sizes[k1]]]\n            for corners in allpre:\n                out+=myconvsum2d(mask,fs,m,corners)\n        raw=1*(out>(fs**2*th)) ## raw image\n        cornerdct[fs]=idx_expand(get_boxes_expand(raw,m),step)\n    return cornerdct\n\n## Choose 'maxboxes' largest first-level boxes with area>='min_box_area' and don't overlap more than maxovl\ndef select_boxes(allcs,min_area,maxovl,maxboxes):\n    allcornerstmp=[]\n    sizes=[]\n    for cs in allcs:\n        size=(cs[1]-cs[0])*(cs[3]-cs[2])\n        if size>=min_area:\n            allcornerstmp.append(cs)\n            sizes.append(size)\n    if not sizes:\n        return []\n    sizes,allcornerstmp=zip(*sorted(zip(sizes,allcornerstmp),reverse=True))\n    allcorners=[]\n    for cs,size in zip(allcornerstmp,sizes):\n        isiso=True\n        ovl=0\n        for pcs in allcorners:\n            if cs[0]>=pcs[1] or cs[1]<=pcs[0] or cs[2]>=pcs[3] or cs[3]<=pcs[2]:\n                continue\n            ovl+=(min(cs[1],pcs[1])-max(cs[0],pcs[0]))*(min(cs[3],pcs[3])-max(cs[2],pcs[2]))/size\n            if ovl>maxovl:\n                isiso=False\n                break\n        if isiso:\n            allcorners.append(cs)\n            if len(allcorners)==maxboxes:\n                return allcorners\n    return allcorners","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:18:57.717963Z","iopub.execute_input":"2023-01-30T19:18:57.71834Z","iopub.status.idle":"2023-01-30T19:18:57.731871Z","shell.execute_reply.started":"2023-01-30T19:18:57.718308Z","shell.execute_reply":"2023-01-30T19:18:57.730948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## recursively split box in 4 if too 'large'\ndef get_boxsplits(cs,mask,deno):\n    if np.sum(mask[cs[0]:cs[1],cs[2]:cs[3]])//deno>max_cells:\n        return get_boxsplits([cs[0],(cs[0]+cs[1])//2,cs[2],(cs[2]+cs[3])//2],mask,deno)\\\n              +get_boxsplits([cs[0],(cs[0]+cs[1])//2,(cs[2]+cs[3])//2,cs[3]],mask,deno)\\\n              +get_boxsplits([(cs[0]+cs[1])//2,cs[1],cs[2],(cs[2]+cs[3])//2],mask,deno)\\\n              +get_boxsplits([(cs[0]+cs[1])//2,cs[1],(cs[2]+cs[3])//2,cs[3]],mask,deno)\n    else:\n        return [[cs[0],cs[1],cs[2],cs[3]]]\n\n## get all box corners with adequate 'size' in terms of cell_area\ndef get_boxes(mask,cell_area,verbose=False):\n    min_area=min_box_area*cell_area\n    ## Boxes from Conv4 max-ovl 0.5\n    bcs_tmp=select_boxes(get_box_corners(mask,4,4)[4],min_area,maxovl,-1)\n    bcs=[]\n    for cs in bcs_tmp:\n        ## For each box, check sum(mask[box]) <= cell_area*cell_space*max_cells\n        n_cells=np.sum(mask[cs[0]:cs[1],cs[2]:cs[3]])//(cell_area*cell_space)\n        if verbose: print(n_cells)\n        if n_cells<=max_cells: bcs.append(cs)\n        else: ## Too 'large', need to split\n            ## Within box, get sub-boxes from Conv2 max-ovl 0.5\n            bmask=np.zeros((n,n))\n            bmask[cs[0]:cs[1],cs[2]:cs[3]]=mask[cs[0]:cs[1],cs[2]:cs[3]]\n            bbcs=select_boxes(get_box_corners(bmask,2,2)[2],min_area,maxovl,-1)\n            for ncs in bbcs:\n                if verbose:\n                    n_cells=np.sum(mask[ncs[0]:ncs[1],ncs[2]:ncs[3]])//(cell_area*cell_space)\n                    print(\" \",n_cells)\n                ## For each sub-box, check n_cells<=max_cells\n                ## if not, recursively split in 4\n                ## keep only those with n_cells>1\n                bbbcs=get_boxsplits(ncs,bmask,cell_area*cell_space)\n                for nncs in bbbcs:\n                    n_cells=np.sum(mask[nncs[0]:nncs[1],nncs[2]:nncs[3]])//(cell_area*cell_space)\n                    if verbose and nncs not in bbcs: print(\"  \",n_cells)\n                    if n_cells>1: bcs.append(nncs)\n    ## From all selected boxes, filter with max-ovl 1/3\n    bcs=select_boxes(bcs,min_area,maxovl_final,-1)\n    ## If no boxes, consider full image\n    if bcs==[]:\n        bcs=[[0,256,0,256]]\n    return bcs","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:02.773725Z","iopub.execute_input":"2023-01-30T19:19:02.774227Z","iopub.status.idle":"2023-01-30T19:19:02.795958Z","shell.execute_reply.started":"2023-01-30T19:19:02.774184Z","shell.execute_reply":"2023-01-30T19:19:02.794179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Show boxes on img0s\ndef plot_boxing(img,cornerdct):\n    plt.close()\n    plt.figure(figsize=(7,7))\n    i=-1\n    for fs,allcorners in cornerdct.items():\n        i+=1\n        for c in allcorners:\n            img[c[0]:c[1],c[2]]=255\n            img[c[0]:c[1],c[3]-1]=255\n            img[c[0],c[2]:c[3]]=255\n            img[c[1]-1,c[2]:c[3]]=255\n    plt.imshow(img,cmap='gray_r')","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:06.124968Z","iopub.execute_input":"2023-01-30T19:19:06.125411Z","iopub.status.idle":"2023-01-30T19:19:06.138443Z","shell.execute_reply.started":"2023-01-30T19:19:06.125372Z","shell.execute_reply":"2023-01-30T19:19:06.137422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions for selecting second-level 'cells' and third-level 'tiles'\nFrom within boxes, select cells (4096^2 pixels), and from within select tiles (1024^2 pixels).\nA cell/tile is picked according to some density threshold (th1 or th2).\nMax 8 boxes per image, 4 cells per box, and 4 tiles per cell. While in a box N_tiles<4*4, select more tiles from box. Boxes/cells/tiles are defined by their corners 'cs'","metadata":{}},{"cell_type":"code","source":"## Select tiles (wt*ht) within a box with corners 'cs' with den threshold 'th'\ndef get_tiles(img,wt,ht,cs,th,maxtiles):\n    if maxtiles<=0:\n        return [],[]\n    wc=cs[3]-cs[2]\n    hc=cs[1]-cs[0]\n    nw=wc//wt \n    nh=hc//ht\n    sws=2*nw-1 ## n steps of half-tile width\n    shs=2*nh-1 ## n steps of half-tile height\n    if nw==0 or nh==0:\n        return [],[]\n    dw=cs[2]+(wc-nw*wt)//2 ## start w index\n    dh=cs[0]+(hc-nh*ht)//2 ## start h index\n    allncs=[]\n    dens=[]\n    locs=[]\n#    print(cs)\n#     print(dh,dh+(shs+1)*ht/2)\n    for i in range(shs):\n        for j in range(sws):\n            ncs=[dh+i*ht//2,dh+(i+2)*ht//2,dw+j*wt//2,dw+(j+2)*wt//2]\n            den=np.sum(img[ncs[0]:ncs[1],ncs[2]:ncs[3]])/(wt*ht)\n            if den>th:\n                allncs.append(ncs)\n                dens.append(den)\n                locs.append([((ncs[1]+ncs[0])/2-cs[0])/hc,((ncs[3]+ncs[2])/2-cs[2])/wc])\n    if not allncs:\n        return [],[]\n    df=pd.DataFrame({'allncs':allncs,'dens':dens,'locs':locs})\n    df=df.sample(frac=1) ## shuffle\n    df.sort_values(by='dens',inplace=True,ascending=False) ## sort by decreasing density\n    df.reset_index(drop=True)\n    ## return max non-overlapping largest den\n    select=[]\n    slocs=[]\n    i=0\n    for i,row in df.iterrows():\n        ncs=row.allncs\n        isiso=True\n        for pncs in select:\n            ## no overlap with 1bin space\n            if ncs[0]>pncs[1] or ncs[1]<pncs[0] or ncs[2]>pncs[3] or ncs[3]<pncs[2]:\n                continue\n            isiso=False\n            break\n        if isiso:\n            select.append(ncs)\n            slocs.append(row.locs)\n            if len(select)==maxtiles:\n                return select,slocs\n    return select,slocs\n            \n## Create class of all selected boxes/cells/tiles and info\ndef get_tile_classes(omask,size,cell_area,boxcorners):\n    mask=np.copy(omask)\n    ## Define cell and tile sizes\n    [H,W]=size\n    wt1=round(cell_size*n/W)\n    ht1=round(cell_size*n/H)\n    wt2=round(tile_size*n/W)\n    ht2=round(tile_size*n/H)\n#    print(H,ht1,ht2,W,wt1,wt2)\n    ## Img\n    ImgC=img_class(size=size)\n    ## Boxes\n    sizes=[]\n    locs=[]\n    count_boxes=0\n    allcorners=[]\n    for cs in boxcorners:\n        n_cells=np.sum(mask[cs[0]:cs[1],cs[2]:cs[3]])//(cell_area*cell_space)\n        if n_cells==0 and count_boxes>=4: continue\n        count_boxes+=1\n        sizes.append([(cs[1]-cs[0])*H//n,(cs[3]-cs[2])*W//n])\n        locs.append([(cs[1]+cs[0])/(2*n),(cs[3]+cs[2])/(2*n)])\n        allcorners.append(cs)\n    ImgC.addboxes(sizes,locs)\n    ## Cells\n    for boxC in ImgC.boxes:\n        cs=allcorners[boxC.idx-2]\n        n_cells=np.sum(mask[cs[0]:cs[1],cs[2]:cs[3]])//(cell_area*cell_space)\n        if n_cells>0:\n            corners,locs=get_tiles(mask,wt1,ht1,cs,th1,min(n_cells,max_cells))\n            boxC.addcells(locs)\n            allcorners+=corners\n    ## CellTiles\n    for cellC in ImgC.cells:\n        cs=allcorners[cellC.idx-2]\n        corners,locs=get_tiles(mask,wt2,ht2,cs,th2,max_tiles)\n        cellC.addctiles(locs)\n        allcorners+=corners\n    ## BoxTiles\n    for boxC in ImgC.boxes:\n        if not boxC.cells:\n            cs=allcorners[boxC.idx-2]\n            corners,locs=get_tiles(mask,wt2,ht2,cs,th2,max_tiles)\n            boxC.addbtiles(locs)\n            allcorners+=corners\n    return ImgC,allcorners","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:09.535142Z","iopub.execute_input":"2023-01-30T19:19:09.535848Z","iopub.status.idle":"2023-01-30T19:19:09.585083Z","shell.execute_reply.started":"2023-01-30T19:19:09.535798Z","shell.execute_reply":"2023-01-30T19:19:09.580366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load and Save Utils\nFunctions for loading data, loading/normalizing/saving images, loading/saving info dicts.\nDefine classes to store all saved box/cell/tile+info. Save/Load classes from dict","metadata":{}},{"cell_type":"code","source":"## get list of images\n## taken from https://www.kaggle.com/code/datark1/eda-images-processing-and-exploration\n#from openslide import OpenSlide\ndef get_data(lab): # lab=train/test/other\n    df=pd.read_csv(\"../input/mayo-clinic-strip-ai/%s.csv\"%lab)\n    images = glob(\"/kaggle/input/mayo-clinic-strip-ai/%s/*\"%lab)\n    img_prop = defaultdict(list)\n    for i, path in enumerate(images):\n        img_path = images[i]\n#        slide = OpenSlide(path)    \n        img_prop['image_id'].append(img_path[-12:-4])\n#        img_prop['width'].append(slide.dimensions[0])\n#        img_prop['height'].append(slide.dimensions[1])\n#        img_prop['nbins'].append(slide.dimensions[0]*slide.dimensions[1])\n        img_prop['size'].append(round(os.path.getsize(img_path) / 1e6, 2))\n        img_prop['path'].append(img_path)\n\n    image_data = pd.DataFrame(img_prop)\n#    image_data['img_aspect_ratio'] = image_data['width']/image_data['height']\n    image_data.sort_values(by='size', inplace=True, ascending=False)\n    image_data.reset_index(inplace=True, drop=True)\n\n    image_data = image_data.merge(df, on='image_id')\n    image_data.head()\n    return image_data\n\n\n## Save or load dct to/from file\ndef loadjsonfile(pathfile):\n    if not os.path.isfile(pathfile):\n        f=open(pathfile,'w')\n        f.write('{}')\n        f.close()\n    with open(pathfile) as f:\n        return byteify(json.load(f))\ndef savejsonfile(obj,pathfile,indent=2,sort=True):\n    with open(pathfile,\"w\") as f:\n        json.dump(obj,f,indent=None,sort_keys=False)\ndef byteify(input):\n   # Encodes any input from unicode to normal string                                                                                                                           \n    if isinstance(input, dict):\n        return {byteify(key):byteify(value) for key,value in input.items()}\n    elif isinstance(input, list):\n        return [byteify(element) for element in input]\n    elif isinstance(input, str):\n        # return input.encode('utf-8')                                                                                                                                         \n        try:\n            return input.decode()\n        except:\n            return input\n    else:\n        return input","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:11.261775Z","iopub.execute_input":"2023-01-30T19:19:11.262183Z","iopub.status.idle":"2023-01-30T19:19:11.273483Z","shell.execute_reply.started":"2023-01-30T19:19:11.262147Z","shell.execute_reply":"2023-01-30T19:19:11.272405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## to split image if too large, return split corners\ndef get_splits(cs,th):\n    if cs[1]-cs[0]>th and cs[3]-cs[2]>th:\n        return get_splits([cs[0],(cs[0]+cs[1])//2,cs[2],(cs[2]+cs[3])//2],th)\\\n              +get_splits([cs[0],(cs[0]+cs[1])//2,(cs[2]+cs[3])//2,cs[3]],th)\\\n              +get_splits([(cs[0]+cs[1])//2,cs[1],cs[2],(cs[2]+cs[3])//2],th)\\\n              +get_splits([(cs[0]+cs[1])//2,cs[1],(cs[2]+cs[3])//2,cs[3]],th)\n    elif cs[1]-cs[0]>th:\n        return get_splits([cs[0],(cs[0]+cs[1])//2,cs[2],cs[3]],th)\\\n              +get_splits([(cs[0]+cs[1])//2,cs[1],cs[2],cs[3]],th)\n    elif cs[3]-cs[2]>th:\n        return get_splits([cs[0],cs[1],cs[0],(cs[2]+cs[3])//2],th)\\\n              +get_splits([cs[0],cs[1],(cs[2]+cs[3])//2,cs[3]],th)\n    else:\n        return [[cs[0],cs[1],cs[2],cs[3]]]\n    \n## return 256*256*3 img, mask, colorInfo=[mask_th, LAB mean, LAB std]\ndef get_subimages(imgid,img0):\n    img0s=cv2.resize(img0,(n,n)) ## rgb small\n    mask,mask_th=getmask(imgid,img0s) ## mask\n    ## get LAB mean/std in mask for norm\n    imgLAB=cv2.cvtColor(img0s,cv2.COLOR_RGB2LAB)\n    meanlab=np.mean(imgLAB[mask],axis=(0))\n    stdlab=np.std(imgLAB[mask],axis=(0))\n    return img0s,mask,[mask_th,list(meanlab),list(stdlab)]\n\n## get binary 256*256 mask (0 is background, 1 is tissue)\ndef getmask(imgid,img0s):\n    imgray=color.rgb2gray(img0s)\n    if imgid in ['48af1a_0','4f6fb1_0','d380dc_0']:\n        mask_th=np.mean(imgray)\n        mask=imgray<mask_th\n        return mask,mask_th\n    imgray1=np.copy(imgray)\n    first=True\n    while True:\n        mean=np.mean(imgray1)\n        std0=np.std(imgray1[imgray1<mean])\n        std1=np.std(imgray1[imgray1>mean])\n        diff=std1-0.1*std0\n#        print(mean,std0,std1,diff)\n        if (not first) and diff>0.01:\n            break\n        first=False\n        mask_th=mean+std1\n        mask=imgray<mask_th\n        imgray1=imgray1[imgray1<mask_th]\n    return mask,mask_th\n\n## normalize img color whithin mask else set to zero\ndef norm_img(img,colorInfo):\n    mask_th,meanlab,stdlab=colorInfo\n    mask=color.rgb2gray(img)<mask_th\n    imgLAB=cv2.cvtColor(img,cv2.COLOR_RGB2LAB)\n    imgLABN=((imgLAB-meanlab)/(stdlab/stdlab_ref)+meanlab_ref).astype(np.uint8)\n    imgN=cv2.cvtColor(imgLABN,cv2.COLOR_LAB2RGB)*mask[:,:,None]\n    return imgN","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:11.335481Z","iopub.execute_input":"2023-01-30T19:19:11.336486Z","iopub.status.idle":"2023-01-30T19:19:11.353726Z","shell.execute_reply.started":"2023-01-30T19:19:11.336448Z","shell.execute_reply":"2023-01-30T19:19:11.352725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Define class for storing meta data on selected box/cell/tile to save\n## All saved obj is 256*256*3, concatenated in a simgle numpy array. \n## The classes below are exported/loaded to json dicts per image.\n## Ordering of saved objects: original_img > boxes > cells > cell_tiles > box_tiles\nclass boxtile_class():\n    def __init__(self,idx,loc,box,img):\n        self.idx=idx\n        self.loc=loc ## loc in box\n        self.box=box\n        self.img=img\n        \nclass celltile_class():\n    def __init__(self,idx,loc,cell,box,img):\n        self.idx=idx\n        self.loc=loc ## loc in cell\n        self.cell=cell\n        self.box=box\n        self.img=img\n    \nclass cell_class():\n    def __init__(self,idx,loc,box,img):\n        self.idx=idx\n        self.loc=loc ## loc in box\n        self.box=box\n        self.img=img\n        self.ctiles=[]\n        self.tc_idxs=[0,0]\n    def addctiles(self,locs):\n        for c_idx,loc in enumerate(locs):\n            idx=self.img.next+0\n            self.img.next+=1\n            tc=celltile_class(idx,loc,self,self.box,self.img)\n            if self.tc_idxs[1]==0:\n                self.tc_idxs=[idx,idx]\n            self.tc_idxs[1]+=1\n            self.ctiles.append(tc)\n        for p in [self.box,self.img]:\n            p.ctiles+=self.ctiles\n            if p.tc_idxs[1]==0:\n                p.tc_idxs=self.tc_idxs+[]\n            else:\n                p.tc_idxs[1]+=len(self.ctiles)\n        \nclass box_class():\n    def __init__(self,idx,size,loc,img):\n        self.idx=idx\n        self.size=size ## size wrt img\n        self.loc=loc ## loc in img\n        self.img=img\n        self.cells=[]\n        self.btiles=[]\n        self.ctiles=[]\n        self.c_idxs=[0,0]\n        self.tb_idxs=[0,0]\n        self.tc_idxs=[0,0]\n    def addcells(self,locs):\n        for loc in locs:\n            idx=self.img.next+0\n            self.img.next+=1\n            c=cell_class(idx,loc,self,self.img)\n            if self.c_idxs[1]==0:\n                self.c_idxs=[idx,idx]\n            self.c_idxs[1]+=1\n            self.cells.append(c)\n        self.img.cells+=self.cells\n        if self.img.c_idxs[1]==0:\n            self.img.c_idxs=self.c_idxs+[]\n        else:\n            self.img.c_idxs[1]+=len(self.cells)\n    def addbtiles(self,locs):\n        for loc in locs:\n            idx=self.img.next+0\n            self.img.next+=1\n            tb=boxtile_class(idx,loc,self,self.img)\n            if self.tb_idxs[1]==0:\n                self.tb_idxs=[idx,idx]\n            self.tb_idxs[1]+=1\n            self.btiles.append(tb)\n        self.img.btiles+=self.btiles\n        if self.img.tb_idxs[1]==0:\n            self.img.tb_idxs=self.tb_idxs+[]\n        else:\n            self.img.tb_idxs[1]+=len(self.btiles)\n    def makeindex(self):\n        self.index=self.cells+self.ctiles+self.btiles\n        \nclass img_class():\n    def __init__(self,imgid=\"\",isplit=0,size=[],colorInfo=[]):\n        self.imgid=imgid\n        self.isplit=isplit\n        self.size=size ## orig size\n        self.colorInfo=colorInfo ## colorInfo\n        self.next=2 ## next idx in saved numpy array, 0 is img0s, 1 is img0s normalized with boxes \n        self.boxes=[]\n        self.cells=[]\n        self.btiles=[]\n        self.ctiles=[]\n        self.b_idxs=[0,0]\n        self.c_idxs=[0,0]\n        self.tb_idxs=[0,0]\n        self.tc_idxs=[0,0]\n    def addboxes(self,sizes,locs):\n        for size,loc in zip(sizes,locs):\n            idx=self.next+0\n            self.next+=1\n            b=box_class(idx,size,loc,self)\n            if self.b_idxs[1]==0:\n                self.b_idxs=[idx,idx]\n            self.b_idxs[1]+=1\n            self.boxes.append(b)\n    def makeindex(self):\n        self.index=self.boxes+self.cells+self.ctiles+self.btiles","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:13.556956Z","iopub.execute_input":"2023-01-30T19:19:13.557344Z","iopub.status.idle":"2023-01-30T19:19:13.579982Z","shell.execute_reply.started":"2023-01-30T19:19:13.557313Z","shell.execute_reply":"2023-01-30T19:19:13.57893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## save all obj classes in dict\ndef save_classes(ImgC):\n    dct={'imgid':ImgC.imgid,'isplit':ImgC.isplit,'size':ImgC.size,'colorInfo':ImgC.colorInfo,'boxes':[]}\n    for boxC in ImgC.boxes:\n        dct['boxes'].append({'size':boxC.size,'loc':boxC.loc,'cells':[],'btiles':[]})\n        for cellC in boxC.cells:\n            dct['boxes'][-1]['cells'].append({'loc':cellC.loc,'ctiles':[]})\n            for ctileC in cellC.ctiles:\n                dct['boxes'][-1]['cells'][-1]['ctiles'].append({'loc':ctileC.loc})\n        for btileC in boxC.btiles:\n            dct['boxes'][-1]['btiles'].append({'loc':btileC.loc})\n    savejsonfile(dct,\"save/book/%s_%s.json\"%(ImgC.imgid,ImgC.isplit))\n\n## rebuild obj classes from dict\ndef make_classes(imglabel,tile_path=\"\"):\n    if tile_path==\"\":\n        tile_path=\"save/book/%s.json\"%imglabel\n    book_path=tile_path.replace('tile','book').replace('.npy','.json')\n    if not os.path.isfile(book_path):\n        print(\"WARNING: File %s not found\"%book_path)\n        return\n    dct=loadjsonfile(book_path)\n    ImgC=img_class(dct['imgid'],dct['isplit'],dct['size'],dct['colorInfo'])\n    ## Boxes\n    sizes=[]\n    locs=[]\n    for b,box in enumerate(dct['boxes']):\n        sizes.append(box['size'])\n        locs.append(box['loc'])\n    ImgC.addboxes(sizes,locs)\n    ## Cells\n    for b,box in enumerate(dct['boxes']):\n        boxC=ImgC.boxes[b]\n        locs=[]\n        for cell in box['cells']:\n            locs.append(cell['loc'])\n        boxC.addcells(locs)\n    ## CellTiles\n    for b,box in enumerate(dct['boxes']):\n        boxC=ImgC.boxes[b]\n        for c,cell in enumerate(box['cells']):\n            cellC=boxC.cells[c]\n            locs=[]\n            for ctile in cell['ctiles']:\n                locs.append(ctile['loc'])\n            cellC.addctiles(locs)\n    ## BoxTiles\n    for b,box in enumerate(dct['boxes']):\n        boxC=ImgC.boxes[b]\n        locs=[]\n        for btile in box['btiles']:\n            locs.append(btile['loc'])\n        boxC.addbtiles(locs)\n    return ImgC\n\n## print info of stored class objs (verbose=0/1/2)\ndef print_classes(ImgC,verbose=0):\n    if verbose==2:\n        print(\"ImgSize:\",ImgC.size,\", ColorInfo:\",ImgC.colorInfo,\"\\n\")\n    ImgC.makeindex()\n    print(\"%s-tot %s-boxes%s %s-cells%s %s-ctiles%s %s-btiles%s\"%(len(ImgC.index),\n                                                                      len(ImgC.boxes),ImgC.b_idxs,\n                                                                      len(ImgC.cells),ImgC.c_idxs,\n                                                                      len(ImgC.ctiles),ImgC.tc_idxs,\n                                                                      len(ImgC.btiles),ImgC.tb_idxs))\n    if verbose:\n        for boxC in ImgC.boxes:\n            boxC.makeindex()\n            print(\" Box%s: %s-tot %s-cells%s %s-ctiles%s %s-btiles%s\"%(boxC.idx,len(boxC.index),\n                                                                  len(boxC.cells),boxC.c_idxs,\n                                                                  len(boxC.ctiles),boxC.tc_idxs,\n                                                                  len(boxC.btiles),boxC.tb_idxs))\n            if verbose==2:\n                print(\" BoxSize: %.2f,%.2f\"%(boxC.size[0],boxC.size[1]),\", BoxLoc: %.2f,%.2f\"%(boxC.loc[0],boxC.loc[1]))\n            for cellC in boxC.cells:\n                print(\"  Cell%s: %s-ctiles%s\"%(cellC.idx,len(cellC.ctiles),cellC.tc_idxs))\n                if verbose==2:\n                    print(\"  CellLoc: %.2f,%.2f\"%(cellC.loc[0],cellC.loc[1]))\n                    if cellC.ctiles:\n                        print(\"   CTileLocs\",[\"%.2f,%.2f\"%(ct.loc[0],ct.loc[1]) for ct in cellC.ctiles])\n            if verbose==2 and boxC.btiles:\n                print(\"   BTileLocs\",[\"%.2f,%.2f\"%(bt.loc[0],bt.loc[1]) for bt in boxC.btiles])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:14.643044Z","iopub.execute_input":"2023-01-30T19:19:14.643408Z","iopub.status.idle":"2023-01-30T19:19:14.663571Z","shell.execute_reply.started":"2023-01-30T19:19:14.643376Z","shell.execute_reply":"2023-01-30T19:19:14.662558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scale_tile_corners(rcs,size):\n    H,W=size\n    ht=rcs[1]-rcs[0]\n    dh=(n-ht)//2\n    hstart=max(0,rcs[0]-dh)\n    if hstart+n>H:\n        hstart-=hstart+n-H\n    wt=rcs[3]-rcs[2]\n    dw=(n-wt)//2 \n    wstart=max(0,rcs[2]-dw)\n    if wstart+n>W:\n        wstart-=wstart+n-W\n    return [hstart,hstart+n,wstart,wstart+n]\n\n## save selected box/tile as 256*256*3 numpy array (temporary)\ndef save_tile(img0,label,rcs,colorInfo,istile):\n    if istile:\n        tile=img0[rcs[0]:rcs[1],rcs[2]:rcs[3],:]\n    else:\n        tile=cv2.resize(img0[rcs[0]:rcs[1],rcs[2]:rcs[3],:],(n,n),interpolation=cv2.INTER_AREA)\n    tile=norm_img(tile,colorInfo)\n    np.save(\"tmp/\"+label+'.npy',tile)\n\n## loop on all corners selected for image, save tiles (after color norm) in concat array\ndef save_tiles(imglabel,img0,img0s,colorInfo,size,allcorners,ImgC):\n    ntiles=len(allcorners)\n    [H,W]=size\n    ## Save all selected tiles separately\n    if not os.path.isdir(\"tmp\"):\n        os.makedirs(\"tmp\")\n    first_tile_index=ImgC.tc_idxs[0]-2 if ImgC.tc_idxs[0] else ImgC.tb_idxs[0]-2\n    for i,cs in enumerate(allcorners):\n        rcs=[cs[0]*H//n, cs[1]*H//n, cs[2]*W//n, cs[3]*W//n]\n        istile=False\n        ## if istile, scale to 256*256 and no resize (always comes out smaller)\n        if i>0 and i>=first_tile_index:\n            istile=True\n#            print(rcs,rcs[1]-rcs[0],rcs[3]-rcs[2])\n            rcs=scale_tile_corners(rcs,size)\n#            print(rcs,rcs[1]-rcs[0],rcs[3]-rcs[2])\n        save_tile(img0,str(i),rcs,colorInfo,istile)\n    ## Get normalized img0s with boxes\n    img0N=norm_img(img0s,colorInfo)\n    white=[255,255,255]\n    for idxs in [ImgC.b_idxs,ImgC.c_idxs,ImgC.tc_idxs,ImgC.tb_idxs]:\n        if idxs[1]==0: continue\n        for i in range(idxs[0],idxs[1]):\n            cs=allcorners[i-2]\n            img0N[cs[0]:cs[1],cs[2]]=white\n            img0N[cs[0]:cs[1],cs[3]-1]=white\n            img0N[cs[0],cs[2]:cs[3]]=white\n            img0N[cs[1]-1,cs[2]:cs[3]]=white\n    ## Save together\n    tiles=np.zeros((ntiles+2,n,n,3))\n    tiles[0,:,:,:]=img0s\n    tiles[1,:,:,:]=img0N\n    for i in range(ntiles):\n        tiles[i+2,:,:,:]=np.load(\"tmp/%s.npy\"%i)\n    tiles=tiles.astype(np.uint8)\n    np.save(\"save/tile/%s.npy\"%imglabel,tiles)\n    shutil.rmtree(\"tmp\")","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:15.515473Z","iopub.execute_input":"2023-01-30T19:19:15.51627Z","iopub.status.idle":"2023-01-30T19:19:15.537149Z","shell.execute_reply.started":"2023-01-30T19:19:15.516228Z","shell.execute_reply":"2023-01-30T19:19:15.536099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{}},{"cell_type":"code","source":"## Parameters #########################################################################################################################\nskip_imgs=['7b9aaa_0','2268cf_0','280c26_0','79d52a_1','b43ebe_0']\n\nn=256 # output img size \nmeanlab_ref=np.array([83.1,163.3,151.5]) ## ref mean color in LAB \nstdlab_ref=np.array([44.1,16.9,21.5]) ## ref std color in LAB\n\n## First-level BOX\nfilter_sizes=[4,2] ## considered sizes of conv for box selection \nbox_thdct={4:0.4, 2:0.5} ## density thresholds for selecting box (per filter size)\nmin_box_area=0.8 # minimum box_area in units of cell_area\ncell_space=4 # reserved cell_area space per cell\nmaxovl=0.5 # max overlap btw boxes\nmaxovl_final=1/3 # final max ovl\nmaxboxes=-1 # max boxes per img\n\n## Second-level CELL\ncell_size=2048 # cell size in original pixels\nth1=0.55  # cell threshold density\nmax_cells=6 # max cells per box\n\n## Third-level TILE\ntile_size=256 # tile size in original pixels\nth2=0.75 # tile threshold density\nmax_tiles=4 # max tiles per cell or box\n###########################################################################################################################","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:16.745624Z","iopub.execute_input":"2023-01-30T19:19:16.745968Z","iopub.status.idle":"2023-01-30T19:19:16.753358Z","shell.execute_reply.started":"2023-01-30T19:19:16.745937Z","shell.execute_reply":"2023-01-30T19:19:16.751869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Doing the stuff\nRun on images, get selected boxes/cells/tiles, save as numpy arrays.\nSave dct with all info to use later","metadata":{}},{"cell_type":"code","source":"## Prep\nfor dirr in ['save','save/book','save/tile']:\n    if not os.path.isdir(dirr):\n        os.makedirs(dirr)\nimage_data=get_data(\"test\")\n#image_data=get_data(\"train\")\n#image_data=get_data(\"other\")\n\n## Start\nlimit=-1\ncount=0\nistart=0\nfor im_idx in range(istart,len(image_data)):\n    if count==limit:\n        break\n    ## Check status\n    imgid=image_data[\"image_id\"][im_idx]\n    if imgid in skip_imgs: # skip bad images \n        continue\n#     if imgid!='008e5c_0':\n#         continue\n    if os.path.isfile(\"save/tile/%s_0.npy\"%imgid):\n        continue\n    print(\"Processing:\",im_idx,imgid)\n    ## Get full image, if too large split it\n    imgpath=image_data['path'][im_idx]\n    img0=io.imread(imgpath) ## full\n    H,W,_=img0.shape\n    splits=get_splits([0,H,0,W],256*256)\n    for isplit,cs in enumerate(splits):\n        imglabel=\"%s_%s\"%(imgid,isplit)\n        ## Get split image\n        if isplit>0:\n            img0=io.imread(imgpath)\n        img0=img0[cs[0]:cs[1],cs[2]:cs[3]]\n        H,W,_=img0.shape\n        ## Get processed images\n        img0s,mask,colorInfo=get_subimages(imgid,img0) ## small image, mask, colorInfo for norm\n        ## Get box corners\n        cell_area=(cell_size*n)**2/(H*W)\n        boxcorners=get_boxes(mask,cell_area,verbose=False)#True)\n        ## Get cell and tile corners, store all info\n        ImgC,allcorners=get_tile_classes(mask,[H,W],cell_area,boxcorners)\n        ImgC.imgid=imgid\n        ImgC.isplit=isplit\n        ImgC.colorInfo=colorInfo\n        ## Make tiles and save all\n        save_tiles(imglabel,img0,img0s,colorInfo,[H,W],allcorners,ImgC)\n        save_classes(ImgC)\n        print_classes(ImgC,verbose=0)\n        ## Free memory\n        del img0\n        del img0s\n        del mask\n        del boxcorners\n        del ImgC\n        del allcorners\n        gc.collect()\n    count+=1\nprint(\"Done\")\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:19:18.463911Z","iopub.execute_input":"2023-01-30T19:19:18.464316Z","iopub.status.idle":"2023-01-30T19:20:47.571921Z","shell.execute_reply.started":"2023-01-30T19:19:18.464283Z","shell.execute_reply":"2023-01-30T19:20:47.570775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#imgid='006388_0'\nimglabel=\"%s_%s\"%(imgid,isplit)\nImgC=make_classes(imglabel)\nprint_classes(ImgC,verbose=2)\ntiles=np.load(\"save/tile/%s.npy\"%imglabel)\nprint(tiles.shape)\nfig,((ax1,ax2,ax3,ax4),(bx1,bx2,bx3,bx4)) = plt.subplots(2,4,figsize=(20,10))\nax1.imshow(tiles[0])\nax2.imshow(tiles[1])\nax3.imshow(tiles[2])\nax4.imshow(tiles[ImgC.b_idxs[1]-1])\nif ImgC.cells:\n    bx1.imshow(tiles[ImgC.c_idxs[0]])\n    bx2.imshow(tiles[ImgC.c_idxs[1]-1])\n    bx3.imshow(tiles[ImgC.c_idxs[1]])\nelse:\n    bx1.imshow(tiles[ImgC.b_idxs[1]])\n    bx2.imshow(tiles[ImgC.b_idxs[1]+1])\n    bx3.imshow(tiles[ImgC.b_idxs[1]+2])    \nbx4.imshow(tiles[-1])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:47.574163Z","iopub.execute_input":"2023-01-30T19:20:47.574556Z","iopub.status.idle":"2023-01-30T19:20:48.687916Z","shell.execute_reply.started":"2023-01-30T19:20:47.574519Z","shell.execute_reply":"2023-01-30T19:20:48.686559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.imshow(tiles[1])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:48.689211Z","iopub.execute_input":"2023-01-30T19:20:48.689971Z","iopub.status.idle":"2023-01-30T19:20:49.003337Z","shell.execute_reply.started":"2023-01-30T19:20:48.689937Z","shell.execute_reply":"2023-01-30T19:20:49.002179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Execute","metadata":{}},{"cell_type":"code","source":"## get list of images\ndef get_processed_data(lab): # lab=train/test/other\n    df=pd.read_csv(\"../input/mayo-clinic-strip-ai/%s.csv\"%lab)\n    tiles=sorted(glob(\"save/tile/*\"))\n    dct={'imglabel':[],'image_id':[],'tile_path':[]}\n    for tile_path in tiles:\n        imglabel=tile_path.rsplit('/',1)[1].split('.')[0]\n        dct['imglabel'].append(imglabel)\n        dct['image_id'].append(imglabel.rsplit('_',1)[0])\n        dct['tile_path'].append(tile_path)\n    image_data=pd.DataFrame(dct)\n    image_data=image_data.merge(df,on='image_id')\n#    image_data.sort_values('image_id',inplace=True)\n    image_data.reset_index(inplace=True,drop=True)\n    return image_data","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:49.00617Z","iopub.execute_input":"2023-01-30T19:20:49.006768Z","iopub.status.idle":"2023-01-30T19:20:49.015395Z","shell.execute_reply.started":"2023-01-30T19:20:49.006731Z","shell.execute_reply":"2023-01-30T19:20:49.014394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=get_processed_data(\"train\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:49.017071Z","iopub.execute_input":"2023-01-30T19:20:49.017655Z","iopub.status.idle":"2023-01-30T19:20:49.056871Z","shell.execute_reply.started":"2023-01-30T19:20:49.01762Z","shell.execute_reply":"2023-01-30T19:20:49.05594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset,Sampler,DataLoader,SubsetRandomSampler\nfrom torchvision import models","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:49.061056Z","iopub.execute_input":"2023-01-30T19:20:49.06325Z","iopub.status.idle":"2023-01-30T19:20:49.84532Z","shell.execute_reply.started":"2023-01-30T19:20:49.063213Z","shell.execute_reply":"2023-01-30T19:20:49.844271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Get Dataset\nclass bdDataset(Dataset):\n    def __init__(self,df,imglabels):\n        cid_max=11\n        bsize_max=256*256\n        self.paths=[]\n        self.labels=[]\n        self.b_idxs=[]\n        self.b_meta=[] ## [center_id,H,W]\n        self.c_idxs=[]\n        self.c_meta=[] ## [1 or .25,hloc,wloc]\n        self.t_idxs=[]\n        self.t_meta=[] ## [hloc,wloc]\n        self.has_cell=[]\n        for imglabel in imglabels:\n            row=df.loc[df['imglabel']==imglabel].iloc[0]\n            ImgC=make_classes(imgid,row.tile_path)\n            for box in ImgC.boxes:\n                has_cell=len(box.cells)>0                \n                self.paths.append(row.tile_path)\n                self.labels.append(int(row.label=='CE'))\n                self.has_cell.append(int(has_cell))\n                self.b_idxs.append(box.idx)\n                self.b_meta.append([row.center_id/cid_max, box.size[0]/bsize_max, box.size[1]/bsize_max])\n                if False: ## PATCH\n                    self.c_idxs.append(0)                    \n                    self.c_meta.append(0)   \n                    self.t_idxs.append(0)                    \n                    self.t_meta.append(0)   \n                elif has_cell:\n                    self.c_idxs.append(box.c_idxs)\n                    self.c_meta.append([[1.]+cell.loc for cell in box.cells])\n                    self.t_idxs.append(box.tc_idxs)                    \n                    self.t_meta.append([])\n                    for cell in box.cells:\n                        self.t_meta[-1].append([ctile.loc for ctile in cell.ctiles])             \n                else:\n                    self.c_idxs.append(box.tb_idxs)\n                    self.c_meta.append([[.25]+btile.loc for btile in box.btiles])\n                    self.t_idxs.append(0)                    \n                    self.t_meta.append(0)                    \n                    \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self,i):\n        tiles=np.load(self.paths[i])\n        has_cell=self.has_cell[i]\n        b_tile=tiles[self.b_idxs[i]].transpose(2,0,1)/255\n#         ## PATCH\n#         c_tiles=0\n#         t_tiles=0\n        c_tiles=tiles[self.c_idxs[i][0]:self.c_idxs[i][1]].transpose(0,3,1,2)/255\n        t_tiles=tiles[self.t_idxs[i][0]:self.t_idxs[i][1]].transpose(0,3,1,2)/255 if has_cell else 0\n        return self.paths[i],self.b_idxs[i],b_tile,self.b_meta[i],c_tiles,self.c_meta[i],t_tiles,self.t_meta[i],self.labels[i]\n\n# train_dataset=bdDataset(df,train_imglabels)\n# valid_dataset=bdDataset(df,valid_imglabels)\n# print(len(train_dataset),len(valid_dataset))\ntest_imglabels=[row.imglabel for _,row in df.iterrows()]\ntest_dataset=bdDataset(df,test_imglabels)\nprint(len(test_dataset))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:49.846832Z","iopub.execute_input":"2023-01-30T19:20:49.847865Z","iopub.status.idle":"2023-01-30T19:20:49.876844Z","shell.execute_reply.started":"2023-01-30T19:20:49.847827Z","shell.execute_reply":"2023-01-30T19:20:49.875582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Get DataLoaders\ndef my_collate(batch):\n    # imgpaths,b_idxs,b_tiles,b_metas,c_tiles,c_metas,t_tiles,t_metas,labels\n    n_batch=len(batch)\n    n_outs=len(batch[0])\n    outs=[]\n    for j in range(n_outs):\n        jouts=[batch[i][j] for i in range(n_batch)]\n        outs.append(jouts)\n    imgpaths,b_idxs,b_tiles,b_metas,c_tiles,c_metas,t_tiles,t_metas,labels=outs\n    b_tiles=[torch.from_numpy(o).unsqueeze(0) for o in b_tiles]\n    b_tiles=torch.cat(b_tiles,dim=0).float() ## tensor(n_batch*img)\n    b_metas=[torch.FloatTensor(o).unsqueeze(0) for o in b_metas]\n    b_metas=torch.cat(b_metas,dim=0) ## tensor(n_batch*3)\n#     ## PATCH\n#     c_tiles=torch.empty(0)\n#     c_metas=torch.empty(0)   \n#     t_tiles=torch.empty(0)\n#     t_metas=torch.empty(0) \n    c_tiles=[torch.from_numpy(o) for o in c_tiles]\n    c_tiles=torch.cat(c_tiles,dim=0).float() ## tensor(sum(n_cells)*img)\n    c_metas=[torch.FloatTensor(o) for o in c_metas] ## list(n_batch)[tensor(n_cells,3)]\n    if t_metas[0]!=0:\n        t_tiles=[torch.from_numpy(o) for o in t_tiles]\n        t_tiles=torch.cat(t_tiles,dim=0).float() ## tensor(sum(n_tiles)*img)\n        tmp_t_metas=[]\n        for i in range(n_batch):\n            tmp_t_metas.append([torch.FloatTensor(o) for o in t_metas[i]])\n        t_metas=tmp_t_metas ## list(n_batch)[list(n_cells)[tensor(n_tiles,3)]]\n    else:\n        t_tiles=torch.empty(0)\n        t_metas=torch.empty(0)\n    labels=torch.tensor(labels) ## tensor(n_batch)\n    return [imgpaths,b_idxs,b_tiles,b_metas,c_tiles,c_metas,t_tiles,t_metas,labels]\n\n## split boxes with/without cells in separate batches\nclass my_sampler(Sampler):\n    def __init__(self,split_indices,batch_size):\n        self.split_indices=split_indices ## list of list of indices, batches created per sublists\n        self.batch_size=batch_size\n        self.batches={}\n        self.n_batches=0\n        self.generate_batches()\n    def generate_batches(self):\n        batch_idx=0\n        for indices in self.split_indices:\n            n=len(indices)\n            n_batches=n//self.batch_size+1\n            if n>0 and n%self.batch_size==0:\n                n_batches-=1\n            batches=np.array_split(np.random.permutation(indices),n_batches)\n            for batch in batches:\n                self.batches[batch_idx]=batch\n                batch_idx+=1\n        self.n_batches=batch_idx\n    def __len__(self):\n        return self.n_batches\n    def __iter__(self):\n        self.generate_batches()\n        batch_order=np.random.permutation(np.arange(self.n_batches))\n        for i in batch_order:\n            yield self.batches[i]\n\nbatch_size=1#len(test_dataset)\ndataloaders={}\n# for phase,dataset in zip(['train','valid'],[train_dataset,valid_dataset]):\nfor phase,dataset in zip(['test'],[test_dataset]):\n    yescell=[i for i,j in enumerate(dataset.has_cell) if j]\n    nocell=[i for i,j in enumerate(dataset.has_cell) if not j]\n    print(\"yescell:\",len(yescell),\", nocell:\",len(nocell))\n    split_indices=[]\n    if len(yescell)>0:\n        split_indices.append(yescell)\n    if len(nocell)>0:\n        split_indices.append(nocell)\n    sampler=my_sampler(split_indices,batch_size)\n    dataloaders[phase]=DataLoader(dataset,batch_sampler=sampler,collate_fn=my_collate)\n    \n# dataloaders={'train':DataLoader(train_dataset,batch_size=batch_size,shuffle=True,collate_fn=my_collate),\n#              'valid':DataLoader(valid_dataset,batch_size=batch_size,shuffle=True,collate_fn=my_collate)}\n# print(len(dataloaders['train']),len(dataloaders['valid']))\nprint(\"batches:\",len(dataloaders['test']))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:20:49.880707Z","iopub.execute_input":"2023-01-30T19:20:49.881085Z","iopub.status.idle":"2023-01-30T19:20:49.903205Z","shell.execute_reply.started":"2023-01-30T19:20:49.881053Z","shell.execute_reply":"2023-01-30T19:20:49.901797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_densemodel():\n#     model=models.densenet121(pretrained=True)\n    model=models.densenet121(pretrained=False)    \n    ## turn off grad descent\n    for name,param in model.named_parameters():\n        param.requires_grad=False\n    ## turn off classifier\n    model.classifier=nn.Identity()\n    return model\ndef printmodelfeatures(model,first=5,last=5):\n    n_feats=sum(1 for f in model.named_parameters())\n    if first>0:\n        print('- first')\n    for i,(name,params) in enumerate(model.named_parameters()):\n        if i==n_feats-5:\n            print('- last')\n        if i<first or n_feats-i<=5:\n            print(name,' ',list(params.shape),' ',params.requires_grad)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:14.195508Z","iopub.execute_input":"2023-01-30T19:24:14.195905Z","iopub.status.idle":"2023-01-30T19:24:14.203682Z","shell.execute_reply.started":"2023-01-30T19:24:14.195873Z","shell.execute_reply":"2023-01-30T19:24:14.202655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class mybdmodel(nn.Module):\n    def __init__(self):\n        super(mybdmodel,self).__init__()\n        ## Sub-features, sf*sf_iter total (pick sf_iter chunks of sf)\n        self.sf=64 ## n_sub_features\n        self.sf_iter=4 ## iter_sub_features \n        \n        ## classifier\n        self.classifier=nn.Linear(256,2)\n        \n        ## box features\n        self.dense_box=load_densemodel() ## out 1024\n        self.linear_box=nn.Sequential(\n            nn.Dropout(0),\n            nn.Linear(1024+3,256), ## 1024 box features + 3 box meta (center_id,H,W)\n            nn.ReLU(),\n            nn.Dropout(0)) ## out 512\n        ## box-cell combine\n        self.combiner_box_cell=nn.Sequential(\n            nn.Linear(256+self.sf+3,256), ## 512 box features + sf cell features + 3 cell meta (size,loc)\n            nn.ReLU()) ## out 512\n        \n        ## cell features\n        self.dense_cell=load_densemodel() ## out 1024 \n        self.linear_cell=nn.Sequential(\n            nn.Dropout(0),\n            nn.Linear(1024+1,256), ## 1024 cell features + center_id\n            nn.ReLU(),\n            nn.Dropout(0)) ## out 512\n        ## cell-tile combine\n        self.combiner_cell_tile=nn.Sequential(\n            nn.Linear(256+self.sf+2,self.sf*self.sf_iter), ## 512 cell features + sf tile features + 2 tile meta (loc)\n            nn.ReLU()) ## out sf_tot\n        \n        ## tile features (used also by cell if cell is tile)\n        self.dense_tile=load_densemodel() ## out 1024\n        self.linear_tile=nn.Sequential(\n            nn.Dropout(0),\n            nn.Linear(1024+1,self.sf*self.sf_iter), ## 1024 tile features + center_id\n            nn.ReLU(),\n            nn.Dropout(0)) ## out sf_tot\n        \n    def get_init_features(self,x,meta,dense,linear):\n        x=dense(x)\n        x=torch.cat([meta,x],dim=1)\n        return linear(x)\n            \n    def get_max_features(self,x,meta,split):\n        ## cell - x: sum(n_cells)*sf_tot, split=n_cells, N=n_boxs, n_i=n_cells_i\n        ## tile - x: sum(n_tiles_percell)*sf_tot, split=n_tiles_percell, N=n_cells, n_i=n_tiles_percell_i\n        N=len(split)\n        x=list(torch.split(x,split,dim=0)) ## list(N)[n_i*sf_tot]\n        for i in range(N):\n            x[i]=list(torch.split(x[i],self.sf,dim=1)) ## list(sf_iter)[n_i*sf]\n            xsums=[torch.sum(xsf,dim=1) for xsf in x[i]] ## list(sf_iter)[n_i]\n            max_idxs=[]\n            for xsum in xsums:\n                _,imax=torch.max(xsum,dim=0)\n                max_idxs.append(imax.cpu().data.numpy())\n            for j in range(self.sf_iter):\n                imax=max_idxs[j]\n                xmax=x[i][j][imax] ## sf                \n                meta_max=meta[i][imax].squeeze(dim=0) \n                x[i][j]=torch.cat([meta_max,xmax],dim=0).unsqueeze(1) ## (sf+meta)*1\n            x[i]=torch.cat(x[i],dim=1).unsqueeze(0) ## 1*(sf+meta)*sf_iter\n        x=torch.cat(x,dim=0) ## N*(sf+meta)*sf_iter\n        x=[xsf.squeeze(dim=2) for xsf in torch.split(x,1,dim=2)] ## list(sf_iter)[N*(sf+meta)]\n        return x\n    \n    def get_combined_features(self,x,list_sf,combiner):\n        ## box/cell - x: n_boxs*512 / list_sf: list(sf_iter)[n_boxs*(sf+3meta)] - n=n_boxs / n_out=512\n        ## cell/tile - x: n_cells*512 / list_sf: list(sf_iter)[n_cells*(sf+2meta)] - n=n_cells_i / n_out=sf\n        xc=[]\n        for sf in list_sf:\n            xsf=torch.cat([sf,x],dim=1) ## n*(512+sf+meta)\n            xsf=combiner(xsf) ## n*n_out\n            xc.append(xsf.unsqueeze(-1)) ## n*n_out*1\n        xc=torch.cat(xc,dim=2) ## n*n_out*sf_iter\n        xc,_=torch.max(xc,dim=2) ## n*n_out\n        return xc\n    \n    def forward(self,b,b_meta,c,c_meta,t,t_meta):\n        ## Count objects\n        has_cell=len(t_tiles)>0\n        n_boxs=b.shape[0]\n        n_cells=[cm.shape[0] for cm in c_meta]\n        if has_cell:\n            n_tiles_perbox=[]\n            n_tiles_percell=[]\n            for i in range(n_boxs):\n                n_tiles_percell.append([tm.shape[0] for tm in t_meta[i]])\n                n_tiles_perbox.append(sum(n_tiles_percell[-1]))\n        \n        ## Get box init features\n        b=self.get_init_features(b,b_meta,self.dense_box,self.linear_box) ## n_boxs*512\n        \n        ## Get cell init features\n        cid_meta=[]\n        for i in range(n_boxs):\n            cid_meta+=[b_meta[i][0].unsqueeze(0)]*n_cells[i]\n        cid_meta=torch.cat(cid_meta,dim=0).unsqueeze(1) ## sum(n_cells)*1\n        if has_cell:\n            c=self.get_init_features(c,cid_meta,self.dense_cell,self.linear_cell) ## sum(n_cells)*512\n            \n            ## Get tile init features\n            cid_meta=[]\n            for i in range(n_boxs):\n                cid_meta+=[b_meta[i][0].unsqueeze(0)]*n_tiles_perbox[i]\n            cid_meta=torch.cat(cid_meta,dim=0).unsqueeze(1) ## sum(n_tiles)*1\n            t=self.get_init_features(t,cid_meta,self.dense_tile,self.linear_tile) ## sum(n_tiles)*sf\n            \n            ## Split (cell and tile) per box\n            c=list(torch.split(c,n_cells,dim=0)) ## list(n_boxs)[n_cells*512]\n            t=list(torch.split(t,n_tiles_perbox,dim=0)) ## list(n_boxs)[n_tiles_perbox*512]\n            for i in range(n_boxs):\n                \n                ## Get tile max features\n                t[i]=self.get_max_features(t[i],t_meta[i],n_tiles_percell[i]) ## list(sf_iter)[n_cells*(sf+2meta)]\n                \n                ## Get cell+tile features\n                c[i]=self.get_combined_features(c[i],t[i],self.combiner_cell_tile) ## n_cells*sf\n            c=torch.cat(c,dim=0) ## sum(n_cells)*sf\n\n        else:\n            c=self.get_init_features(c,cid_meta,self.dense_tile,self.linear_tile) ## sum(n_cells)*sf\n            \n        ## Get cell max features\n        c=self.get_max_features(c,c_meta,n_cells) ## list(sf_iter)[n_boxs*(sf+3meta)]\n        \n        ## Get box+cell features\n        b=self.get_combined_features(b,c,self.combiner_box_cell) ## n_boxs*512\n        \n        ## Classify\n        if False: ## skip cells and tiles\n            b=torch.cat([torch.zeros(n_boxs,self.sf+3).to(device),b],dim=1) ## n_boxs*516\n            b=self.combiner_box_cell(b) ## n_boxs*512\n        return self.classifier(b)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:15.825851Z","iopub.execute_input":"2023-01-30T19:24:15.826236Z","iopub.status.idle":"2023-01-30T19:24:15.855666Z","shell.execute_reply.started":"2023-01-30T19:24:15.826202Z","shell.execute_reply":"2023-01-30T19:24:15.854495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel=mybdmodel().to(device)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:22.929877Z","iopub.execute_input":"2023-01-30T19:24:22.930594Z","iopub.status.idle":"2023-01-30T19:24:26.682434Z","shell.execute_reply.started":"2023-01-30T19:24:22.93055Z","shell.execute_reply":"2023-01-30T19:24:26.681279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"../input/mybdnw/model_boom_statedct.pt\",map_location=torch.device(device)))","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:34.996296Z","iopub.execute_input":"2023-01-30T19:24:34.996663Z","iopub.status.idle":"2023-01-30T19:24:35.981083Z","shell.execute_reply.started":"2023-01-30T19:24:34.99663Z","shell.execute_reply":"2023-01-30T19:24:35.980068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\nall_tpaths=[]\nall_outputs=[]\nfor i,(tpaths,b_idx,b_tiles,b_metas,c_tiles,c_metas,t_tiles,t_metas,_) in enumerate(dataloaders['test']):\n    b_tiles=b_tiles.to(device)\n    b_metas=b_metas.to(device)\n    c_tiles=c_tiles.to(device)\n    t_tiles=t_tiles.to(device)\n    for i in range(len(c_metas)):\n        c_metas[i]=c_metas[i].to(device)\n        if len(t_tiles)>0:\n            for j in range(len(t_metas[i])):\n                t_metas[i][j]=t_metas[i][j].to(device)\n    outputs=model(b_tiles,b_metas,c_tiles,c_metas,t_tiles,t_metas)\n    outputs=torch.nn.Softmax(dim=1)(outputs).cpu().data.numpy()\n    all_tpaths.append(tpaths)\n    all_outputs.append(outputs)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:38.72279Z","iopub.execute_input":"2023-01-30T19:24:38.723231Z","iopub.status.idle":"2023-01-30T19:24:45.668764Z","shell.execute_reply.started":"2023-01-30T19:24:38.723195Z","shell.execute_reply":"2023-01-30T19:24:45.66779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dct={}\nfor tpaths,outputs in zip(all_tpaths,all_outputs):\n    for i,tpath in enumerate(tpaths):\n        pid=tpath.rsplit('/',1)[-1].split('_',1)[0]\n        if pid not in dct:\n            dct[pid]={'CE':[],'LAA':[]}\n        dct[pid]['LAA'].append(outputs[i][0])\n        dct[pid]['CE'].append(outputs[i][1])","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:46.632727Z","iopub.execute_input":"2023-01-30T19:24:46.633118Z","iopub.status.idle":"2023-01-30T19:24:46.640026Z","shell.execute_reply.started":"2023-01-30T19:24:46.633083Z","shell.execute_reply":"2023-01-30T19:24:46.638869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_ids=[]\nlass=[]\nces=[]\nfor pid in dct:\n    patient_ids.append(pid)\n    max_laa=max(dct[pid]['LAA'])\n    max_ce=max(dct[pid]['CE'])\n    if max_laa>max_ce:\n        lass.append(max_laa)\n        ces.append(1-max_laa)\n    else:\n        lass.append(1-max_ce)\n        ces.append(max_ce)\n\nsubmission=pd.DataFrame({\n    \"patient_id\": patient_ids,\n    \"CE\": ces,\n    \"LAA\": lass})\n\ndisplay(submission)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:24:50.218888Z","iopub.execute_input":"2023-01-30T19:24:50.219609Z","iopub.status.idle":"2023-01-30T19:24:50.233254Z","shell.execute_reply.started":"2023-01-30T19:24:50.21957Z","shell.execute_reply":"2023-01-30T19:24:50.232239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:25:04.105901Z","iopub.execute_input":"2023-01-30T19:25:04.10663Z","iopub.status.idle":"2023-01-30T19:25:04.115048Z","shell.execute_reply.started":"2023-01-30T19:25:04.10659Z","shell.execute_reply":"2023-01-30T19:25:04.113781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls\n!rm -r save\n!ls","metadata":{"execution":{"iopub.status.busy":"2023-01-30T19:40:27.502721Z","iopub.execute_input":"2023-01-30T19:40:27.503171Z","iopub.status.idle":"2023-01-30T19:40:30.653432Z","shell.execute_reply.started":"2023-01-30T19:40:27.50313Z","shell.execute_reply":"2023-01-30T19:40:30.652065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}