{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\n\nimport fastai\nfrom fastai.vision import *\nfrom fastai.callbacks import SaveModelCallback\nimport os\nfrom radam import *\nfrom csvlogger import *\nfrom mish_activation import *\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import cohen_kappa_score,confusion_matrix\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nCUDA_LAUNCH_BLOCKING=1\nfastai.__version__\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\nsys.path.insert(0,\"../input/squeezeexcitationnet/\")\nfrom squeeze import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[suspicious slides](https://www.kaggle.com/dannellyz/collection-of-600-suspicious-slides-data-loader)  \n[akensert's removing penmarks](https://www.kaggle.com/akensert/panda-removal-of-pen-marks)  \n[akensert's tiling](https://www.kaggle.com/akensert/panda-optimized-tiling-tf-data-dataset)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# !rm -rf ./models","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"# ##remove this cell if run locally\n# !rm -rf ./cache\n# !mkdir 'cache'\n# !mkdir 'cache/torch'\n# !mkdir 'cache/torch/checkpoints'\n# #!cp '../input/pytorch-pretrained-models/semi_supervised_resnext50_32x4-ddb3e555.pth' 'cache/torch/checkpoints/'\n# !cp '../input/pytorch-se-resnext/se_resnext50_32x4d-a260b3a4.pth' 'cache/torch/checkpoints/'\n# torch.hub.DEFAULT_CACHE_DIR = 'cache'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = 128\nbs = 20\nnfolds = 4\nSEED = 2020\nN = 12 #number of tiles per image\n#TRAIN = '../input/panda-16x128x128-tiles-data/train/'\nTRAIN = '../input/panda-generating-data2/train_rand2000_256sz_16tiles/'\nLABELS = '../input/labels/train.csv'","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data  \nUse stratified KFold split.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"files = sorted([p[:32] for p in os.listdir(TRAIN)])\nlen(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data():\n    import pandas as pd\n    #susp = pd.read_csv('../input/suspicious/PANDA_Suspicious_Slides.csv')\n    # ['marks', 'No Mask', 'Background only', 'No cancerous tissue but ISUP Grade > 0', 'tiss', 'blank']\n    #to_drop = susp.query(\"reason in ['marks','Background only','tiss','blank']\")['image_id']\n    df = pd.read_csv(LABELS).set_index('image_id')\n    #good_index = list(set(df.index)-set(to_drop))\n    files = sorted([p[:32] for p in os.listdir(TRAIN)])\n    df = df.loc[files]\n    df = df.reset_index()\n    splits = StratifiedKFold(n_splits=nfolds, random_state=SEED, shuffle=True)\n    splits = list(splits.split(df,df.isup_grade))\n    folds_splits = np.zeros(len(df)).astype(np.int)\n    for i in range(nfolds): \n        folds_splits[splits[i][1]] = i\n    df['split'] = folds_splits\n    df['gleason_score']=df['gleason_score'].replace('negative','0+0')\n    df[['prim_gleason','secon_gleason']] = df.gleason_score.str.split(\"+\",expand=True)\n    df[['prim_gleason','secon_gleason']] = df[['prim_gleason','secon_gleason']].astype(np.int64)\n    df['prim_gleason']=df['prim_gleason'].replace(3,1)\n    df['prim_gleason']=df['prim_gleason'].replace(4,2)\n    df['prim_gleason']=df['prim_gleason'].replace(5,3)\n    df['secon_gleason']=df['secon_gleason'].replace(3,1)\n    df['secon_gleason']=df['secon_gleason'].replace(4,2)\n    df['secon_gleason']=df['secon_gleason'].replace(5,3)\n    print(\"****************df shape:\",df.shape,\"***********************\")\n    print(\">>>>>>>>>Before sampling<<<<<<<<<<<<<\")\n    for isup in [0,1,2,3,4,5]:\n        print(\"isup grade:\",isup,\"| n_instances:\",df.query('isup_grade=={0}'.format(isup)).shape[0],\"| corresponding gleason score:\",df[['isup_grade','gleason_score']].query('isup_grade=={0}'.format(isup))['gleason_score'].unique())\n        print(\"----\"*20)\n    #df.drop([df[df['image_id']==\"b0a92a74cb53899311acc30b7405e101\"].index[0]],inplace=True)\n    #b0a92a74cb53899311acc30b7405e101 is the only image id with gleason 4+3 mapping to isup=2\n    #df = pd.concat([df.query('isup_grade==0').iloc[:1200],df.query('isup_grade==1').iloc[:1200],df.query('isup_grade==2 or isup_grade==3 or isup_grade==4 or isup_grade==5')],axis=0)\n    df = df.sample(n=5000,random_state=SEED).reset_index(drop=True)#shuffling\n    print(\">>>>>>>>>After sampling<<<<<<<<<<\")\n    for isup in [0,1,2,3,4,5]:\n        print(\"isup grade:\",isup,\"| n_instances:\",df.query('isup_grade=={0}'.format(isup)).shape[0],\"| corresponding gleason score:\",df[['isup_grade','gleason_score']].query('isup_grade=={0}'.format(isup))['gleason_score'].unique())\n        print(\"----\"*20)\n    return df\ndf = get_data()\ndf[['isup_grade','split','prim_gleason','secon_gleason']].hist(bins=50)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nimport seaborn as sns\nsns.countplot(df['data_provider'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\nCheck [this kernel](https://www.kaggle.com/iafoss/panda-16x128x128-tiles) for image stats. Since I use zero padding and background corresponds to 255, I invert images as 255-img when load them. Therefore, the mean value is computed as '1 - val'.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#for 2000 images obtained of size 256 and tiles=16\nmean = torch.tensor([1.0-0.68688968, 1.0-0.44634704, 1.0-0.61367611])\nstd = torch.tensor([0.46521431,0.46922062,0.42265951])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# mean = torch.tensor([1.0-0.90949707, 1.0-0.8188697, 1.0-0.87795304])\n# std = torch.tensor([0.36357649, 0.49984502, 0.40477625])\n# #mean: [0.96589806 0.9326964  0.95441414] , std: [0.30177967 0.4173849  0.33891267]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The code below (in the hidden cell) creates ImageItemList capable of loading multiple tiles of an image. It is specific for fast.ai, and pure Pytorch code would be much simpler.","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"def open_image(fn:PathOrStr, div:bool=True, convert_mode:str='RGB', cls:type=Image,after_open:Callable=None)->Image:\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\", UserWarning) # EXIF warning from TiffPlugin\n        x = PIL.Image.open(fn).convert(convert_mode)\n    if after_open: \n        x = after_open(x)\n    x = pil2tensor(x,np.float32)\n    if div: \n        x.div_(255)\n    return cls(1.0-x) #invert image for zero padding\n\nclass MImage(ItemBase):\n    def __init__(self, imgs):\n        self.obj  = (imgs)\n        self.data = [(imgs[i].data - mean[...,None,None])/std[...,None,None] for i in range(len(imgs))]\n    \n    def apply_tfms(self, tfms,*args, **kwargs):\n        for i in range(len(self.obj)):\n            self.obj[i] = self.obj[i].apply_tfms(tfms, *args, **kwargs)\n            self.data[i] = (self.obj[i].data - mean[...,None,None])/std[...,None,None]\n        return self\n    \n    def __repr__(self): \n        return f'{self.__class__.__name__} {img.shape for img in self.obj}'\n    \n    def to_one(self):\n        img = torch.stack(self.data,1)\n        img = img.view(3,-1,N,sz,sz).permute(0,1,3,2,4).contiguous().view(3,-1,sz*N)\n        return Image(1.0 - (mean[...,None,None]+img*std[...,None,None]))\n\nclass MImageItemList(ImageList):\n    def __init__(self, *args, **kwargs):\n        super().__init__(*args, **kwargs)\n    \n    def __len__(self)->int: return len(self.items) or 1 \n    \n    def get(self, i):\n        fn = Path(self.items[i])\n        fnames = [Path(str(fn)+'_'+str(i)+'.png')for i in range(N)]\n        imgs = [open_image(fname, convert_mode=self.convert_mode, after_open=self.after_open)\n               for fname in fnames]\n        return MImage(imgs)\n\n    def reconstruct(self, t):\n        return MImage([mean[...,None,None]+_t*std[...,None,None] for _t in t])\n    \n    def show_xys(self, xs, ys, figsize:Tuple[int,int]=(300,50), **kwargs):\n        rows = min(len(xs),8)\n        fig, axs = plt.subplots(rows,1,figsize=figsize)\n        for i, ax in enumerate(axs.flatten() if rows > 1 else [axs]):\n            xs[i].to_one().show(ax=ax, y=ys[i], **kwargs)\n        plt.tight_layout()\n        \n\n#collate function to combine multiple images into one tensor\ndef MImage_collate(batch:ItemsList)->Tensor:\n    result = torch.utils.data.dataloader.default_collate(to_data(batch))\n    if isinstance(result[0],list):\n        result = [torch.stack(result[0],1),result[1]]\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(fold=0):\n    return (MImageItemList.from_df(df, path='.', folder=TRAIN, cols='image_id')\n      .split_by_idx(df.index[df.split == fold].tolist())\n      .label_from_df(cols=['isup_grade'])\n      .transform(get_transforms(flip_vert=True,max_rotate=15),size=sz,padding_mode='zeros')\n      .databunch(bs=bs,num_workers=5,device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')))\ndata = get_data(0)\n#data.show_batch()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# i[1] contains labels, indicating isup grades\n# preds,y,losses = learn.get_preds(with_loss=True)\n# interp = ClassificationInterpretation(learn, preds, y, losses)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in data.dl():\n    print(i[0][0].shape,i[0][7][0].shape,i[1].shape)\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_image(img,is_mask=False):\n    '''\n    To display image/mask \n    args: img, image\n          is_mask, boolean True if greyscale mask is passed\n    '''\n    from matplotlib import pyplot as plt\n    %matplotlib inline\n    if is_mask:\n        plt.imshow(img,cmap='gray')\n    else:\n        plt.imshow(img)\n    plt.show()            ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models  \n- [Semi-Weakly Supervised ImageNet pretrained ResNeXt50 model](https://github.com/facebookresearch/semi-supervised-ImageNet1K-models)  \n- **ResNeSt50**: employs Split-Attention block that enables attention across feature-map groups by stacking these Split-Attention blocks ResNet-style. Better results on 224 x 224.  \nm = torch.hub.load('zhanghang1989/ResNeSt', 'resnest50', pretrained=True)  \n- **SE-ResNext50**:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# m = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models',model='resnext50_32x4d_ssl')\n# #there are 8 blocks in resnext 50\n# print(list(m.children())[:-2][0],'\\n',list(m.children())[:-2][1],'\\n',list(m.children())[:-2][2],'\\n',list(m.children())[:-2][3])\n# for i in [4,5,6,7]:\n#     print(list(m.children())[:-2][i])\n#     print(\"-\"*100)#,list(m.children())[:-2][5],list(m.children())[:-2][6],list(m.children())[:-2][7]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nprim_dict=Counter(df['prim_gleason'])\nsec_dict= Counter(df['secon_gleason'])\n#Method1\n# prim_weights = [1 - (prim_dict[i] / sum(prim_dict.values())) for i in [0,1,2,3]]\n# prim_weights = torch.FloatTensor(prim_weights).to('cuda')\n# sec_weights = [1 - (sec_dict[i] / sum(sec_dict.values())) for i in [0,1,2,3]]\n# sec_weights = torch.FloatTensor(sec_weights).to('cuda')\n\n#Method2\nprim_weights = torch.FloatTensor([2,1,1,10]).to('cuda')#torch.FloatTensor([max(prim_dict.values())/prim_dict[i]  for i in [0,1,2,3]]).to('cuda')\nsec_weights = torch.FloatTensor([2,1,1,10]).to('cuda')#torch.FloatTensor([max(sec_dict.values())/sec_dict[i]  for i in [0,1,2,3]]).to('cuda')\nprim_weights,sec_weights,prim_dict,sec_dict\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[MuTltiTaskLoss](https://towardsdatascience.com/self-paced-multitask-learning-76c26e9532d0)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class ConfusionMatrix(Callback):\n#     \"Computes the confusion matrix.\"\n\n#     def on_train_begin(self, **kwargs):\n#         self.n_classes = 0\n\n#     def on_epoch_begin(self, **kwargs):\n#         self.cm = None\n\n#     def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n#         preds = last_output.argmax(-1).view(-1).cpu()\n#         targs = last_target.cpu()\n#         if self.n_classes == 0:\n#             self.n_classes = last_output.shape[-1]\n#         if self.cm is None: self.cm = torch.zeros((self.n_classes, self.n_classes), device=torch.device('cpu'))\n#         cm_temp_numpy = self.cm.numpy()\n#         np.add.at(cm_temp_numpy, (targs ,preds), 1)\n#         self.cm = torch.from_numpy(cm_temp_numpy)\n\n#     def on_epoch_end(self, **kwargs):\n#         self.metric = self.cm\n        \n# @dataclass\n# class KappaScore1(ConfusionMatrix):\n#     \"Computes the rate of agreement (Cohens Kappa).\"\n#     weights:Optional[str]=None      # None, `linear`, or `quadratic`\n        \n#     def on_batch_end(self, last_output:Tensor, last_target:Tensor, **kwargs):\n# #         print(\"Customised scoring function....\")\n# #         print(last_output[0].shape,last_output[1].shape,last_target.shape)\n#         preds,targs = get_isup_preds_targs_2targets(last_output,last_target)#convert gleasons-->isup for evaluatio\n\n#         if self.n_classes == 0:\n#             self.n_classes = 6 #n_classes in isup_grade\n#         if self.cm is None: \n#             #This executes only once\n#             self.cm = torch.zeros((self.n_classes, self.n_classes), device=torch.device('cpu'))\n#         cm_temp_numpy = self.cm.numpy()\n#         np.add.at(cm_temp_numpy, (targs ,preds), 1)\n#         self.cm = torch.from_numpy(cm_temp_numpy)\n        \n        \n#     def on_epoch_end(self, last_metrics, **kwargs):\n#         sum0 = self.cm.sum(dim=0)\n#         sum1 = self.cm.sum(dim=1)\n#         expected = torch.einsum('i,j->ij', (sum0, sum1)) / sum0.sum()\n#         if self.weights is None:\n#             w = torch.ones((self.n_classes, self.n_classes))\n#             w[self.x, self.x] = 0\n#         elif self.weights == \"linear\" or self.weights == \"quadratic\":\n#             w = torch.zeros((self.n_classes, self.n_classes))\n#             w += torch.arange(self.n_classes, dtype=torch.float)\n#             w = torch.abs(w - torch.t(w)) if self.weights == \"linear\" else (w - torch.t(w)) ** 2\n#         else: raise ValueError('Unknown weights. Expected None, \"linear\", or \"quadratic\".')\n#         k = torch.sum(w * self.cm) / torch.sum(w * expected)\n#         return add_metrics(last_metrics, 1-k)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_resnext(layers, pretrained, progress, **kwargs):\n    from torchvision.models.resnet import ResNet, Bottleneck\n    model = ResNet(Bottleneck, layers, **kwargs)\n    model.load_state_dict(torch.load('../input/resnext-50-ssl/semi_supervised_resnext50_32x4-ddb3e555.pth'))\n    return model\n\n\nclass GleasonISUP(nn.Module):\n    def __init__(self):\n        super().__init__()\n        #m = torch.hub.load('zhanghang1989/ResNeSt', 'resnest50', pretrained=True)\n        #m = get_resnext([3, 4, 6, 3], pretrained=True, progress=False, groups=32,width_per_group=4)\n        m = se_resnext50_32x4d(4,loss='softmax', pretrained=True)\n        self.enc = nn.Sequential(*list(m.children())[:-2])       \n        nc = list(m.children())[-1].in_features\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),\n                                  Flatten(),\n#                                   nn.Linear(2*nc,512),\n#                                   Mish(),\n#                                   nn.BatchNorm1d(512), \n#                                   nn.Dropout(0.5),\n                                  nn.Linear(2*nc,512),\n                                  Mish(),\n                                  nn.BatchNorm1d(512), \n                                  nn.Dropout(0.3))\n        self.prim =  nn.Sequential(nn.Linear(512,128),nn.ReLU(),nn.BatchNorm1d(128),nn.Dropout(0.3),nn.Linear(128,4))\n        self.sec  =  nn.Sequential(nn.Linear(512,128),nn.ReLU(),nn.BatchNorm1d(128),nn.Dropout(0.3),nn.Linear(128,4))\n        self.isup  =  nn.Sequential(nn.Linear(512,128),nn.ReLU(),nn.BatchNorm1d(128),nn.Dropout(0.3),nn.Linear(128,6))\n      \n        \n        \n    def forward(self, *x):\n        \n        \n        shape = x[0].shape\n        n = len(x)\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n        #x: bs*N x 3 x 128 x 128\n        \n        \n        x = self.enc(x)\n        \n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n        #concatenate the output for tiles into a single map\n        x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous().view(-1,shape[1],shape[2]*n,shape[3])\n        #x: bs x C x N*4 x 4\n        x = self.head(x)\n        #x: bs x n\n        prim_gleason = self.prim(x)\n        sec_gleason  = self.sec(x)\n        isup = self.isup(x)\n        preds = [prim_gleason,sec_gleason,isup]\n        return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_resnext(layers, pretrained, progress, **kwargs):\n    from torchvision.models.resnet import ResNet, Bottleneck\n    model = ResNet(Bottleneck, layers, **kwargs)\n    model.load_state_dict(torch.load('../input/resnext-50-ssl/semi_supervised_resnext50_32x4-ddb3e555.pth'))\n    return model\n\n\nclass Gleason(nn.Module):\n    def __init__(self):\n        super().__init__()\n        #m = torch.hub.load('zhanghang1989/ResNeSt', 'resnest50', pretrained=True)\n        #m = get_resnext([3, 4, 6, 3], pretrained=True, progress=False, groups=32,width_per_group=4)\n        m = se_resnext50_32x4d(4,loss='softmax', pretrained=True)\n        self.enc = nn.Sequential(*list(m.children())[:-2])       \n        nc = list(m.children())[-1].in_features\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),\n                                  Flatten(),\n                                  nn.Linear(2*nc,512),\n                                  Mish(),\n                                  nn.BatchNorm1d(512), \n                                  nn.Dropout(0.5),\n                                  nn.Linear(512,256),\n                                  Mish(),\n                                  nn.BatchNorm1d(256), \n                                  nn.Dropout(0.3))\n        self.prim =  nn.Sequential(nn.Linear(256,128),Mish(),nn.BatchNorm1d(128),nn.Dropout(0.3),nn.Linear(128,4))\n        self.sec  =  nn.Sequential(nn.Linear(256,128),Mish(),nn.BatchNorm1d(128),nn.Dropout(0.3),nn.Linear(128,4))\n      \n      \n        \n        \n    def forward(self, *x):\n        \n        \n        shape = x[0].shape\n        n = len(x)\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n        #x: bs*N x 3 x 128 x 128\n        \n        \n        x = self.enc(x)\n        \n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n        #concatenate the output for tiles into a single map\n        x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous().view(-1,shape[1],shape[2]*n,shape[3])\n        #x: bs x C x N*4 x 4\n        x = self.head(x)\n        #x: bs x n\n        prim_gleason = self.prim(x)\n        sec_gleason  = self.sec(x)\n        preds = [prim_gleason,sec_gleason]\n        return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_isup_preds_targs(preds,target):\n    lookup_map = {(0,0):0,(1,1):1,(1,2):2,(2,1):3,(2,2):4,(3,1):4,(1,3):4,(2,3):5,(3,2):5,(3,3):5}\n    lookup_map2 = {(0,1):1,(0,2):1,(0,3):2,(1,0):1,(2,0):3,(3,0):4}#or all 0\n    prim_preds = preds[0].argmax(-1).view(-1,1)\n    sec_preds  = preds[1].argmax(-1).view(-1,1)\n    temp_preds = torch.cat([prim_preds,sec_preds],dim=1)\n    \n    count = 0\n    errors = 0\n    temp = []\n    for i in np.array(temp_preds.cpu()):\n        count+=1\n        try:\n            temp.append(lookup_map[tuple(i)])\n        except KeyError:\n            print(tuple(i),\" missing\")\n            print('target={0},prediction={1}'.format(isup_target[i],isup_preds[i]))\n            errors+=1\n            temp.append(lookup_map2[tuple(i)])\n    print(\"count={0},errors={1}\".format(count,errors),\"correct=\",count-errors)\n    isup_preds = torch.tensor(temp,dtype=torch.long,device='cpu')\n    temp = []\n    for i in np.array(target.cpu()):\n        temp.append(lookup_map[tuple(i)])\n    isup_targs = torch.tensor(temp,dtype=torch.long,device='cpu')    \n    return isup_preds,isup_targs\n\nclass BiTaskGleasonLoss(nn.Module):\n    def __init__(self, task_num):\n        super(BiTaskGleasonLoss, self).__init__()\n        self.task_num = task_num\n        self.log_vars = nn.Parameter(torch.zeros((task_num)))\n\n    def forward(self, preds, targets):\n        temp=[]\n        primloss = nn.CrossEntropyLoss(weight=prim_weights)\n        prim_preds  = preds[0]\n        prim_target = targets[:,0].long()\n        loss0 = primloss(prim_preds,prim_target)\n        temp.append(loss0)\n        precision0 = torch.exp(-self.log_vars[0])\n        loss0 = precision0*loss0 + self.log_vars[0]   \n        \n        secloss  = nn.CrossEntropyLoss(weight=sec_weights)\n        sec_preds  = preds[1]\n        sec_target = targets[:,1].long()\n        loss1 = secloss(sec_preds,sec_target)\n        temp.append(loss1)\n        precision1 = torch.exp(-self.log_vars[1])\n        loss1 = precision1*loss1 + self.log_vars[1]   \n        \n        \n        print(\"precisions:\", precision0,precision1)\n        print(self.log_vars[0],self.log_vars[1].dtype)\n        print(\"simple loss:\",sum(temp))\n        print(\"complicated loss:\",\"loss0+loss1:\",loss0+loss1,\"loss0+loss1:\",loss0+loss1)\n        return loss1+loss0\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_isup_preds_targs_2targets(preds,targs):\n    #predictions\n    prim_preds = preds[0].argmax(-1).view(-1,1)\n    sec_preds  = preds[1].argmax(-1).view(-1,1)\n    gleason_preds = np.array(torch.cat([prim_preds,sec_preds],dim=1).cpu())#converting to np.array for tuple()\n    #targets\n    target = np.array(targs.cpu())#converting to np.array to cast to tuple()\n    gleason_target = target[:,0:2]\n    \n    lookup_map = {(0,0):0,(1,1):1,(1,2):2,(2,1):3,(2,2):4,(3,1):4,(1,3):4,(2,3):5,(3,2):5,(3,3):5}\n    lookup_map2 = {(0,1):1,(0,2):1,(0,3):2,(1,0):1,(2,0):3,(3,0):4}\n    #lookup_map's keys are indices of gleason scores and not gleason scores themselves!\n    temp1 = []#for predictions\n    temp2 = []#for targets\n    count = 0\n    errors = 0\n    for i in range(len(gleason_preds)):\n        count+=1\n        try:\n            temp1.append(lookup_map[tuple(gleason_preds[i])])\n        except KeyError:\n            errors+=1\n            temp1.append(lookup_map2[tuple(gleason_preds[i])])\n            print(\">>>>>>>>>>missing<<<<<<<<<<<\")\n        finally:\n            print('target={0},prediction={1}'.format(gleason_target[i],tuple(gleason_preds[i])))\n            temp2.append(lookup_map[tuple(gleason_target[i])])\n            print(\"-\"*30)\n            \n    print(\"count={0},errors={1}\".format(count,errors),\"correct=\",count-errors)\n    final_preds = torch.tensor(temp1,dtype=torch.long,device='cpu')\n    final_targs = torch.tensor(temp2,dtype=torch.long,device='cpu')    \n    return final_preds,final_targs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class TriTaskGleasonLoss(nn.Module):\n#     def __init__(self, task_num):\n#         super(TriTaskGleasonLoss, self).__init__()\n#         self.task_num = task_num\n#         self.log_vars = nn.Parameter(torch.zeros((task_num)))\n\n#     def forward(self, preds, targets):\n# #         print(\"type(preds):\",type(preds),\"| len(preds):\",len(preds),\"| preds[0].shape:\",preds[0].shape,\"| preds[1].shape:\",preds[1].shape,\"| preds[2].shape:\",preds[2].shape)\n# #         print(\"type(targets):\",type(targets),\"| targets[:,0].shape:\",targets[:,0].shape,\"| targets[:,1].shape:\",targets[:,1].shape,\"| targets[:,2].shape:\",targets[:,2].shape)\n        \n#         temp=[]\n#         primloss = nn.CrossEntropyLoss(weight=prim_weights)\n#         prim_preds  = preds[0]\n#         prim_target = targets[:,0].long()\n#         loss0 = primloss(prim_preds,prim_target)\n#         temp.append(loss0)\n#         precision0 = torch.exp(-self.log_vars[0])\n#         loss0 = precision0*loss0 + self.log_vars[0]   \n        \n#         secloss  = nn.CrossEntropyLoss(weight=sec_weights)\n#         sec_preds  = preds[1]\n#         sec_target = targets[:,1].long()\n#         loss1 = secloss(sec_preds,sec_target)\n#         temp.append(loss1)\n#         precision1 = torch.exp(-self.log_vars[1])\n#         loss1 = precision1*loss1 + self.log_vars[1]   \n        \n#         crossEntropy = nn.CrossEntropyLoss()\n#         isup_preds  = preds[2]\n#         isup_target = targets[:,2].long()\n#         loss2 = crossEntropy(isup_preds,isup_target)\n#         temp.append(loss2)\n#         precision2 = torch.exp(-self.log_vars[2])\n#         loss2 = precision2*loss2 + self.log_vars[2]   \n#         print(\"precisions:\", precision0,precision1,precision2)\n#         print(self.log_vars[0],self.log_vars[1].dtype,self.log_vars[2])\n#         print(\"simple loss:\",sum(temp))\n#         print(\"complicated loss:\",\"loss0+loss1+loss2:\",loss0+loss1+loss2,\"loss0+loss1:\",loss0+loss1)\n#         print(\"-\"*20)\n#         return loss0*loss1*loss2\n\n# def get_isup_preds_targs_3targets(preds,targs):\n#     #predictions\n#     prim_preds = preds[0].argmax(-1).view(-1,1)\n#     sec_preds  = preds[1].argmax(-1).view(-1,1)\n#     temp_preds = np.array(torch.cat([prim_preds,sec_preds],dim=1).cpu())#converting to np.array for tuple()\n#     isup_preds = preds[2].argmax(-1).view(-1).cpu()\n#     #targets\n#     target = np.array(targs.cpu())#converting to np.array to cast to tuple()\n#     gleason_target = target[:,0:2]\n#     isup_target = target[:,2]\n    \n#     lookup_map = {(0,0):0,(1,1):1,(1,2):2,(2,1):3,(2,2):4,(3,1):4,(1,3):4,(2,3):5,(3,2):5,(3,3):5}\n#     #lookup_map's keys are indices of gleason scores and not gleason scores themselves!\n#     temp1 = []#for predictions\n#     temp2 = []#for targets\n#     count = 0\n#     errors = 0\n#     for i in range(len(temp_preds)):\n# #         temp1.append(isup_preds[i])\n# #         temp2.append(isup_target[i])\n# #         print('target={0},prediction={1}'.format(isup_target[i],isup_preds[i]))\n#         count+=1\n#         try:\n#             temp1.append(lookup_map[tuple(temp_preds[i])])\n#             temp2.append(lookup_map[tuple(gleason_target[i])])\n#         except KeyError:\n#             print(tuple(temp_preds[i]),\" is missing!\")\n#             print('target={0},prediction={1}'.format(isup_target[i],isup_preds[i]))\n#             errors+=1\n#             temp1.append(isup_preds[i])\n#             temp2.append(isup_target[i])\n#     print(\"count={0},errors={1}\".format(count,errors),\"correct=\",count-errors)\n#     final_preds = torch.tensor(temp1,dtype=torch.long,device='cpu')\n#     final_targs = torch.tensor(temp2,dtype=torch.long,device='cpu')    \n#     return final_preds,final_targs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, arch='resnext50_32x4d_ssl', n=6, pre=True):\n        super().__init__()\n        m = se_resnext50_32x4d(4,loss='softmax', pretrained=True)\n        self.enc = nn.Sequential(*list(m.children())[:-2])       \n        nc = list(m.children())[-1].in_features\n        self.head = nn.Sequential(AdaptiveConcatPool2d(),\n                                  Flatten(),\n                                  nn.Linear(2*nc,512),\n                                  Mish(),\n                                  nn.BatchNorm1d(512), \n                                  nn.Dropout(0.5),\n                                  nn.Linear(512,n))\n        \n        \n    def forward(self, *x):\n        shape = x[0].shape\n        n = len(x)\n        x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n        #x: bs*N x 3 x 128 x 128\n        x = self.enc(x)\n        #x: bs*N x C x 4 x 4\n        shape = x.shape\n        #concatenate the output for tiles into a single map\n        x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous().view(-1,shape[1],shape[2]*n,shape[3])\n        #x: bs x C x N*4 x 4\n        x = self.head(x)\n        #x: bs x n\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data  = get_data(3)\n# model = Gleason()\n# gleason_loss = BiTaskGleasonLoss(2)\n# learn = Learner(data,model,loss_func=gleason_loss, opt_func=RAdam, metrics=[KappaScore(weights='quadratic')]).to_fp16()\n# learn.loss_func=learn.loss_func.to('cuda')\n# learn.model=learn.model.to('cuda')\n# learn.clip_grad = 1.0\n# learn.split([learn.model.enc[0],learn.model.enc[1],learn.model.enc[2],learn.model.enc[3],learn.model.enc[4],learn.model.head,nn.ModuleList([learn.model.prim,learn.model.sec])])\n# learn.freeze()#first, train only the last layer(the heads)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data  = get_data(3)\nmodel = Model()\ngleason_loss = LabelSmoothingCrossEntropy()\nlearn = Learner(data,model,loss_func=gleason_loss, opt_func=Over9000, metrics=[KappaScore(weights='quadratic')]).to_fp16()\nlearn.loss_func=learn.loss_func.to('cuda')\nlearn.model=learn.model.to('cuda')\nlearn.clip_grad = 1.0\nlearn.split([learn.model.enc[0],learn.model.enc[1],learn.model.enc[2],learn.model.enc[3],learn.model.enc[4],learn.model.head])\nlearn.freeze()#first, train only the last layer(the heads)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.cuda.get_device_name(),torch.cuda.get_device_properties(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.lr_find()\n# # figures out what is the fastest I can train this neural network without making it zip off the rails and get blown apart\n# learn.recorder.plot()\n# plt.title(\"Loss Vs Learning Rate\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# learn.fit_one_cycle(5, max_lr=5e-2, div_factor=100, pct_start=0.0, callbacks =[SaveModelCallback(learn,name='stage1',monitor='kappa_score')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.load('stage5')\nlearn.unfreeze()\n# learn.lr_find()\n# learn.recorder.plot()\n# plt.title(\"Loss Vs Learning Rate\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(8, max_lr=slice(5e-4,5e-2),div_factor=100,pct_start=0.0,callbacks = [SaveModelCallback(learn,name='stage5',monitor='kappa_score')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fname = 'RNXT50'\n# pred,target = [],[]\n# for fold in range(nfolds):#nfolds\n#     print(\"-\"*40,\"Fold-\",fold,\"-\"*40)\n#     data  = get_data(fold)\n#     model = GleasonModel()\n#     gleason_loss = TriTaskGleasonLoss(3)\n#     learn = Learner(data,model,loss_func=gleason_loss, opt_func=Over9000, metrics=[KappaScore(weights='quadratic')]).to_fp16()\n#     learn.loss_func=learn.loss_func.to('cuda')\n#     learn.model=learn.model.to('cuda')\n#     learn.clip_grad = 1.0\n#     learn.split([learn.model.enc[0:7],learn.model.enc[7],learn.model.head,nn.ModuleList([learn.model.prim,learn.model.sec,learn.model.isup])])\n#     learn.freeze()#first, train only the last layer(the heads)\n#     learn.fit_one_cycle(1, max_lr=1e-1, div_factor=25, pct_start=0.0, \n#       callbacks = [SaveModelCallback(learn,name='stage1_{0}'.format(fold),monitor='kappa_score')])\n#     logger = CSVLogger(learn, f'log_{fname}_{fold}')\n#     learn.load('stage1_{0}'.format(fold))\n#     learn.freeze_to(1)\n#     learn.fit_one_cycle(1, max_lr=slice(5e-6,5e-4), div_factor=25, pct_start=0.0, \n#       callbacks = [SaveModelCallback(learn,name='stage2_{0}'.format(fold),monitor='kappa_score')])\n    \n#     torch.save(learn.model.state_dict(), f'{fname}_{fold}.pth')\n    \n#     learn.model.eval()\n#     with torch.no_grad():\n#         for step, (x, y) in progress_bar(enumerate(data.dl(DatasetType.Valid)),total=len(data.dl(DatasetType.Valid))):\n#             #print(len(x),x[0].shape)\n#             p = learn.model(*x) \n#             preds,targs = get_isup_preds_targs(p,y)\n#             pred.append(preds)\n#             target.append(targs)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# p = torch.argmax(torch.cat(pred,dim=0),1)      \n# t = torch.cat(target)\n# print(cohen_kappa_score(t,p,weights='quadratic'))\n# print(confusion_matrix(t,p))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# confusion_matrix(torch.cat(target),torch.cat(pred,dim=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cohen_kappa_score(torch.cat(target),torch.cat(pred,dim=0),weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# !rm -r 'cache'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install -q torchviz\n# import torch\n# from torchviz import make_dot\n# model = Model()\n# x = torch.randn(2, 3, 128, 128).requires_grad_(True)\n# y = model(x)\n# make_dot(y, params=dict(list(model.named_parameters()) + [('x', x)])).render(\"attached\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class Model(nn.Module):\n#     def __init__(self, arch='resnext50_32x4d_ssl', n=6, pre=True):\n#         super().__init__()\n#         #m = torch.hub.load('zhanghang1989/ResNeSt', 'resnest50', pretrained=True)\n#         m = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', arch)\n#         self.enc = nn.Sequential(*list(m.children())[:-2])       \n#         nc = list(m.children())[-1].in_features\n#         self.head = nn.Sequential(AdaptiveConcatPool2d(),\n#                                   Flatten(),\n#                                   nn.Linear(2*nc,128),\n#                                   Mish(),\n#                                   nn.BatchNorm1d(128), \n#                                   nn.Dropout(0.5),\n#                                   nn.Linear(128,n))\n        \n        \n#     def forward(self, *x):\n#         shape = x[0].shape\n#         n = len(x)\n#         x = torch.stack(x,1).view(-1,shape[1],shape[2],shape[3])\n#         #x: bs*N x 3 x 128 x 128\n#         x = self.enc(x)\n#         #x: bs*N x C x 4 x 4\n#         shape = x.shape\n#         #concatenate the output for tiles into a single map\n#         x = x.view(-1,n,shape[1],shape[2],shape[3]).permute(0,2,1,3,4).contiguous().view(-1,shape[1],shape[2]*n,shape[3])\n#         #x: bs x C x N*4 x 4\n#         x = self.head(x)\n#         #x: bs x n\n#         return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# #UNDERSAMPLING\n# import seaborn as sns\n# from imblearn.under_sampling import RandomUnderSampler\n# rus = RandomUnderSampler(random_state=SEED)\n# X_resampled, y_resampled = rus.fit_resample(df[['image_id', 'data_provider','split']].iloc[:8000], df['isup_grade'].iloc[:8000])\n# df = pd.concat([pd.DataFrame(X_resampled,columns=['image_id', 'data_provider','split']),pd.DataFrame(y_resampled,columns=['isup_grade'])],axis=1)\n# df = df.sample(frac=1,random_state=SEED).reset_index(drop=True)\n# sns.countplot(df['isup_grade'])\n# df.head()\n# df = df.groupby(['split'], group_keys=False).apply(lambda x: x.sample(min(len(x), 1000),random_state=SEED))\n# df[['isup_grade','split']].hist(bins=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# t1=torch.tensor([[-8.8875e+01,  3.9062e+01, -6.1406e+00, -6.6625e+01],\n#         [-3.3781e+01,  1.4578e+01, -8.6133e-01, -2.5047e+01],\n#         [-2.9297e+00,  1.2412e+00,  9.8584e-01, -2.6113e+00],\n#         [-1.6992e-01,  3.9355e-01,  4.3701e-01, -1.1953e+00],\n#         [-5.7910e-01,  6.8457e-01,  4.6216e-01, -1.4893e+00],\n#         [-1.7639e-01,  4.2603e-01,  3.8843e-01, -1.2490e+00],\n#         [-1.6297e+01,  7.1523e+00,  4.3018e-01, -1.2164e+01],\n#         [-1.6333e-01,  4.3164e-01,  3.7207e-01, -1.2646e+00],\n#         [-4.8309e-02,  4.4946e-01,  3.0298e-01, -1.1963e+00],\n#         [-3.6844e+01,  1.6312e+01, -2.1621e+00, -2.7594e+01],\n#         [        0,         1,         0,         0],\n#         [-7.6721e-02,  4.3408e-01,  3.4644e-01, -1.1943e+00],\n#         [-8.0875e+01,  3.5188e+01, -3.9707e+00, -5.9656e+01],\n#         [-6.1625e+01,  2.7312e+01, -3.2949e+00, -4.6594e+01],\n#         [-3.5906e+01,  1.5633e+01, -9.3945e-01, -2.6281e+01],\n#         [-2.1641e+01,  9.4062e+00,  6.9385e-01, -1.5695e+01],\n#         [-2.8781e+01,  1.2414e+01,  4.8706e-01, -2.0859e+01],\n#         [-3.0266e+01,  1.3586e+01, -1.4209e+00, -2.2922e+01],\n#         [        0,         1,         0,         0],\n#         [-3.6531e+01,  1.6172e+01, -1.5479e+00, -2.7625e+01],\n#         [-6.9580e-02,  4.5190e-01,  3.1958e-01, -1.2119e+00],\n#         [-6.0181e-02,  4.5117e-01,  3.1421e-01, -1.2178e+00],\n#         [-2.8711e-01,  3.8086e-01,  4.9854e-01, -1.2412e+00],\n#         [-4.4373e-02,  4.3896e-01,  3.1714e-01, -1.1748e+00],\n#         [-6.0028e-02,  4.4263e-01,  3.2349e-01, -1.1914e+00],\n#         [-9.6741e-02,  4.4019e-01,  3.5449e-01, -1.2178e+00],\n#         [-3.0762e+00,  1.5293e+00,  6.8506e-01, -2.8242e+00],\n#         [-5.3345e-02,  4.5386e-01,  3.0737e-01, -1.2119e+00],\n#         [-1.2711e+01,  5.7656e+00,  3.3203e-01, -9.6797e+00],\n#         [-6.6406e-02,  4.4727e-01,  3.2471e-01, -1.1963e+00],\n#         [-4.9042e-02,  4.3872e-01,  3.0420e-01, -1.1709e+00],\n#         [-1.6641e+00,  1.1172e+00,  5.8203e-01, -2.0605e+00]])\n# t2=torch.tensor([[-5.9812e+01,  1.6828e+01, -1.5188e+01,  4.7781e+01],\n#         [-2.2938e+01,  7.0039e+00, -5.7188e+00,  1.8125e+01],\n#         [-2.1074e+00,  8.9258e-01, -4.4336e-01,  1.7598e+00],\n#         [-4.0796e-01,  5.6299e-01, -8.6121e-02, -2.2437e-01],\n#         [-5.1025e-01,  8.9648e-01, -3.6938e-01, -2.2949e-02],\n#         [-4.5142e-01,  6.2695e-01, -2.0935e-01, -1.9116e-01],\n#         [-1.1297e+01,  3.5859e+00, -2.5879e+00,  8.7109e+00],\n#         [-4.3408e-01,  6.1182e-01, -1.8848e-01, -2.2412e-01],\n#         [-2.7954e-01,  5.9863e-01, -2.1533e-01, -3.5278e-01],\n#         [-2.4562e+01,  7.7227e+00, -7.1094e+00,  2.0703e+01],\n#         [        0,         1,         0,         0],\n#         [-3.1030e-01,  5.6934e-01, -1.3806e-01, -3.4082e-01],\n#         [-5.5281e+01,  1.5883e+01, -1.3398e+01,  4.2781e+01],\n#         [-4.1875e+01,  1.1969e+01, -1.0406e+01,  3.2938e+01],\n#         [-2.4562e+01,  7.4766e+00, -6.0117e+00,  1.9375e+01],\n#         [-1.5336e+01,  5.4688e+00, -4.2070e+00,  1.1961e+01],\n#         [-2.0141e+01,  6.2305e+00, -4.4219e+00,  1.5086e+01],\n#         [-2.0578e+01,  6.3203e+00, -5.8203e+00,  1.7156e+01],\n#         [        0,         1,         0,         0],\n#         [-2.4766e+01,  7.2578e+00, -6.2344e+00,  1.9750e+01],\n#         [-3.0591e-01,  6.0107e-01, -2.0142e-01, -3.3472e-01],\n#         [-2.9712e-01,  5.8789e-01, -1.8433e-01, -3.5400e-01],\n#         [-5.7617e-01,  5.8154e-01, -1.0052e-01, -8.4595e-02],\n#         [-2.5415e-01,  5.9082e-01, -1.9971e-01, -3.5474e-01],\n#         [-2.8394e-01,  6.0303e-01, -2.0618e-01, -3.4229e-01],\n#         [-3.3081e-01,  5.8594e-01, -1.5564e-01, -3.2251e-01],\n#         [-2.0703e+00,  9.4531e-01, -6.0254e-01,  1.8076e+00],\n#         [-2.9077e-01,  5.9424e-01, -1.9482e-01, -3.5474e-01],\n#         [-8.9766e+00,  2.7832e+00, -2.2129e+00,  7.0469e+00],\n#         [-2.8809e-01,  6.1279e-01, -2.1301e-01, -3.4375e-01],\n#         [-2.7637e-01,  6.0547e-01, -2.3462e-01, -3.2910e-01],\n#         [-1.1592e+00,  1.0586e+00, -5.2344e-01,  7.5098e-01]])\n# preds = [t1,t2]\n# target = torch.tensor([[1., 2.],\n#         [0., 0.],\n#         [2., 3.],\n#         [3., 3.],\n#         [0., 0.],\n#         [2., 1.],\n#         [0., 0.],\n#         [2., 2.],\n#         [2., 3.],\n#         [1., 1.],\n#         [2., 3.],\n#         [0., 0.],\n#         [2., 1.],\n#         [1., 2.],\n#         [1., 1.],\n#         [1., 2.],\n#         [2., 2.],\n#         [0., 0.],\n#         [2., 2.],\n#         [2., 3.],\n#         [2., 2.],\n#         [0., 0.],\n#         [3., 3.],\n#         [1., 2.],\n#         [2., 1.],\n#         [3., 3.],\n#         [2., 3.],\n#         [0., 0.],\n#         [2., 2.],\n#         [3., 2.],\n#         [2., 2.],\n#         [0., 0.]])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Notebooks for MultiTask:  \nhttps://github.com/hosseinshn/Basic-Multi-task-Learning/blob/master/MTL-Pytorch.ipynb  \nhttps://github.com/sugi-chan/pytorch_multitask/blob/master/pytorch%20multi-task-Copy2.ipynb","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}