{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Important: For full training, use DEBUG=False and increase epochs.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Came to know of the colored library from https://www.kaggle.com/tarunpaparaju/twitter-challenge-roberta-sentiment-predictor\nI even learnt to use TPU from his kernels. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"In this kernel:\n\n1. Finetune vanilla seresnext50 pretrained model on PANDA dataset. \n2. MultilabelStratifiedKFold used.\n3. Tiles approach with 16 tiles.\n4. No head\n5. Loss: MSELoss\n6. Epochs 4\n7. Optimizer: Adam\n8. LR: 1e-4\n9. Scheduler: ReduceLROnPlateau\n\nGPU 1 epoch per fold = 7 mins, \nTPU 1 epoch simultaneously 4 folds = 15 mins","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install ../input/pretrainedmodels/pretrainedmodels-0.7.4/pretrainedmodels-0.7.4/ > /dev/null","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install iterative-stratification","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Pytorch XLA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev\n!export XLA_USE_BF16=1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q colored","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Import Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd \nimport glob\nimport random\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torch.nn import GroupNorm\nfrom torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\nimport torch.nn.functional as F\nfrom sklearn.model_selection import StratifiedKFold\nfrom iterstrat.ml_stratifiers import MultilabelStratifiedKFold\nimport torchvision.models as models\nimport albumentations as A\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom PIL import Image\nfrom tqdm.notebook import tqdm\nfrom sklearn.metrics import cohen_kappa_score, confusion_matrix\nimport pretrainedmodels\nimport cv2\nimport tarfile\nfrom functools import partial\nimport scipy as sp\n\nfrom colored import fg, bg, attr\nimport time\nfrom joblib import Parallel, delayed\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.distributed.xla_multiprocessing as xmp\n\nprint(os.listdir('/kaggle/input/'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATA_DIR = '/kaggle/input/prostate-cancer-grade-assessment/'\nIMG_DIR = '/kaggle/input/panda-td-tiles-16x128x128/'\nMARKER_DIR = '/kaggle/input/marker-images/marker_images/'\n\nn_folds=4\nseed=42\nsigmaX=10\nbs=16\ndevice='cuda' if torch.cuda.is_available() else 'cpu'\nnum_epochs=6\nLR=1e-4      # important. Performance significantly worse on 1e-3\nnum_tiles=16\nnum_classes=1\nDEBUG=True\narch = pretrainedmodels.__dict__['se_resnext50_32x4d']\nif DEBUG:\n    num_epochs=1\n    debug_samples=100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_everything(seed)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(DATA_DIR+'train.csv')\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ids = list(set([id.split('_')[0] for id in os.listdir(IMG_DIR)]))\n# marker_ids = list(set([id.split('.')[0] for id in os.listdir(MARKER_DIR)]))\ntrain = train[train['image_id'].isin(valid_ids)].reset_index(drop=True)\n# train = train[~train['image_id'].isin(marker_ids)].reset_index(drop=True)\n\nif DEBUG:\n    train = train.sample(n=debug_samples, random_state=seed).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('New shape: ', train.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Multilabel Stratified Folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = train[['isup_grade','data_provider']].values\nskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True, random_state=seed)\ntrain['fold'] = -1\n\nfor i,(trn_idx, val_idx) in enumerate(skf.split(train,Y)):\n    train.loc[val_idx,'fold'] = i","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# DataSet","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDA(Dataset):\n    def __init__(self,df, transform, mode='train'):\n        self.df = df\n        self.transform=transform\n        self.mode='train'\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        img_id = self.df['image_id'].values[idx]\n        provider = self.df['data_provider'].values[idx]\n        img = [cv2.imread(IMG_DIR+img_id+f'_{16-i-1}.png') for i in range(num_tiles)]\n        \n        img=cv2.hconcat([cv2.vconcat([img[0], img[1], img[2], img[3]]),\n                        cv2.vconcat([img[4], img[5], img[6], img[7]]),\n                        cv2.vconcat([img[8], img[9], img[10], img[11]]),\n                        cv2.vconcat([img[12], img[13], img[14], img[15]])])\n        \n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n          \n        if self.transform:\n            img = self.transform(image=img)['image']\n            \n        \n        if self.mode!='test':    \n            label = self.df['isup_grade'][idx]\n            \n    \n        return {'image': img,\n                'provider': provider,\n               'label': torch.tensor(label, dtype=torch.float)}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# New Layers","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"from torch.nn.parameter import Parameter\n\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM,self).__init__()\n        self.p = Parameter(torch.ones(1)*p)\n        self.eps = eps\n    def forward(self, x):\n        return gem(x, p=self.p, eps=self.eps)       \n    def __repr__(self):\n        return self.__class__.__name__ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + ', ' + 'eps=' + str(self.eps) + ')'\n    \n    \nclass Conv2d_ws(nn.Conv2d):\n\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1,padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros'):\n        super(nn.Conv2d, self).__init__(in_channels, out_channels, kernel_size, stride,padding, dilation, bias=True ,padding_mode='zeros',\n                                       groups=1, output_padding='zeros', transposed=False)\n\n\n\n    def forward(self, x):\n        weight = self.weight\n        weight_mean = weight.mean(dim=1, keepdim=True).mean(dim=2,\n                                  keepdim=True).mean(dim=3, keepdim=True)\n        weight = weight - weight_mean\n        std = weight.view(weight.size(0), -1).std(dim=1).view(-1, 1, 1, 1) + 1e-5\n        weight = weight / std.expand_as(weight)\n        return F.conv2d(x, weight, self.bias, self.stride,\n                        self.padding, self.dilation, self.groups)\n    \ndef convert_to_gem(model):\n    for child_name, child in model.named_children():\n        if isinstance(child, nn.AdaptiveAvgPool2d):\n            setattr(model, child_name, GeM())\n        else:\n            convert_to_gem(child)\n            \ndef convert_to_conv2d(model):\n    for child_name, child in model.named_children():\n        if child_name not in ['fc1','fc2']:\n            if isinstance(child, nn.Conv2d):\n                in_feat = child.in_channels\n                out_feat = child.out_channels\n                ker_size = child.kernel_size\n                stride = child.stride\n                padding = child.padding\n                dilation = child.dilation\n                groups = child.groups\n                setattr(model, child_name, Conv2d_ws(in_channels=in_feat, out_channels=out_feat, kernel_size=ker_size, stride=stride,padding = padding, dilation=dilation, groups=groups))\n            else:\n                convert_to_conv2d(child)\n                \ndef convert_to_groupnorm(model):\n    for child_name, child in model.named_children():\n            if isinstance(child, nn.BatchNorm2d):\n                num_features = child.num_features\n                setattr(model, child_name, GroupNorm(num_groups=32, num_channels=num_features))\n            else:\n                convert_to_groupnorm(child)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":" I have replaced the final avg_pool layer with GeM layer. Using Conv2d with Weight Standardization (WS) increases each epoch time by almost 4 minutes. \n GroupNorm paper: https://arxiv.org/pdf/1803.08494.pdf . \n Conv2d with WS paper: https://arxiv.org/abs/1903.10520","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class PANDA_MODEL(nn.Module):\n    def __init__(self, pretrained=False, classes=num_classes):\n        super(PANDA_MODEL, self).__init__()\n        \n        self.model = arch(pretrained=None)\n        if pretrained:\n            self.model.load_state_dict(torch.load('../input/pytorch-se-resnext/se_resnext50_32x4d-a260b3a4.pth'))\n        self.model.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.model.last_linear = nn.Linear(self.model.last_linear.in_features, num_classes)\n        \n    def forward(self, x):\n        x = self.model(x)\n        return x           ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# OptimizedRounder","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class OptimizedRounder():\n    def __init__(self):\n        self.coef_ = 0\n\n    def _kappa_loss(self, coef, X, y):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n\n        ll = cohen_kappa_score(y, X_p, weights='quadratic')\n        return -ll\n\n    def fit(self, X, y):\n        loss_partial = partial(self._kappa_loss, X=X, y=y)\n        initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]\n        self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')\n\n    def predict(self, X, coef):\n        X_p = np.copy(X)\n        for i, pred in enumerate(X_p):\n            if pred < coef[0]:\n                X_p[i] = 0\n            elif pred >= coef[0] and pred < coef[1]:\n                X_p[i] = 1\n            elif pred >= coef[1] and pred < coef[2]:\n                X_p[i] = 2\n            elif pred >= coef[2] and pred < coef[3]:\n                X_p[i] = 3\n            elif pred >= coef[3] and pred < coef[4]:\n                X_p[i] = 4\n            else:\n                X_p[i] = 5\n        return X_p\n\n    def coefficients(self):\n        return self.coef_['x']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train and Validation Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(dataloader, model, device, optimizer, criterion):\n\n    model.train()\n    train_loss=0\n    length = len(dataloader)\n    optimizer.zero_grad()\n    iterator = tqdm(enumerate(dataloader), total=length)\n    prediction, truth=[], []\n    \n    for i, batch in iterator:\n        img=batch['image'].to(device)\n        label=batch['label'].to(device)\n        \n        output=model(img)\n        loss=criterion(output.view(-1), label)\n        loss.backward()\n            \n#         optimizer.step()\n        xm.optimizer_step(optimizer, barrier=True)\n        optimizer.zero_grad()\n        \n        \n        prediction.append(output.detach().cpu().numpy())\n        truth.append(label.detach().cpu().numpy())\n        train_loss+=loss.item()/length\n    \n    prediction = np.concatenate(prediction)\n    truth = np.concatenate(truth)\n    \n    return train_loss, prediction, truth\n\n\ndef validate(dataloader, model, device, criterion):\n\n    model.eval()\n    val_loss=0\n    length = len(dataloader)\n    iterator = tqdm(enumerate(dataloader), total=length)\n    prediction, truth=[], []\n    kprediction, ktruth=[], []\n    rprediction, rtruth=[], []\n    \n    with torch.no_grad():\n        for i, batch in iterator:\n            img=batch['image'].to(device)\n            provider=batch['provider']\n            label=batch['label'].to(device)\n        \n            output=model(img)\n            loss=criterion(output.view(-1), label)\n            \n            pred = output.detach().cpu().numpy()\n            prediction.append(pred)\n            \n            kindex=[i for i,pro in enumerate(provider) if pro=='karolinska']\n            kprediction.append(pred[kindex])\n            ktruth.append(label.detach().cpu().numpy()[kindex])\n\n            rindex=[i for i,pro in enumerate(provider) if pro=='radboud']\n            rprediction.append(pred[rindex])\n            rtruth.append(label.detach().cpu().numpy()[rindex]) \n            \n            truth.append(label.detach().cpu().numpy())\n            val_loss+=loss.item()/length\n\n    prediction = np.concatenate(prediction)\n    truth = np.concatenate(truth)\n\n    kprediction = np.concatenate(kprediction)\n    ktruth = np.concatenate(ktruth)\n\n    rprediction = np.concatenate(rprediction)\n    rtruth = np.concatenate(rtruth)\n    \n    return val_loss, prediction, truth, kprediction, ktruth, rprediction, rtruth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tfm = A.Compose([A.HorizontalFlip(p=0.5),\n                       A.VerticalFlip(p=0.5),\n                      A.Normalize(mean=[0.485, 0.456, 0.406],   \n                                  std=[0.229, 0.224, 0.225]),\n                      ToTensorV2()])\n\nvalid_tfm = A.Compose([A.Normalize(mean=[0.485, 0.456, 0.406],\n                                   std=[0.229, 0.224, 0.225]),\n                      ToTensorV2()])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ntrain_fold = train[train['fold']!=0].reset_index(drop=True)\n\ntrain_dataset = PANDA(train_fold, train_tfm, mode='train')\n    \nplt.figure(figsize=(18,18))\nfor i in range(9):\n    plt.subplot(3,3,i+1)\n    index=np.random.randint(len(train_fold))\n    img, provider, label=train_dataset[index]['image'], train_dataset[index]['provider'], train_dataset[index]['label']\n    plt.title('{}, {}'.format(provider,label))\n    plt.imshow(img.numpy().transpose(1,2,0))\n    plt.axis('off')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main Training","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"fonts = [(fg(82),attr('reset')), (fg(169),attr('reset'))]\n\ndef train_fold(fold):\n    \n    \n    device = xm.xla_device(fold + 1)\n    \n    optimized_rounder = OptimizedRounder()\n        \n    val_fold = train[train['fold']==fold].reset_index(drop=True)\n    train_fold = train[train['fold']!=fold].reset_index(drop=True)\n\n    train_dataset = PANDA(train_fold, train_tfm, mode='train')\n    val_dataset = PANDA(val_fold, valid_tfm, mode='validate')\n    \n    train_loader = DataLoader(train_dataset, batch_size=bs, shuffle=True, num_workers=4)\n    val_loader = DataLoader(val_dataset, batch_size=bs, shuffle=False, num_workers=4)\n\n\n    model = PANDA_MODEL(pretrained=True)\n    model = model.to(device)\n\n\n    optimizer = optim.Adam(model.parameters(), lr=LR)\n    criterion = nn.MSELoss()\n    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, patience=2, factor=0.5, verbose=True)\n    \n    best_loss=np.inf\n    best_score=0\n    best_opt_score=0\n\n\n    for epoch in range(num_epochs):\n        start = time.time()\n        \n        train_loss, train_pred, train_truth = train_model(train_loader, model, device, optimizer, criterion)\n        train_score = cohen_kappa_score(np.round(train_pred), train_truth, weights='quadratic')\n        \n        val_loss, val_pred, val_truth, kpre, ktru, rpre, rtru = validate(val_loader, model, device, criterion)\n        val_score = cohen_kappa_score(np.round(val_pred), val_truth, weights='quadratic')\n        \n        optimized_rounder.fit(val_pred, val_truth)\n        coefficients = optimized_rounder.coefficients()\n        final_preds = optimized_rounder.predict(val_pred, coefficients)\n        \n        opt_val_score = cohen_kappa_score(final_preds, val_truth, weights='quadratic')\n        \n        scheduler.step(val_loss)\n        \n        end = time.time()\n        total = np.round(end-start,2)\n        \n        pre = 'Fold: %s{}%s'.format(fold+1)%fonts[0] + ' Epoch: %s{}%s'.format(epoch+1)%fonts[0]\n        content =   'Train Loss: %s{:.4f}%s'.format(train_loss) % fonts[1]+\\\n                    ' Val Loss: %s{:.4f}%s'.format(val_loss) % fonts[1]+\\\n                    ' Train Kappa: %s{:.4f}%s'.format(train_score) % fonts[1]+\\\n                    ' Val Kappa: %s{:.4f}%s'.format(val_score) % fonts[1]+\\\n                    ' Optimized Val Kappa: %s{:.4f}%s'.format(opt_val_score) % fonts[1]+\\\n                    ' Coefficients: %s{}%s \\n'.format(coefficients) % fonts[1]\n        time_ = 'Time: %s{}%s s'.format(total) % fonts[1]\n\n        print(pre+'\\n'+content+'\\n'+time_+'\\n')\n        \n        with open(f'log_{fold+1}.txt', 'a') as appender:\n                appender.write(content + '\\n')\n\n        if val_loss<best_loss:\n            torch.save(model.state_dict(), f'SE_RNXT50_loss_{fold}.pt')\n            best_loss=val_loss\n            loss_kpre, loss_ktru, loss_rpre, loss_rtru = kpre, ktru, rpre, rtru\n        \n        if opt_val_score>best_opt_score:\n            torch.save(model.state_dict(), f'SE_RNXT50_opt_qwk_{fold}.pt')\n            best_opt_score=opt_val_score\n            opt_coefs = coefficients\n            optqwk_kpre, optqwk_ktru, optqwk_rpre, optqwk_rtru = kpre, ktru, rpre, rtru\n        \n        if val_score>best_score:\n            torch.save(model.state_dict(), f'SE_RNXT50_qwk_{fold}.pt')\n            best_score=val_score\n            qwk_kpre, qwk_ktru, qwk_rpre, qwk_rtru = kpre, ktru, rpre, rtru\n    \n        recorder={'model': model.state_dict(), 'optim': optimizer.state_dict(), 'scheduler': scheduler.state_dict()}\n        torch.save(recorder, 'recorder.pth')\n    \n    # In each fold we have saved 3 models by monitoring on Validation loss, QWK score and Optimized QWK score.\n    # Here we find QWK score for each of it and their confusion matrices with Karolinska and Radboud separately. \n    \n    l_kscore = cohen_kappa_score(np.round(loss_kpre), loss_ktru, weights='quadratic')\n    l_rscore = cohen_kappa_score(np.round(loss_rpre), loss_rtru, weights='quadratic')\n    print('Fold %s{}%s, LOSS MODEL \\n'.format(fold+1) % fonts[0]+\\\n          'Karolinska QWK: %s{:.4f}%s \\n'.format(l_kscore) % fonts[1]+\\\n          'Radboud QWK: %s{:.4f}%s \\n'.format(l_rscore) % fonts[1])\n    \n    print('Confusion matrix for Karolinska, Fold {}, Loss model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(loss_ktru, np.round(loss_kpre))))\n    \n    print('Confusion matrix for Radboud, Fold {}, Loss model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(loss_rtru, np.round(loss_rpre))))\n    \n    q_kscore = cohen_kappa_score(np.round(qwk_kpre), qwk_ktru, weights='quadratic')\n    q_rscore = cohen_kappa_score(np.round(qwk_rpre), qwk_rtru, weights='quadratic')\n    print('Fold %s{}%s, QWK MODEL \\n'.format(fold+1) % fonts[0]+\\\n          'Karolinska QWK: %s{:.4f}%s \\n'.format(q_kscore) % fonts[1]+\\\n          'Radboud QWK: %s{:.4f}%s \\n'.format(q_rscore) % fonts[1])\n    \n    print('Confusion matrix for Karolinska, Fold {}, QWK model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(qwk_ktru, np.round(qwk_kpre))))\n    \n    print('Confusion matrix for Radboud, Fold {}, QWK model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(qwk_rtru, np.round(qwk_rpre))))\n    \n    optqwk_kpre = optimized_rounder.predict(optqwk_kpre, opt_coefs)\n    optqwk_rpre = optimized_rounder.predict(optqwk_rpre, opt_coefs)\n    oq_kscore = cohen_kappa_score(optqwk_kpre, optqwk_ktru, weights='quadratic')\n    oq_rscore = cohen_kappa_score(optqwk_rpre, optqwk_rtru, weights='quadratic')\n    print('Fold %s{}%s Optimized QWK MODEL \\n'.format(fold+1) % fonts[0]+\\\n          'Karolinska QWK: %s{:.4f}%s \\n'.format(oq_kscore) % fonts[1]+\\\n          'Radboud QWK: %s{:.4f}%s \\n'.format(oq_rscore) % fonts[1])\n    \n    print('Confusion matrix for Karolinska, Fold {}, Optimized QWK model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(optqwk_ktru, optqwk_kpre)))\n    \n    print('Confusion matrix for Radboud, Fold {}, Optimized QWK model \\n'.format(fold+1)+\\\n          '{} \\n'.format(confusion_matrix(optqwk_rtru, optqwk_rpre))) \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"Parallel(n_jobs=n_folds, backend=\"threading\")(delayed(train_fold)(i) for i in range(n_folds))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}