{"cells":[{"metadata":{},"cell_type":"markdown","source":"The Kernel uses the EfficientNet B4 to get a LB 0.6. The model is trained with three stages of decremented learning rate.\nI'm yet to add CV Folds and this Kernel runs for only one fold. \nTo add in the Future - \n1. 4-Fold CV\n2. N_tiles\n3. Try Cyclic learning rates\n\nHope this helps the viewer. It is not supposed to be score grabber but a simple Pytorch Implementation for beginners.\nPlease Upvote if you like it.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Loading the Important Libraries","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, sys, warnings, random, time, cv2\nimport pandas as pd\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport skimage.io\nfrom PIL import Image\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nimport albumentations\n\nfrom albumentations.pytorch import ToTensorV2\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n# from efficientnet_pytorch import model as enet\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.plotting.register_matplotlib_converters()\npd.options.display.max_rows=50\npd.options.display.max_columns=100\nplt.rcParams.update({'font.size':18})\nsns.set_style('darkgrid')\nplt.rcParams.update({'font.family':'Humor Sans'})\nplt.xkcd();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fixing Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 69\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\npackage_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\nfrom efficientnet_pytorch import model as enet\n\nProgress_Bar = True\nDEBUG = False\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment/'\ntrain_img_dir = os.path.join(data_dir, 'train_images')\ntrain_df = pd.read_csv(data_dir+'train.csv')\ntrain_df = train_df.sample(1000).reset_index(drop=True) if DEBUG else train_df\n\ndisplay(train_df.head())\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(5, shuffle=True, random_state=SEED)\ntrain_df['fold'] = -1\nfor i, (tr_idx, val_idx) in enumerate(skf.split(train_df, train_df['isup_grade'])):\n    train_df.loc[val_idx, 'fold'] = i\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['data_provider', 'gleason_score'], inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Build_Dataset(Dataset):\n    '''Builds Dataset to be fed to Neural Network\n       :param df: train_df or test_df\n       :param resize: tuple, eg(256, 256)\n       :param mode: string train or test \n       :param: augmentations: Image augmentations\n    '''\n    def __init__(self, df, mode='train', augmentations=None, sz=128, n_tiles=16):\n        self.df = df\n        self.mode = mode\n        self.augmentations = augmentations\n        self.N = n_tiles\n        self.sz = sz\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        if self.mode == 'train':\n            img_path = os.path.join(train_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)[-1]\n            label = self.df['isup_grade'].values[idx]\n            \n        if self.mode == 'test':\n            img_path = os.path.join(test_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)[-1]\n            label = -1\n        \n        N = self.N\n        sz = self.sz\n        pad0, pad1 = (sz - image.shape[0]%sz)%sz, (sz - image.shape[1]%sz)%sz\n        image = np.pad(image, [[pad0//2, pad0-pad0//2], [pad1//2, pad1-pad1//2], [0,0]], constant_values=255)\n        image = image.reshape(image.shape[0]//sz, sz, image.shape[1]//sz, sz, 3)\n        image = image.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n        if len(image)<N:\n            image = np.pad(image, [[0,N-len(image)], [0,0], [0,0], [0,0]], constant_values=255)\n        idxs = np.argsort(image.reshape(image.shape[0],-1).sum(-1))[:N]\n        tiles = image[idxs]\n\n#         tiles = self.make_tiles(image)\n        \n        \n        image = tiles\n#         for img in tiles:\n#             if self.augmentations:\n#                 augmented = self.augmentations(image=img)\n#                 img = augmented['image']\n#             image.append(img)\n            \n        image = cv2.vconcat([cv2.hconcat([image[0], image[1], image[2], image[3]]), \n                             cv2.hconcat([image[4], image[5], image[6], image[7]]), \n                             cv2.hconcat([image[8], image[9], image[10], image[11]]), \n                             cv2.hconcat([image[12], image[13], image[14], image[15]])])\n            \n#         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        image = np.array(image)\n        \n        if self.augmentations:\n                augmented = self.augmentations(image=image)\n                image = augmented['image']\n        \n#         final_img = []\n#         for i in range(4):\n#             img_row=[]\n#             for j in range(4):\n#                 img_row.append(image[4*i+j])\n#             final_img.append(img_row)\n#         image = np.array(final_img).transpose(0,2,1,3,4).reshape(512,512,3)\n        \n        return image, label\n     \n#     @classmethod    \n#     def makes_tiles(self, image):\n#         N = self.N\n#         sz = self.sz\n#         pad0, pad1 = (sz - img.shape[0]%sz)%sz, (sz - img.shape[1]%sz)%sz\n#         image = np.pad(image, [[pad0//2, pad0-pad0//2], [pad1//2, pad1-pad1//2], [0,0]])\n#         image = image.resize(image.shape[0]//sz, sz, image[1]//sz, sz, 3)\n#         image = image.transpose(0,2,1,3,4).reshape(-1,sz,sz,3)\n#         if len(image)<N:\n#             image = np.pad(image, [[0,N-len(image)], [0,0], [0,0], [0,0]])\n#         idxs = np.argsort(image.reshape(image.shape[0],-1).sum(-1))[:N]\n#         image = image[idxs]\n#         return image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting some Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(images):\n\n#     n_images = len(images)\n#     fig,ax = plt.subplots(nrows=4, ncols=4, figsize=(7,7), sharex=True, sharey=True)\n#     for i in range(4):\n#         for j in range(4):\n#             ax[i,j].imshow(images[4*i+j])\n    fig = plt.figure(figsize=(10,10))\n    plt.imshow(images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nN_IMAGES = 1\n\ntrain_data = Build_Dataset(train_df, sz=128, mode='train')\nimage,label = train_data[10]\nprint(image.shape)\nplot_images(image)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Processing The Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Image-net standard mean and std\n# mean = [0.485, 0.456, 0.406]\n# std = [0.229, 0.224, 0.225]\n\nmean = [0.90949707, 0.8188697,  0.87795304]\nstd = [0.36357649, 0.49984502, 0.40477625]\n\n#Defining train and test transforms\ntrain_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Normalize(mean=mean, std=std, always_apply=True),\n    albumentations.pytorch.ToTensorV2(),\n])\ntest_transforms = albumentations.Compose([\n    albumentations.Normalize(mean=mean, std=std, always_apply=True),\n    albumentations.pytorch.ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrainied_model = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    'efficientnet-b4': '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth'\n}\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrainied_model[backbone]))\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n    \n    def extract(self, x):\n        return self.enet(x)\n    \n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = enetv2('efficientnet-b0', 5).to(device)\n# loss_criterion = nn.CrossEntropyLoss().to(device)\n# optimizer=optim.Adam(model.parameters())\n\n# print(f'The model has {count_parameters(model):,} trainable parameters')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Training and Validation epochs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    for (x, y) in bar:\n        \n        x = x.to(device, dtype=torch.float)\n        y = y.to(device, dtype=torch.long)\n        optimizer.zero_grad()\n        y_pred = model(x)\n        loss = criterion(y_pred, y)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        if Progress_Bar:\n            bar.set_description('Training loss: %.5f' % (loss_np))\n        \n    return epoch_loss/len(iterator)\n\ndef evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    preds = []\n    preds = np.array(preds)\n    targets = []\n    targets = np.array(targets)\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        \n        for (x, y) in bar:\n        \n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.long)\n            y_pred = model(x)\n            loss = criterion(y_pred, y)\n            loss_np = loss.detach().cpu().numpy()\n            epoch_loss += loss_np\n            preds = np.append(preds, np.argmax(y_pred.detach().cpu().numpy(), axis = 1))\n            targets = np.append(targets, y.detach().cpu().numpy())\n#             preds = preds.reshape(-1)\n#             targets = targets.reshape(-1)\n            \n            if Progress_Bar:\n                bar.set_description('Validation loss: %.5f' % (loss_np))\n            \n    \n            \n    return epoch_loss/len(iterator), metrics.cohen_kappa_score(targets, preds, weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Training Loop","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, loss_criterion, device, epochs):\n    \"\"\" Fits a dataset to model\"\"\"\n    best_valid_loss = float('inf')\n    \n    train_losses = []\n    valid_losses = []\n    valid_metric_scores = []\n    \n    for epoch in range(epochs):\n    \n        start_time = time.time()\n    \n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        valid_loss, valid_metric_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_metric_scores.append(valid_metric_score)\n\n        if valid_loss > best_valid_loss:\n            best_valid_loss = valid_loss\n            torch.save(model.state_dict(), f'{model_name}.pt')\n    \n        end_time = time.time()\n\n        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n    \n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'Train Loss: {train_loss:.3f}')\n        print(f'Val. Loss: {valid_loss:.3f} |  Val. Metric Score: {valid_metric_score:.3f}')\n        \n    return train_losses, valid_losses, valid_metric_scores\n        \n#     return pd.DataFrame({f'{model_name}_Training_Loss':train_losses, \n#                         f'{model_name}_Training_Acc':train_accs, \n#                         f'{model_name}_Validation_Loss':valid_losses, \n#                         f'{model_name}_Validation_Acc':valid_accs})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training with 5-Fold CV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_loss=[]\nval_loss=[]\nval_metric=[]\n\nfor fold in range(1):\n    print(f\"Fitting on Fold {fold+1}\")\n    #Make Train and Valid DataFrame from fold\n    train_df_fold = train_df[train_df['fold'] != fold]\n    valid_df_fold = train_df[train_df['fold'] == fold]\n    \n    #Build and load Dataset\n    train_data = Build_Dataset(train_df_fold, mode='train', augmentations=train_transforms)\n    valid_data = Build_Dataset(valid_df_fold, mode='train', augmentations=test_transforms)\n    train_iterator = DataLoader(train_data, shuffle=True, batch_size=16, num_workers=4)\n    valid_iterator = DataLoader(valid_data, batch_size=16, num_workers=4)\n    \n    #Initialize model, loss and optimizer\n    model = enetv2('efficientnet-b0', out_dim=6).to(device)\n    loss_criterion = nn.CrossEntropyLoss().to(device)\n    opt1=optim.Adam(model.parameters(), lr=1e-3, betas=(0.9,0.999))\n    \n    temp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b0', train_iterator, valid_iterator, opt1, loss_criterion, device, epochs=3)\n    \n    tr_loss+=temp_tr_loss\n    val_loss+=temp_val_loss\n    val_metric+=temp_val_metric\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt2 = optim.Adam(model.parameters(), lr=1e-4, betas=(0.9, 0.999))\ntemp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b0', train_iterator, valid_iterator, opt2, loss_criterion, device, epochs=4)\n\ntr_loss+=temp_tr_loss\nval_loss+=temp_val_loss\nval_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt3 = optim.Adam(model.parameters(), lr=1e-5, betas=(0.9, 0.999))\ntemp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b0', train_iterator, valid_iterator, opt3, loss_criterion, device, epochs=2)\n\ntr_loss+=temp_tr_loss\nval_loss+=temp_val_loss\nval_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# opt4 = optim.Adam(model.parameters(), lr=0.00001, betas=(0.9, 0.99))\n# temp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b4', train_iterator, valid_iterator, opt4, loss_criterion, device, epochs=4)\n\n# tr_loss+=temp_tr_loss\n# val_loss+=temp_val_loss\n# val_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Plotting the Losses and the Metric**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams.update({'font.size':18})\nsns.set_style('darkgrid')\nplt.rcParams.update({'font.family':'Humor-Sans'})\n\nfig,ax = plt.subplots(nrows=1, ncols=2, figsize=(15,4))\nax[0].plot(tr_loss)\nax[0].set_title('Training and Validation Loss')\nax[0].plot(val_loss)\nax[0].set_ylim((0,2))\nax[0].set_xlabel('Epoch')\n\nax[1].plot(val_metric)\nax[1].set_title('Val Cohen Score')\nax[1].set_xlabel('Epoch')\n\n\nax[0].legend();\nax[1].legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntorch.save(model.state_dict(), f'enetb0-trained.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Making submission to leaderboard","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(model, iterator, device):\n    \n    preds = []\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        \n        for (x, y) in bar:\n        \n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.long)\n            y_pred = model(x)\n            preds.append(np.argmax(y_pred.detach().cpu().numpy(), axis = 1))\n            \n    preds = np.array(preds)\n    preds = preds.reshape(-1)\n            \n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(data_dir+'test.csv')\nsample = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')\ntest_df.drop(columns=['data_provider'], inplace=True)\ntest_img_dir = '../input/prostate-cancer-grade-assessment/test_images'\n    \n# #Build and load Test Data\n# test_data = Build_Dataset(test_df, resize=(256, 256), mode='test', augmentations=test_transforms)\n# test_iterator = DataLoader(test_data, batch_size=2, num_workers=4)\n    \n# #Get predictions\n# y_pred = get_predictions(model, test_iterator, device)\n    \n# #Submit Predictions\n# test_df['isup_grade'] = y_pred\n# test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submit(sample):\n    if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):\n        test_data = Build_Dataset(test_df, resize=(256, 256), mode='test', augmentations=test_transforms)\n        test_iterator = DataLoader(test_data, batch_size=2, num_workers=4)\n        preds = get_predictions(model, test_iterator, device)\n        sample['isup_grade'] = preds\n    return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = submit(sample)\nsubmission['isup_grade'] = submission['isup_grade'].astype(int)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}