{"cells":[{"metadata":{},"cell_type":"markdown","source":"The Kernel uses the EfficientNet B4 to get a LB 0.6. The model is trained with three stages of decremented learning rate.\nI'm yet to add CV Folds and this Kernel runs for only one fold. \nTo add in the Future - \n1. 4-Fold CV\n2. N_tiles\n3. Try Cyclic learning rates\n\nHope this helps the viewer. It is not supposed to be score grabber but a simple Pytorch Implementation for beginners.\nPlease Upvote if you like it.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Loading the Important Libraries","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os, sys, warnings, random, time, cv2\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport skimage.io\nfrom PIL import Image\nfrom sklearn import metrics\nfrom sklearn.model_selection import StratifiedKFold\nfrom IPython.display import display\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch\nimport albumentations\nfrom albumentations.pytorch import ToTensorV2\nfrom torch import nn, optim\nfrom torch.optim import lr_scheduler\nfrom torch.functional import F \nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n# from efficientnet_pytorch import model as enet\n\n%matplotlib inline\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.plotting.register_matplotlib_converters()\npd.options.display.max_rows=50\npd.options.display.max_columns=100\nplt.rcParams.update({'font.size':18})\nsns.set_style('darkgrid')\nplt.rcParams.update({'font.family':'Humor Sans'})\nplt.xkcd();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fixing Config","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 69\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)\npackage_path = '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master'\nsys.path.append(package_path)\nfrom efficientnet_pytorch import model as enet\n\nProgress_Bar = False\nDEBUG = False\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir = '../input/prostate-cancer-grade-assessment/'\ntrain_img_dir = os.path.join(data_dir, 'train_images')\ntrain_df = pd.read_csv(data_dir+'train.csv')\ntrain_df = train_df.sample(1000).reset_index(drop=True) if DEBUG else train_df\n\ndisplay(train_df.head())\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create Folds","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"skf = StratifiedKFold(5, shuffle=True, random_state=SEED)\ntrain_df['fold'] = -1\nfor i, (tr_idx, val_idx) in enumerate(skf.split(train_df, train_df['isup_grade'])):\n    train_df.loc[val_idx, 'fold'] = i\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns=['data_provider', 'gleason_score'], inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class Build_Dataset(Dataset):\n    '''Builds Dataset to be fed to Neural Network\n       :param df: train_df or test_df\n       :param resize: tuple, eg(256, 256)\n       :param mode: string train or test \n       :param: augmentations: Image augmentations\n    '''\n    def __init__(self, df, resize=None, mode='train', augmentations=None):\n        self.df = df\n        self.resize = resize\n        self.mode = mode\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        if self.mode == 'train':\n            img_path = os.path.join(train_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)\n            label = self.df['isup_grade'].values[idx]\n            \n        if self.mode == 'test':\n            img_path = os.path.join(test_img_dir, self.df['image_id'].values[idx]) + '.tiff'\n            image = skimage.io.MultiImage(img_path)\n            label = -1\n            \n        if self.resize is not None:\n            image = cv2.resize(image[-1], (self.resize[0], self.resize[1]))\n            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n            \n        image = np.array(image)\n        \n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            image = augmented[\"image\"]\n            \n        #pytorch expects CHW instead of HWC\n#         image = np.transpose(image, (2, 0, 1)).astype(np.float32)\n        \n        return image, label\n            ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plotting some Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(images):\n\n    n_images = len(images)\n\n    rows = int(np.sqrt(n_images))\n    cols = int(np.sqrt(n_images))\n\n    fig = plt.figure(figsize=(20,10))\n    for i in range(rows*cols):\n        ax = fig.add_subplot(rows, cols, i+1)\n        ax.imshow(np.array(images[i][0]))\n        ax.set_title('ISUP: '+str(images[i][1]))\n        ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N_IMAGES = 9\n\ntrain_data = Build_Dataset(train_df, resize=(512, 512), mode='train')\nimages = [(image, label) for image, label in [train_data[i] for i in range(N_IMAGES)]] \nplot_images(images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Processing The Images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Image-net standard mean and std\nmean = [0.485, 0.456, 0.406]\nstd = [0.229, 0.224, 0.225]\n\n#Defining train and test transforms\ntrain_transforms = albumentations.Compose([\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Normalize(mean=mean, std=std, always_apply=True),\n    albumentations.pytorch.ToTensorV2(),\n])\ntest_transforms = albumentations.Compose([\n    albumentations.Normalize(mean=mean, std=std, always_apply=True),\n    albumentations.pytorch.ToTensorV2(),\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"pretrainied_model = {\n    'efficientnet-b0': '../input/efficientnet-pytorch/efficientnet-b0-08094119.pth',\n    'efficientnet-b4': '../input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth'\n}\n\nclass enetv2(nn.Module):\n    def __init__(self, backbone, out_dim):\n        super(enetv2, self).__init__()\n        self.enet = enet.EfficientNet.from_name(backbone)\n        self.enet.load_state_dict(torch.load(pretrainied_model[backbone]))\n        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n        self.enet._fc = nn.Identity()\n    \n    def extract(self, x):\n        return self.enet(x)\n    \n    def forward(self, x):\n        x = self.extract(x)\n        x = self.myfc(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ndef epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = enetv2('efficientnet-b0', 5).to(device)\n# loss_criterion = nn.CrossEntropyLoss().to(device)\n# optimizer=optim.Adam(model.parameters())\n\n# print(f'The model has {count_parameters(model):,} trainable parameters')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Training and Validation epochs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, device):\n    \n    epoch_loss = 0\n    model.train()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    for (x, y) in bar:\n        \n        x = x.to(device, dtype=torch.float)\n        y = y.to(device, dtype=torch.long)\n        optimizer.zero_grad()\n        y_pred = model(x)\n        loss = criterion(y_pred, y)\n        loss.backward()\n        optimizer.step()\n        loss_np = loss.detach().cpu().numpy()\n        epoch_loss += loss_np\n        if Progress_Bar:\n            bar.set_description('Training loss: %.5f' % (loss_np))\n        \n    return epoch_loss/len(iterator)\n\ndef evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    preds = []\n    preds = np.array(preds)\n    targets = []\n    targets = np.array(targets)\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        \n        for (x, y) in bar:\n        \n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.long)\n            y_pred = model(x)\n            loss = criterion(y_pred, y)\n            loss_np = loss.detach().cpu().numpy()\n            epoch_loss += loss_np\n            preds = np.append(preds, np.argmax(y_pred.detach().cpu().numpy(), axis = 1))\n            targets = np.append(targets, y.detach().cpu().numpy())\n#             preds = preds.reshape(-1)\n#             targets = targets.reshape(-1)\n            \n            if Progress_Bar:\n                bar.set_description('Validation loss: %.5f' % (loss_np))\n            \n    \n            \n    return epoch_loss/len(iterator), metrics.cohen_kappa_score(targets, preds, weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Training Loop","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit_model(model, model_name, train_iterator, valid_iterator, optimizer, loss_criterion, device, epochs):\n    \"\"\" Fits a dataset to model\"\"\"\n    best_valid_loss = float('inf')\n    \n    train_losses = []\n    valid_losses = []\n    valid_metric_scores = []\n    \n    for epoch in range(epochs):\n    \n        start_time = time.time()\n    \n        train_loss = train(model, train_iterator, optimizer, loss_criterion, device)\n        valid_loss, valid_metric_score = evaluate(model, valid_iterator, loss_criterion, device)\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        valid_metric_scores.append(valid_metric_score)\n\n        if valid_loss < best_valid_loss:\n            best_valid_loss = valid_loss\n            torch.save(model.state_dict(), f'{model_name}.pt')\n    \n        end_time = time.time()\n\n        epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n    \n        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n        print(f'Train Loss: {train_loss:.3f}')\n        print(f'Val. Loss: {valid_loss:.3f} |  Val. Metric Score: {valid_metric_score:.3f}')\n        \n    return train_losses, valid_losses, valid_metric_scores\n        \n#     return pd.DataFrame({f'{model_name}_Training_Loss':train_losses, \n#                         f'{model_name}_Training_Acc':train_accs, \n#                         f'{model_name}_Validation_Loss':valid_losses, \n#                         f'{model_name}_Validation_Acc':valid_accs})","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training with 5-Fold CV","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_loss=[]\nval_loss=[]\nval_metric=[]\n\nfor fold in range(1):\n    print(f\"Fitting on Fold {fold+1}\")\n    #Make Train and Valid DataFrame from fold\n    train_df_fold = train_df[train_df['fold'] != fold]\n    valid_df_fold = train_df[train_df['fold'] == fold]\n    \n    #Build and load Dataset\n    train_data = Build_Dataset(train_df_fold, resize=(256, 256), mode='train', augmentations=train_transforms)\n    valid_data = Build_Dataset(valid_df_fold, resize=(256, 256), mode='train', augmentations=test_transforms)\n    train_iterator = DataLoader(train_data, shuffle=True, batch_size=16, num_workers=4)\n    valid_iterator = DataLoader(valid_data, batch_size=16, num_workers=4)\n    \n    #Initialize model, loss and optimizer\n    model = enetv2('efficientnet-b4', out_dim=6).to(device)\n    loss_criterion = nn.CrossEntropyLoss().to(device)\n    opt1=optim.Adam(model.parameters(), lr=0.01, betas=(0.9,0.999))\n    \n    temp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b4', train_iterator, valid_iterator, opt1, loss_criterion, device, epochs=3)\n    \n    tr_loss+=temp_tr_loss\n    val_loss+=temp_val_loss\n    val_metric+=temp_val_metric\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt2 = optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999))\ntemp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b4', train_iterator, valid_iterator, opt2, loss_criterion, device, epochs=4)\n\ntr_loss+=temp_tr_loss\nval_loss+=temp_val_loss\nval_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt3 = optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.999))\ntemp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b4', train_iterator, valid_iterator, opt3, loss_criterion, device, epochs=3)\n\ntr_loss+=temp_tr_loss\nval_loss+=temp_val_loss\nval_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"opt4 = optim.Adam(model.parameters(), lr=0.00001, betas=(0.9, 0.99))\ntemp_tr_loss, temp_val_loss, temp_val_metric = fit_model(model, 'efficientnet-b4', train_iterator, valid_iterator, opt4, loss_criterion, device, epochs=4)\n\ntr_loss+=temp_tr_loss\nval_loss+=temp_val_loss\nval_metric+=temp_val_metric","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Plotting the Losses and the Metric**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tr_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.rcParams.update({'font.size':18})\nsns.set_style('darkgrid')\nplt.rcParams.update({'font.family':'Humor-Sans'})\n\nfig,ax = plt.subplots(nrows=1, ncols=2, figsize=(20,5))\nax[0].plot(tr_loss)\nax[0].set_title('Training and Validation Loss')\nax[0].plot(val_loss)\nax[0].set_ylim((1,2))\nax[0].set_xlabel('Epoch')\n\nax[1].plot(val_metric)\nax[1].set_title('Val Cohen Score')\nax[1].set_xlabel('Epoch')\n\n\nax[0].legend();\nax[1].legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Making submission to leaderboard","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_predictions(model, iterator, device):\n    \n    preds = []\n    model.eval()\n    bar = tqdm(iterator) if Progress_Bar else iterator\n    \n    with torch.no_grad():\n        \n        for (x, y) in bar:\n        \n            x = x.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.long)\n            y_pred = model(x)\n            preds.append(np.argmax(y_pred.detach().cpu().numpy(), axis = 1))\n            \n    preds = np.array(preds)\n    preds = preds.reshape(-1)\n            \n    return preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(data_dir+'test.csv')\nsample = pd.read_csv('../input/prostate-cancer-grade-assessment/sample_submission.csv')\ntest_df.drop(columns=['data_provider'], inplace=True)\ntest_img_dir = '../input/prostate-cancer-grade-assessment/test_images'\n    \n# #Build and load Test Data\n# test_data = Build_Dataset(test_df, resize=(256, 256), mode='test', augmentations=test_transforms)\n# test_iterator = DataLoader(test_data, batch_size=2, num_workers=4)\n    \n# #Get predictions\n# y_pred = get_predictions(model, test_iterator, device)\n    \n# #Submit Predictions\n# test_df['isup_grade'] = y_pred\n# test_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def submit(sample):\n    if os.path.exists('../input/prostate-cancer-grade-assessment/test_images'):\n        test_data = Build_Dataset(test_df, resize=(256, 256), mode='test', augmentations=test_transforms)\n        test_iterator = DataLoader(test_data, batch_size=2, num_workers=4)\n        preds = get_predictions(model, test_iterator, device)\n        sample['isup_grade'] = preds\n    return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = submit(sample)\nsubmission['isup_grade'] = submission['isup_grade'].astype(int)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}