{"cells":[{"metadata":{},"cell_type":"markdown","source":"> # Finetuning Resnet50 plus cyclic learning rate\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom torchvision import datasets, models, transforms\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.utils import make_grid\nfrom torch.optim import lr_scheduler\nimport matplotlib.pyplot as plt\nfrom collections import OrderedDict\nimport os\nfrom functools import reduce\nimport random\nimport skimage.io\nimport importlib\nfrom PIL import Image\n#import pretrainedmodels\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import cohen_kappa_score\nfrom itertools import product\nimport copy","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"model_name=\"wide_resnet50_2\"\ninitial_lr=0.01\nl2_=0.0005\ncycles=6\nimage_shape=224\nepoch_cycle=60\nn_predictions=4\nweight_file = \"ens_%s_%d_l2_%.6f.pt\" % (model_name,image_shape,l2_)\n\nis_submitting=os.path.exists('../input/prostate-cancer-grade-assessment/test_images')\ntraining_mode = False\non_kaggle=True\n\n\nfilepath = \"../input/prostate-cancer-grade-assessment/train_images/fdb9d38ce2b5a1d31bb030cd2d3a03b9.tiff\"\ntrain_csv = \"../input/prostate-cancer-grade-assessment/train.csv\"\ntest_csv = \"../input/prostate-cancer-grade-assessment/test.csv\"\ntrain_path = \"../input/prostate-cancer-grade-assessment/train_images\"\nsubmission_path = '../input/prostate-cancer-grade-assessment/test_images'\n\nsub_data = pd.read_csv(test_csv)\nfull = pd.read_csv(train_csv)\n#Subset train.csv to rows for which images exist in \"train_images\" folder. Useful for test environment that dont contain all images\nfull_files = os.listdir(\"../input/prostate-cancer-grade-assessment/train_images\")\nfull_files = [f.replace(\".tiff\", \"\") for f in full_files]\navailable_files = list(set(full_files) & set(full['image_id']))\nfull = full.loc[full['image_id'].isin(available_files), :]\n\ntrain, test = train_test_split(full, test_size=.2,random_state=123)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_transform = {\n    \"train\": transforms.Compose([\n        transforms.RandomResizedCrop(image_shape,scale=(0.5,0.8)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                             std=[0.229, 0.224, 0.225])\n    ]),\n    \"test\": transforms.Compose([\n        transforms.RandomResizedCrop(image_shape,scale=(0.5,0.8)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                             std=[0.229, 0.224, 0.225])\n    ])\n}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read and resize image\nEach image is split into tiles of equal sizes (50x50) and tiles that have no data (all pixels are white) are discarded. The remaining tiles are arraged into a square grid.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\ndef tile(image, size=50):\n    h, w, c = image.shape\n    h_pad = size - h % size\n    w_pad = size - w % size\n    image2 = np.pad(image, ((0, h_pad), (0, w_pad), (0, 0)), constant_values=255)\n\n    n_h = int(image2.shape[0] / size)\n    n_w = int(image2.shape[1] / size)\n\n    tiles = list(product(range(n_h), range(n_w)))\n    #drop tiles that are blank\n    tiles2=[]\n    for t in tiles:\n        t_h, t_w = t\n        section=image2[t_h * size:(t_h + 1) * size, t_w * size:(t_w + 1) * size, :]\n        if not np.all(section.reshape((-1,))==255):\n            tiles2.append(section)\n    if len(tiles2)==0:\n        print(\"Warning: blank image: returning white image of shape 500,500,3\")\n        return np.zeros((500, 500, 3), dtype='uint8') + 255\n\n    n_total = len(tiles2)\n    sh = int(np.ceil(np.sqrt(n_total)))\n\n    im = np.zeros((sh * size, sh * size, 3), dtype='uint8') + 255\n\n    random.shuffle(tiles2)\n\n    k = 0\n    for i in range(sh):\n        for j in range(sh):\n            if k == len(tiles2): break\n            im[i * size:(i + 1) * size, j * size:(j + 1) * size, :] = tiles2[k]\n            k = k + 1\n    return im\n\n\ndef load_image(filepath):\n    img = skimage.io.MultiImage(filepath)\n    image = img[-1]\n    image = tile(image)\n    image = Image.fromarray(image).convert(\"RGB\")\n    return image\n\ntest_image=load_image(filepath)\nplt.imshow(test_image)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data loaders\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class PandaDataset(Dataset):\n    def __init__(self, df, image_dir, labels=False, transforms=None):\n        self.df = df\n        self.transforms = transforms\n        self.image_dir = image_dir\n        self.labels = labels # if labels is set to false, the __getitem__ function returns 0 as the label\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, item):\n        image = load_image(os.path.join(self.image_dir, self.df['image_id'].iloc[item] + \".tiff\"))\n        if self.transforms:\n            image = self.transforms(image)\n        if self.labels:\n            labs = self.df['isup_grade'].iloc[item]\n            return image, labs\n        else:\n            return image,0\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train, test and submission data loaders","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = PandaDataset(df=train, labels=True, image_dir=train_path, transforms=data_transform['train'])\ntrainloader = DataLoader(train_df, batch_size=12,\n                         shuffle=True, num_workers=4)\n\ntest_df = PandaDataset(df=test, labels=True, image_dir=train_path, transforms=data_transform['test'])\ntestloader = DataLoader(test_df, batch_size=12, shuffle=False, num_workers=4)\n\nsubmission_df = PandaDataset(sub_data, image_dir=submission_path, labels=False, transforms=data_transform['test'])\nsubmission_loader = DataLoader(submission_df, batch_size=12, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot: grid of train images","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"data_iter=iter(trainloader)\nss,_=data_iter.next()\ngrid=make_grid(ss, nrow=3)\nplt.imshow(grid.permute(1,2,0))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning rate schedule\nCyclic learning rate with cosine annealing is used. ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def schedule_fun(t,lr_max=initial_lr,lr_min=0.0,T=epoch_cycle):\n    t=t%T\n    return lr_min+0.5*(lr_max-lr_min)*(1+np.cos(t/T*np.pi))\n\nlearning_rate=[schedule_fun(i,lr_max=0.1,lr_min=0.0, T=60) for i in range(300)]\nplt.plot(learning_rate)\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning rate\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Wide resnet 50 model\nThe last layer of wide resnet 50 model is replaced with a dense layer with 6 output units (logits)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model=getattr(models,model_name)(pretrained=training_mode)\n#model = model = pretrainedmodels.__dict__[model_name](num_classes=1000,pretrained='imagenet')\nnum_ftrs = model.fc.in_features\nmodel.fc = nn.Linear(num_ftrs, 6)\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\noptimizer = optim.SGD(model.parameters(), lr=1.0)\nscheduler = lr_scheduler.LambdaLR(optimizer,schedule_fun,last_epoch=-1)\ncriterion = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Kappa\nFunction for calculating kappa using sklearn metrics.cohen_kappa_score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def kappa(y1, y2):\n    return cohen_kappa_score(y1, y2, labels=[0, 1, 2, 3, 4, 5], weights='quadratic')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model training\nThe models is trained for 360 epochs with cyclic learning rate. 6 cycles with 60 epochs each","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"weight_list = []\n\nepochs = cycles*epoch_cycle\nepoch = 0\nif training_mode:\n    losses=[]\n    kappas=[]\n    for epoch in range(epoch, epochs):\n        print(\"Epoch %d: learning rate=%.6f\" % (epoch, scheduler.get_last_lr()[0]))\n        train_loss = 0.0\n        test_loss = 0.0\n        kappa_train = []\n        kappa_test = []\n        model.train()\n        for images, labels in trainloader:\n            images = images.to(device)\n            labels = labels.to(device)\n            optimizer.zero_grad()\n            logits = model(images)\n            loss = criterion(logits, labels)\n\n            loss.backward()\n\n            optimizer.step()\n            pred_train = logits.argmax(dim=1)\n            kappa_train.append(kappa(pred_train.cpu(), labels.cpu()))\n            train_loss += loss.item() * images.size(0)\n\n        model.eval()\n        for images, labels in testloader:\n            with torch.no_grad():\n                images = images.to(device)\n                labels = labels.to(device)\n                logits = model(images)\n                loss = criterion(logits, labels)\n                pred_test = logits.argmax(dim=1)\n                kappa_test.append(kappa(pred_test.cpu(), labels.cpu()))\n                test_loss += loss.item() * images.size(0)\n        print(\"Epoch %d: loss %.3f val_loss %.3f | kappa %.3f val_kappa %.3f\" % (\n            epoch, train_loss / train_df.__len__(), test_loss / test_df.__len__(), np.mean(kappa_train),\n            np.mean(kappa_test)))\n        losses.append((train_loss / train_df.__len__(), test_loss / test_df.__len__()))\n        kappas.append((np.mean(kappa_train), np.mean(kappa_test)))\n        if (epoch+1) % epoch_cycle==0:\n            weight_list.append(model.state_dict())\n        scheduler.step()\n\n    torch.save(weight_list,weight_file)\n\n    #plot of training/test loss and kappa\n    fig,axes=plt.subplots(1,2)\n    axes[0].plot(losses)\n    axes[0].set_ylim((np.min(losses),min(3,np.max(losses))))\n    axes[0].set_title(\"Losses\")\n    axes[1].plot(kappas)\n    axes[1].set_title(\"Kappa\")\n    if display:\n        plt.show()\n    else:\n        plt.savefig(\"/home/pmwaniki/Dropbox/tmp/finetune_ens_%s_%d\" % (model_name,image_shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if on_kaggle:\n    weight_file2=os.path.join(\"../input/results3\",weight_file)\nelse:\n    weight_file2=weight_file\n\n\n\n\nif not training_mode: weight_list=torch.load(weight_file2,map_location=device)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_eval_ens=[]\nfor i in range(len(weight_list)):\n    pred_eval=[]\n    model.load_state_dict(weight_list[i])\n    model.eval()\n    for p in range(n_predictions):\n        pp=list()\n        for images, labels in testloader:\n            with torch.no_grad():\n                images = images.to(device)\n                logits = model(images)\n                probs=torch.nn.Softmax(dim=1)(logits)\n                if torch.cuda.is_available():\n                    probs=probs.cpu()\n                pp.append(probs)\n        pp=np.concatenate(pp)\n        pred_eval.append(pp)\n    pred_eval_ens.append(np.stack(pred_eval).mean(axis=0))\npred_eval_ens2=np.stack(pred_eval_ens,axis=0)\npred_eval_ens3=pred_eval_ens2.mean(axis=0)\npred_eval_cat=np.argmax(pred_eval_ens3,axis=1)\nprint(\"Kappa ensemble eval= %.3f\" % kappa(test['isup_grade'],pred_eval_cat))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_submitting:\n    pred_test_ens = []\n    for i in range(len(weight_list)):\n        pred_eval = []\n        model.load_state_dict(weight_list[i])\n        model.eval()\n        for p in range(n_predictions):\n            pp = list()\n            for images, labels in submission_loader:\n                with torch.no_grad():\n                    images = images.to(device)\n                    logits = model(images)\n                    probs = torch.nn.Softmax(dim=1)(logits)\n                    if torch.cuda.is_available():\n                        probs = probs.cpu()\n                    pp.append(probs)\n            pp = np.concatenate(pp)\n            pred_eval.append(pp)\n        pred_test_ens.append(np.stack(pred_eval).mean(axis=0))\n    pred_test_ens2 = np.stack(pred_test_ens, axis=0)\n    pred_test_ens3 = pred_test_ens2.mean(axis=0)\n    pred_test_cat = np.argmax(pred_test_ens3, axis=1)\nelse:\n    rand_preds = []\n    for i in range(len(sub_data)):\n        rand_preds.append(random.randint(0, 5))\n    pred_test_cat=np.array(rand_preds)\n\nsub_data['isup_grade'] = pred_test_cat\ntest_df = sub_data[[\"image_id\", \"isup_grade\"]]\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}