{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Mayo Clinic - STRIP AI - Transfer Learning with pytorch pretrained resnet\n\n\n#### Key notes:\n- this script is based on [pytorch transfer learning example](https://github.com/pytorch/tutorials/blob/master/beginner_source/transfer_learning_tutorial.py) using pretrained resnet\n- dataset used in this notebook are images resized to max(height,width)=500 using `PIL` (pillow) package\n    -  note this dataset is missing 2 images - one from *train* folder and one from *other* folder\n    -  for EDA and image processing, refer [this notebook](https://www.kaggle.com/code/xxxxyyyy80008/mayo-clinic-strip-ai-eda-and-image-processing) and this post [Process images with pyvips package and handle the memory limitation issue](https://www.kaggle.com/competitions/mayo-clinic-strip-ai/discussion/340052)\n- the pytorch pretrained models are downloaded from pytorch website and added to the notebook via `+Add Data`\n    -  this is to make the script still work when the notebook is set `offline`\n- for simplicity, there is no addtional image processing other than resizing the original images\n\n\n\n#### References: \n- [pytorch transfer learning totorial](https://github.com/pytorch/tutorials/blob/master/beginner_source/transfer_learning_tutorial.py)\n- [pytorch list of pretrained models](https://pytorch.org/vision/stable/models.html)","metadata":{}},{"cell_type":"raw","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T05:20:40.615014Z","iopub.execute_input":"2022-07-25T05:20:40.615484Z","iopub.status.idle":"2022-07-25T05:20:40.913042Z","shell.execute_reply.started":"2022-07-25T05:20:40.615392Z","shell.execute_reply":"2022-07-25T05:20:40.912243Z"}}},{"cell_type":"code","source":"import os\nnext(os.walk('/kaggle/input'))   ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:16.87893Z","iopub.execute_input":"2022-07-28T06:07:16.879473Z","iopub.status.idle":"2022-07-28T06:07:16.890176Z","shell.execute_reply.started":"2022-07-28T06:07:16.879439Z","shell.execute_reply":"2022-07-28T06:07:16.889203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#basic libs\n\nimport pandas as pd\nimport numpy as np\nimport os\nfrom pathlib import Path\n\nfrom datetime import datetime, timedelta\nimport time\nfrom dateutil.relativedelta import relativedelta\n\nimport gc\nimport copy\nimport shutil\n#additional data processing\n\nimport pyarrow.parquet as pq\nimport pyarrow as pa\n\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\n\n\n#visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n#load images\nimport matplotlib.image as mpimg\nimport PIL\nfrom PIL import Image\n\n\n\n\n#settings\npd.options.display.max_rows = 100\npd.options.display.max_columns = 100\n\nImage.MAX_IMAGE_PIXELS = None\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport pytorch_lightning as pl\nrandom_seed=1234\npl.seed_everything(random_seed)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:16.891934Z","iopub.execute_input":"2022-07-28T06:07:16.892312Z","iopub.status.idle":"2022-07-28T06:07:16.906872Z","shell.execute_reply.started":"2022-07-28T06:07:16.892276Z","shell.execute_reply":"2022-07-28T06:07:16.905471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torch.backends.cudnn as cudnn\nimport torchvision\nfrom torchvision import datasets, models, transforms\n\n\ncudnn.benchmark = True\nplt.ion()   # interactive mode","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:16.909074Z","iopub.execute_input":"2022-07-28T06:07:16.909496Z","iopub.status.idle":"2022-07-28T06:07:16.918163Z","shell.execute_reply.started":"2022-07-28T06:07:16.909463Z","shell.execute_reply":"2022-07-28T06:07:16.917075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load data and prep data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/train.csv')\nother_df = pd.read_csv('/kaggle/input/mayo-clinic-strip-ai/other.csv')","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:16.919647Z","iopub.execute_input":"2022-07-28T06:07:16.920244Z","iopub.status.idle":"2022-07-28T06:07:16.947875Z","shell.execute_reply.started":"2022-07-28T06:07:16.920208Z","shell.execute_reply":"2022-07-28T06:07:16.946895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"other_df['label'] = 'Other'","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:16.94925Z","iopub.execute_input":"2022-07-28T06:07:16.951361Z","iopub.status.idle":"2022-07-28T06:07:16.961406Z","shell.execute_reply.started":"2022-07-28T06:07:16.951332Z","shell.execute_reply":"2022-07-28T06:07:16.960231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([train_df[['image_id', 'patient_id', 'label']], \n                other_df[['image_id', 'patient_id', 'label']]], axis=0)\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:21.308263Z","iopub.execute_input":"2022-07-28T06:07:21.309446Z","iopub.status.idle":"2022-07-28T06:07:21.327481Z","shell.execute_reply.started":"2022-07-28T06:07:21.309403Z","shell.execute_reply":"2022-07-28T06:07:21.326564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_df, other_df\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:22.503253Z","iopub.execute_input":"2022-07-28T06:07:22.503704Z","iopub.status.idle":"2022-07-28T06:07:22.722007Z","shell.execute_reply.started":"2022-07-28T06:07:22.503666Z","shell.execute_reply":"2022-07-28T06:07:22.720976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the distribution of target labels\ndf['label'].value_counts()/df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:24.076525Z","iopub.execute_input":"2022-07-28T06:07:24.076952Z","iopub.status.idle":"2022-07-28T06:07:24.090257Z","shell.execute_reply.started":"2022-07-28T06:07:24.076914Z","shell.execute_reply":"2022-07-28T06:07:24.089027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert the lable into numeric type\nlabel_dict ={'CE':0, 'LAA':1, 'Other':2}\ndf['target']=df['label'].map(label_dict)\ndf['target'].value_counts()/df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:25.696987Z","iopub.execute_input":"2022-07-28T06:07:25.697564Z","iopub.status.idle":"2022-07-28T06:07:25.717597Z","shell.execute_reply.started":"2022-07-28T06:07:25.697518Z","shell.execute_reply":"2022-07-28T06:07:25.716678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:27.046326Z","iopub.execute_input":"2022-07-28T06:07:27.046748Z","iopub.status.idle":"2022-07-28T06:07:27.10494Z","shell.execute_reply.started":"2022-07-28T06:07:27.046714Z","shell.execute_reply":"2022-07-28T06:07:27.103873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split data into train and eval sets\nX_train, X_val, y_train, y_val = train_test_split(df[['image_id', 'label', 'target']], df['target'], test_size=0.25, random_state=1234)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:28.286192Z","iopub.execute_input":"2022-07-28T06:07:28.287261Z","iopub.status.idle":"2022-07-28T06:07:28.297068Z","shell.execute_reply.started":"2022-07-28T06:07:28.287221Z","shell.execute_reply":"2022-07-28T06:07:28.296113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the label distribution in train and val datasets\nprint('train data: ', len(X_train),  '-'*50)\ndisplay(X_train['label'].value_counts()/X_train.shape[0])\nprint('valuation data:', len(X_val), '-'*50)\ndisplay(X_val['label'].value_counts()/X_val.shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:29.563853Z","iopub.execute_input":"2022-07-28T06:07:29.564223Z","iopub.status.idle":"2022-07-28T06:07:29.579753Z","shell.execute_reply.started":"2022-07-28T06:07:29.564194Z","shell.execute_reply":"2022-07-28T06:07:29.578786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Dataset class","metadata":{}},{"cell_type":"code","source":"#define image dataset class\nimport torch\nfrom torch.utils.data import (Dataset, DataLoader)\n\n\n\nimg_folder = '/kaggle/input/stripai-traindata/train_images/all'\n\n\nclass IMG_Data(Dataset):\n    \n    def __init__(self, data): \n        \n\n        self.data = data\n        \n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        \n        image_id =self.data.iloc[idx]['image_id']\n        img_path = f\"({img_folder}/{image_id}.tif\"\n        y = self.data.iloc[idx]['target']\n        if Path(img_path).exists():\n            img = Image.open(img_path)\n            #transform image: crop image >> to tensor >> normalize\n            img = transforms.functional.center_crop(img, 500)\n            img = np.asarray(img, np.uint8)            \n        else:\n            img = np.zeros((500, 500, 3), np.uint8)\n            \n        x = torch.FloatTensor(img.transpose((2,0,1))  ) #need to slip the data in shape n_channels*height*width\n        \n        return x, y\n    \n\ndef load_data(df, batch_size, n_workers=0, shuffle=False):\n    data = IMG_Data(df)\n    \n    loader = DataLoader(data, batch_size=batch_size, num_workers=n_workers, shuffle=shuffle)\n    \n    return loader","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:31.965878Z","iopub.execute_input":"2022-07-28T06:07:31.966827Z","iopub.status.idle":"2022-07-28T06:07:31.977433Z","shell.execute_reply.started":"2022-07-28T06:07:31.96678Z","shell.execute_reply":"2022-07-28T06:07:31.976444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"######################################################################\n# Training the model\n# ------------------\n#\n# Now, let's write a general function to train a model. Here, we will\n# illustrate:\n#\n# -  Scheduling the learning rate\n# -  Saving the best model\n#\n# In the following, parameter ``scheduler`` is an LR scheduler object from\n# ``torch.optim.lr_scheduler``.\n\n\n\ndef train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch}/{num_epochs - 1}')\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            for inputs, labels in dataloaders[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')\n    print(f'Best val Acc: {best_acc:4f}')\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:33.319408Z","iopub.execute_input":"2022-07-28T06:07:33.319742Z","iopub.status.idle":"2022-07-28T06:07:33.332261Z","shell.execute_reply.started":"2022-07-28T06:07:33.319714Z","shell.execute_reply":"2022-07-28T06:07:33.331204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train model - resnet18: train the last layer\n\nThe following model training process will freeze all the network except the final layer.\n","metadata":{}},{"cell_type":"code","source":"pl.seed_everything(random_seed)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:35.766744Z","iopub.execute_input":"2022-07-28T06:07:35.767077Z","iopub.status.idle":"2022-07-28T06:07:35.775683Z","shell.execute_reply.started":"2022-07-28T06:07:35.76705Z","shell.execute_reply":"2022-07-28T06:07:35.774665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#load data to dataloaer\ndataloaders = {'train': load_data(X_train, 4), 'val': load_data(X_val, 4)}\ndataset_sizes = {'train': len(X_train), 'val':len(X_val)}","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:37.134621Z","iopub.execute_input":"2022-07-28T06:07:37.134961Z","iopub.status.idle":"2022-07-28T06:07:37.141186Z","shell.execute_reply.started":"2022-07-28T06:07:37.134934Z","shell.execute_reply":"2022-07-28T06:07:37.140253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:42.658926Z","iopub.execute_input":"2022-07-28T06:07:42.659293Z","iopub.status.idle":"2022-07-28T06:07:42.667063Z","shell.execute_reply.started":"2022-07-28T06:07:42.659263Z","shell.execute_reply":"2022-07-28T06:07:42.666107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n######################################################################\n# ConvNet as fixed feature extractor\n# ----------------------------------\n#\n# Here, we need to freeze all the network except the final layer. We need\n# to set ``requires_grad = False`` to freeze the parameters so that the\n# gradients are not computed in ``backward()``.\n#\n# You can read more about this in the documentation\n# `here <https://pytorch.org/docs/notes/autograd.html#excluding-subgraphs-from-backward>`__.\n#\n\nmodel_conv = models.resnet18()\nmodel_conv.load_state_dict(torch.load('/kaggle/input/pytorch-pretrained/resnet18-f37072fd.pth'))\nfor param in model_conv.parameters():\n    param.requires_grad = False\n\n# Parameters of newly constructed modules have requires_grad=True by default\nnum_ftrs = model_conv.fc.in_features\nmodel_conv.fc = nn.Linear(num_ftrs, 3)\n\nmodel_conv = model_conv.to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\n\n# Observe that only parameters of final layer are being optimized as\n# opposed to before.\noptimizer_conv = optim.SGD(model_conv.fc.parameters(), lr=0.001, momentum=0.9)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:47.532572Z","iopub.execute_input":"2022-07-28T06:07:47.532915Z","iopub.status.idle":"2022-07-28T06:07:51.420951Z","shell.execute_reply.started":"2022-07-28T06:07:47.532886Z","shell.execute_reply":"2022-07-28T06:07:51.420002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel_conv = train_model(model_conv, criterion, optimizer_conv,\n                         exp_lr_scheduler, num_epochs=50)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:07:52.615544Z","iopub.execute_input":"2022-07-28T06:07:52.615894Z","iopub.status.idle":"2022-07-28T06:12:55.589Z","shell.execute_reply.started":"2022-07-28T06:07:52.615865Z","shell.execute_reply":"2022-07-28T06:12:55.587985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train model - resnet50: train all layers\n\nThe following model training process will train all layers\n","metadata":{}},{"cell_type":"code","source":"model_ft = models.resnet50()\nmodel_ft.load_state_dict(torch.load('/kaggle/input/pytorch-pretrained/resnet50-0676ba61.pth'))\nnum_ftrs = model_ft.fc.in_features\n\n\nmodel_ft.fc = nn.Linear(num_ftrs, 3)\n\nmodel_ft = model_ft.to(device)\n\ncriterion = nn.CrossEntropyLoss()\n\n# Observe that all parameters are being optimized\noptimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)\n\nmodel_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,\n                       num_epochs=50)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T06:12:55.591007Z","iopub.execute_input":"2022-07-28T06:12:55.591622Z","iopub.status.idle":"2022-07-28T06:38:03.452878Z","shell.execute_reply.started":"2022-07-28T06:12:55.591583Z","shell.execute_reply":"2022-07-28T06:38:03.451779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}