{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install lightning","metadata":{"execution":{"iopub.status.busy":"2023-11-30T01:35:00.453794Z","iopub.execute_input":"2023-11-30T01:35:00.454232Z","iopub.status.idle":"2023-11-30T01:35:17.545711Z","shell.execute_reply.started":"2023-11-30T01:35:00.454202Z","shell.execute_reply":"2023-11-30T01:35:17.544236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport albumentations as A\nimport torch\nimport cv2\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nBATCH_SIZE = 16\nOUT_DIR = \"/kaggle/working/log/freeze\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n        \n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-30T02:12:38.85998Z","iopub.execute_input":"2023-11-30T02:12:38.86042Z","iopub.status.idle":"2023-11-30T02:12:38.869063Z","shell.execute_reply.started":"2023-11-30T02:12:38.86036Z","shell.execute_reply":"2023-11-30T02:12:38.868193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv');\ndf = df[df.is_tma == False]\ndf.shape\n#data_csv.iloc[1].image_id\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T01:36:18.143426Z","iopub.execute_input":"2023-11-30T01:36:18.144955Z","iopub.status.idle":"2023-11-30T01:36:18.189998Z","shell.execute_reply.started":"2023-11-30T01:36:18.144905Z","shell.execute_reply":"2023-11-30T01:36:18.188591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom albumentations.augmentations.geometric.resize import LongestMaxSize\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom albumentations.augmentations.crops.transforms import CenterCrop\nfrom torchvision import transforms\ncode_mapping = {\"HGSC\": 0,\n      \"LGSC\": 1,\n      \"EC\": 2,\n      \"CC\": 3,\n      \"MC\": 4}\n\nclass ubc_data(Dataset):\n    def __init__(self, df):\n        #self.vl = var_list\n        self.df = df\n    def __getitem__(self,idx):\n        cur_row = self.df.iloc[idx]\n        img = cv2.imread(f\"/kaggle/input/UBC-OCEAN/train_thumbnails/{cur_row.image_id}_thumbnail.png\")\n        #print(type(img))\n        transform = A.Compose([LongestMaxSize(224), A.PadIfNeeded(min_height=224, min_width=224, border_mode=0, value=(0,0,0)), ToTensorV2()])\n        norm = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.22])\n        return norm(transform(image=img)[\"image\"].float()), code_mapping[cur_row.label]\n\n        \n    def __len__(self):\n        return len(self.df.index)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T01:36:23.713205Z","iopub.execute_input":"2023-11-30T01:36:23.714653Z","iopub.status.idle":"2023-11-30T01:36:23.724695Z","shell.execute_reply.started":"2023-11-30T01:36:23.714599Z","shell.execute_reply":"2023-11-30T01:36:23.723761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, val_df = train_test_split(df, test_size=0.20)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T01:36:26.524489Z","iopub.execute_input":"2023-11-30T01:36:26.524932Z","iopub.status.idle":"2023-11-30T01:36:26.532506Z","shell.execute_reply.started":"2023-11-30T01:36:26.5249Z","shell.execute_reply":"2023-11-30T01:36:26.53108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_data = ubc_data(train_df)\nval_data = ubc_data(val_df)\ntrain_loader = DataLoader(train_data, \n                          batch_size = BATCH_SIZE, \n                          num_workers=3,\n                          shuffle = True)\nval_loader = DataLoader(val_data, \n                          batch_size = BATCH_SIZE, \n                        num_workers=3,\n                          shuffle = False)\nprint(train_data[0][0].shape)\nimage = train_data[0][0].permute(1, 2, 0)\nplt.imshow(image.numpy())\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T02:13:22.35338Z","iopub.execute_input":"2023-11-30T02:13:22.353898Z","iopub.status.idle":"2023-11-30T02:13:22.884049Z","shell.execute_reply.started":"2023-11-30T02:13:22.35386Z","shell.execute_reply":"2023-11-30T02:13:22.882398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\n\nclass my_resnet18(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.net = torch.hub.load('pytorch/vision:v0.10.0', 'resnet18', pretrained=True)\n        ct = 0\n        for child in self.net.children():\n            ct += 1\n            if ct < 8:\n                for param in child.parameters():\n                    param.requires_grad = False\n            print(child.name)\n        print(ct)\n        \n    def forward(self,x):\n        return self.net(x)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:31:13.032899Z","iopub.execute_input":"2023-11-30T03:31:13.033325Z","iopub.status.idle":"2023-11-30T03:31:13.041831Z","shell.execute_reply.started":"2023-11-30T03:31:13.033292Z","shell.execute_reply":"2023-11-30T03:31:13.040255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model\n\n\nimport lightning.pytorch as pl\nimport torch.nn.functional as F\nfrom sklearn.metrics import accuracy_score\n\nclass my_resnet18_PL(pl.LightningModule):\n    def __init__(self, resnet):\n        super().__init__()\n        #self.num_outs = num_classes\n        self.acc_list = []\n        self.E_acc = []\n        self.val_lost_list = []\n        self.E_val_loss = []\n        self.train_lost_list = []\n        self.E_train_loss = []\n        self.epoch = 0\n        self.resnet = resnet\n        self.classifier = nn.Sequential(nn.Linear(1000, 5))\n        \n    def forward(self, x):\n        return self.classifier(self.resnet(x))\n    def training_step(self, batch, batch_idx):\n        # training_step defines the train loop.\n        x, y_t = batch\n        \n        y_p = self(x)\n        #print(\"xshape\", x.shape)\n        #print(\"y_P\", y_p.shape)\n        \n        #print(\"y_p\", y_p)\n        #print(\"x\", x)\n        loss = F.cross_entropy(y_p, y_t)\n        self.train_lost_list.append(loss.item())\n        return loss\n    \n   \n    def validation_step(self, batch, batch_idx):\n        with torch.no_grad():\n            # training_step defines the train loop.\n            #print(\"batch\", batch)\n            x, y_t = batch\n            y_p = self(x)\n            #print(\"y_p\", y_p)\n            #print(\"y_t\", y_t)\n            loss = F.cross_entropy(y_p, y_t)\n            temp = torch.zeros(BATCH_SIZE, 5)\n            #for i in range(BATCH_SIZE):\n            #    temp[i, y_t[i]] = 1\n            #y_t = temp\n            #print(y_t.shape, y_p.shape)\n            #print(y_p)\n            #print(torch.argmax(y_p, dim=1))\n            y_p = torch.argmax(y_p, dim=1)\n            acc = accuracy_score(y_t, y_p)\n\n            #self.log(\"val_loss\", loss.item(),prog_bar=True)\n            self.acc_list.append(acc)\n            self.val_lost_list.append(loss.item())\n            #return loss\n    \n    def on_train_epoch_end(self) -> None:\n        loss = sum(self.train_lost_list) / len(self.train_lost_list)\n        self.E_train_loss.append(loss)\n    \n    def on_validation_epoch_end(self) -> None:\n       \n        loss = sum(self.val_lost_list) / len(self.val_lost_list)\n        acc = sum(self.acc_list) / len(self.acc_list)\n        self.E_val_loss.append(loss)\n        self.E_acc.append(acc)\n        \n        if ((self.epoch+1) % 2) == 0:\n            figure_1, train_ax = plt.subplots()\n            figure_2, valid_ax = plt.subplots()\n            #print(model.train_lost_list)\n            train_ax.plot(self.E_train_loss, color='blue')\n            train_ax.set_xlabel('epochs')\n            train_ax.set_ylabel('train loss')\n            valid_ax.plot(self.E_val_loss, color='red')\n            valid_ax.plot(self.E_acc, color='green')\n            valid_ax.set_xlabel('epochs')\n            valid_ax.set_ylabel('metric')\n            figure_1.savefig(f\"{OUT_DIR}/train_loss_{self.epoch+1}.png\")\n            figure_2.savefig(f\"{OUT_DIR}/valid_loss_{self.epoch+1}.png\")\n            torch.save(self.resnet.state_dict(), \n                f\"{OUT_DIR}/resnet_{self.epoch+1}_model.pth\")\n            print(f'SAVING PLOTS COMPLETE...{self.epoch+1}')\n        self.epoch += 1\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(), lr=0.04)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T02:22:06.524154Z","iopub.execute_input":"2023-11-30T02:22:06.525108Z","iopub.status.idle":"2023-11-30T02:22:06.543582Z","shell.execute_reply.started":"2023-11-30T02:22:06.525059Z","shell.execute_reply":"2023-11-30T02:22:06.541335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = my_resnet18_PL(my_resnet18())","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:31:16.945912Z","iopub.execute_input":"2023-11-30T03:31:16.946345Z","iopub.status.idle":"2023-11-30T03:31:17.314259Z","shell.execute_reply.started":"2023-11-30T03:31:16.946309Z","shell.execute_reply":"2023-11-30T03:31:17.31204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer = pl.Trainer(max_epochs=50, default_root_dir=\"/kaggle/working/log\", log_every_n_steps=13)\ntrainer.fit(model=model, train_dataloaders=train_loader, val_dataloaders = val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-11-30T02:24:41.496288Z","iopub.execute_input":"2023-11-30T02:24:41.497718Z","iopub.status.idle":"2023-11-30T03:07:02.562275Z","shell.execute_reply.started":"2023-11-30T02:24:41.497651Z","shell.execute_reply":"2023-11-30T03:07:02.560506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /kaggle/working/log/freeze/","metadata":{"execution":{"iopub.status.busy":"2023-11-30T02:24:30.593672Z","iopub.execute_input":"2023-11-30T02:24:30.594991Z","iopub.status.idle":"2023-11-30T02:24:31.665127Z","shell.execute_reply.started":"2023-11-30T02:24:30.594914Z","shell.execute_reply":"2023-11-30T02:24:31.663738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(model.E_val_loss[:15])\nprint(model.E_acc[:])\nprint(model.E_train_loss[:15])","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:10:55.022941Z","iopub.execute_input":"2023-11-30T03:10:55.023476Z","iopub.status.idle":"2023-11-30T03:10:55.032391Z","shell.execute_reply.started":"2023-11-30T03:10:55.023432Z","shell.execute_reply":"2023-11-30T03:10:55.030569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ndevice = DEVICE\ny_gt_all = []\ny_pr_all = []\nmodel.eval()\nwith torch.no_grad():  # declare no gradient operations\n    for step, minibatch in enumerate(val_loader):\n        # 1. Get a minibatch data for testing\n        x, y_gt = minibatch[0], minibatch[1]\n        x = x.to(device)        # of size (batchsize, 3, H, W)\n        y_gt = y_gt.to(device)  # of size (batchsize, 1)\n\n        # 2. Compute the forward pass\n        y_pr = model(x)         # of size (batchsize, n_classes)\n        y_pr = F.softmax(y_pr, dim=1)\n\n        # 3. Get y_gt and y_pr to compute the performance metric of the test set\n        y_gt = y_gt.detach().cpu().numpy()\n        y_pr = y_pr.detach().cpu().numpy()\n        y_gt_all = np.concatenate((y_gt_all, y_gt), axis=0) if len(y_gt_all) > 0 else y_gt\n        y_pr_all = np.concatenate((y_pr_all, y_pr), axis=0) if len(y_pr_all) > 0 else y_pr\n","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:08:31.033271Z","iopub.execute_input":"2023-11-30T03:08:31.033974Z","iopub.status.idle":"2023-11-30T03:08:44.500734Z","shell.execute_reply.started":"2023-11-30T03:08:31.033924Z","shell.execute_reply":"2023-11-30T03:08:44.49899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_mat = confusion_matrix(y_gt_all, np.argmax(y_pr_all, axis=1))\nconf_mat","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:08:47.390831Z","iopub.execute_input":"2023-11-30T03:08:47.391313Z","iopub.status.idle":"2023-11-30T03:08:47.40786Z","shell.execute_reply.started":"2023-11-30T03:08:47.391277Z","shell.execute_reply":"2023-11-30T03:08:47.406199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nlabel_mapping = {0:\"HGSC\",\n      1:\"LGSC\",\n      2:\"EC\",\n      3:\"CC\",\n      4:\"MC\"}\ntick_labels = [label_mapping[i] for i in range(5)]\nsns.heatmap(conf_mat, annot=True, vmax=24, xticklabels=tick_labels, yticklabels=tick_labels, cmap= sns.color_palette(\"YlOrBr_r\", as_cmap=True))\nplt.xlabel(\"Predicted Label\")\nplt.ylabel(\"True Label\")\nplt.title(\"Confusion Matrix\")\n#plt.savefig('conf_mat.png')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:43:24.610906Z","iopub.execute_input":"2023-11-30T03:43:24.611326Z","iopub.status.idle":"2023-11-30T03:43:25.016661Z","shell.execute_reply.started":"2023-11-30T03:43:24.611294Z","shell.execute_reply":"2023-11-30T03:43:25.015386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import fbeta_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import confusion_matrix\n\ndef compute_metrics(y_pr, y_gt, label_list):\n    \"\"\"\n    Compute performance metrics of y_pr and y_gt\n    Args:\n        y_pr: 2D array of size (batchsize, n_classes)\n        y_gt: 1D array of size (batchsize,)\n        label_list: list of labels of the classification problem\n    Returns: dictionary of metrics:\n    \"\"\"\n \n    if len(label_list) == 2:\n        # Get the prob. of label-1 class\n        y_pr = y_pr[:, 1]\n        auc = roc_auc_score(y_true=y_gt, y_score=y_pr)\n\n        # Get the output labels of the y_pr\n        threshold = 0.5\n        y_pr[y_pr >= threshold] = 1.0\n        y_pr[y_pr < threshold] = 0.0\n        accuracy = accuracy_score(y_true=y_gt, y_pred=y_pr)\n        precision = precision_score(y_true=y_gt, y_pred=y_pr, pos_label=1, \n                                    average='binary', zero_division=1)\n        recall = recall_score(y_true=y_gt, y_pred=y_pr, pos_label=1, \n                              average='binary')\n        f1_score = fbeta_score(y_true=y_gt, y_pred=y_pr, beta=1, pos_label=1, \n                               average='binary')\n        f2_score = fbeta_score(y_true=y_gt, y_pred=y_pr, beta=2, pos_label=1, \n                               average='binary')\n\n    else:\n        # Compute the one-hot coding of the y-gt\n        try: \n            y_onehot = np.zeros(y_pr.shape)\n            for k in range(len(y_gt)):\n                y_onehot[k, y_gt[k]] = 1\n            auc = roc_auc_score(y_true=y_onehot, y_score=y_pr)\n        \n        except Exception: # error when not all classes presented in y_gt\n            auc = 0\n\n        # Get the output labels of the y_pr\n        y_pr = np.argmax(y_pr, axis=1)\n        accuracy = accuracy_score(y_true=y_gt, y_pred=y_pr)\n        precision = precision_score(y_true=y_gt, y_pred=y_pr, labels=label_list, \n                                    average='macro', zero_division=1)\n        recall = recall_score(y_true=y_gt, y_pred=y_pr, pos_label=1, \n                              labels=label_list, average='macro')\n        f1_score = fbeta_score(y_true=y_gt, y_pred=y_pr, beta=1, \n                               labels=label_list, average='macro')\n        f2_score = fbeta_score(y_true=y_gt, y_pred=y_pr, beta=1, \n                               labels=label_list, average='macro')\n\n    return {'accuracy': accuracy, 'precision': precision, 'recall': recall,\n            'f1_score': f1_score, 'f2_score': f2_score, 'auc': auc}","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:43:52.072747Z","iopub.execute_input":"2023-11-30T03:43:52.073195Z","iopub.status.idle":"2023-11-30T03:43:52.090272Z","shell.execute_reply.started":"2023-11-30T03:43:52.073161Z","shell.execute_reply":"2023-11-30T03:43:52.088932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_id_list = list(range(5))\nperformance = compute_metrics(y_pr_all, y_gt_all, class_id_list)\nprint(f\"Testing performance: \")\nfor k, v in performance.items():\n    print(f'\\t{k}: \\t{v:.4f}')","metadata":{"execution":{"iopub.status.busy":"2023-11-30T03:44:53.826203Z","iopub.execute_input":"2023-11-30T03:44:53.826792Z","iopub.status.idle":"2023-11-30T03:44:53.851641Z","shell.execute_reply.started":"2023-11-30T03:44:53.826739Z","shell.execute_reply":"2023-11-30T03:44:53.849964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l=[[v] for k,v in performance.items()]\n\ndf = pd.DataFrame(l)\ndf.index = performance.keys()","metadata":{"execution":{"iopub.status.busy":"2023-11-30T04:03:23.89159Z","iopub.execute_input":"2023-11-30T04:03:23.892104Z","iopub.status.idle":"2023-11-30T04:03:23.909482Z","shell.execute_reply.started":"2023-11-30T04:03:23.892066Z","shell.execute_reply":"2023-11-30T04:03:23.907498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l","metadata":{},"execution_count":null,"outputs":[]}]}