{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6874344,"sourceType":"datasetVersion","datasetId":3950227},{"sourceId":149293348,"sourceType":"kernelVersion"},{"sourceId":3729,"sourceType":"modelInstanceVersion","modelInstanceId":2656}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\n

🔬UBC: Fine-tuning with Loss for Class Imbalance🔬

\n
\n","metadata":{}},{"cell_type":"markdown","source":"# In this notebook...\n- Implement \"CustomLoss\" class\n - Use \"Focal Loss\" or \"Weighted Cross Entropy Loss\" for imbalanced classes\n- Calculate mean and std for train dataset and use them for \"Normalize\"","metadata":{}},{"cell_type":"markdown","source":"# Great notebooks! Thank you🌟\n- https://www.kaggle.com/code/gunesevitan/ubc-ocean-eda\n- https://www.kaggle.com/code/motono0223/ubc-finetune-cnn-without-thumbnails","metadata":{}},{"cell_type":"markdown","source":"
\n

📝Import Libraries📝

\n
\n","metadata":{"execution":{"iopub.status.busy":"2023-12-01T06:59:44.016241Z","iopub.execute_input":"2023-12-01T06:59:44.016532Z","iopub.status.idle":"2023-12-01T06:59:44.021397Z","shell.execute_reply.started":"2023-12-01T06:59:44.016506Z","shell.execute_reply":"2023-12-01T06:59:44.020385Z"}}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torchvision\n\nimport torchmetrics\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict, Counter\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, StratifiedGroupKFold\n\n# For Image Models\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:43.250033Z","iopub.execute_input":"2023-12-02T13:55:43.250731Z","iopub.status.idle":"2023-12-02T13:55:52.202462Z","shell.execute_reply.started":"2023-12-02T13:55:43.250696Z","shell.execute_reply":"2023-12-02T13:55:52.201129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Training Config🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n \"seed\": 42,\n \"epochs\": 5,\n \"img_size\": 2048,\n \"model_name\": \"tf_efficientnet_b0_ns\",\n \"checkpoint_path\" : \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n \"pretrained\" : \"/kaggle/input/ubc-efficienetnetb0-fold1of10-2048pix-thumbnails/Recall0.9178_Acc0.9437_Loss0.1685_epoch9.bin\",\n \"num_classes\": 5,\n \"train_batch_size\": 2,\n \"valid_batch_size\": 4,\n \"learning_rate\": 2e-5, # <-4e-5\n \"scheduler\": 'CosineAnnealingLR',\n \"min_lr\": 2e-6,\n \"T_max\": 500,\n \"weight_decay\": 1e-6,\n # Add loss setting\n \"loss_type\": 'focal',\n \"loss_params\": dict(alpha=1, gamma=2),\n \"fold\" : 0,\n \"n_fold\": 10,\n \"n_accumulate\": 1,\n \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.205315Z","iopub.execute_input":"2023-12-02T13:55:52.206015Z","iopub.status.idle":"2023-12-02T13:55:52.239574Z","shell.execute_reply.started":"2023-12-02T13:55:52.205973Z","shell.execute_reply":"2023-12-02T13:55:52.238603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Set Seeds and Directory Path🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.241206Z","iopub.execute_input":"2023-12-02T13:55:52.242203Z","iopub.status.idle":"2023-12-02T13:55:52.27169Z","shell.execute_reply.started":"2023-12-02T13:55:52.242163Z","shell.execute_reply":"2023-12-02T13:55:52.268592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/ubc-crop-training-raw-images'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.274472Z","iopub.execute_input":"2023-12-02T13:55:52.27527Z","iopub.status.idle":"2023-12-02T13:55:52.279867Z","shell.execute_reply.started":"2023-12-02T13:55:52.275216Z","shell.execute_reply":"2023-12-02T13:55:52.279023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

📒Read Train Paths and Set Encoder📒

\n
\n","metadata":{}},{"cell_type":"code","source":"df_ori = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\ndf_ori","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.281033Z","iopub.execute_input":"2023-12-02T13:55:52.281401Z","iopub.status.idle":"2023-12-02T13:55:52.315001Z","shell.execute_reply.started":"2023-12-02T13:55:52.281369Z","shell.execute_reply":"2023-12-02T13:55:52.314115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf[\"file_path\"] = df[\"file_path\"].apply(lambda x: f\"{ROOT_DIR}/{x}\" )\ndf","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.316278Z","iopub.execute_input":"2023-12-02T13:55:52.316653Z","iopub.status.idle":"2023-12-02T13:55:52.354553Z","shell.execute_reply.started":"2023-12-02T13:55:52.316621Z","shell.execute_reply":"2023-12-02T13:55:52.353679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"label\"].unique()","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.355713Z","iopub.execute_input":"2023-12-02T13:55:52.35601Z","iopub.status.idle":"2023-12-02T13:55:52.365Z","shell.execute_reply.started":"2023-12-02T13:55:52.355985Z","shell.execute_reply":"2023-12-02T13:55:52.364011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ndf['label_str'] = df['label']\ndf_ori[\"label\"] = encoder.fit_transform(df_ori['label'])\ndf['label'] = encoder.transform(df['label_str'])\n\nwith open(\"label_encoder.pkl\", \"wb\") as fp:\n joblib.dump(encoder, fp)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.36622Z","iopub.execute_input":"2023-12-02T13:55:52.366725Z","iopub.status.idle":"2023-12-02T13:55:52.376142Z","shell.execute_reply.started":"2023-12-02T13:55:52.366691Z","shell.execute_reply":"2023-12-02T13:55:52.375023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CLASS_WEIGHTS = Counter(df[\"label\"].values)\nCLASS_WEIGHTS = [ df.shape[0] / CLASS_WEIGHTS[i] for i in sorted(df[\"label\"].unique()) ]\nCLASS_WEIGHTS = [ val / sum(CLASS_WEIGHTS) for val in CLASS_WEIGHTS ]\nCLASS_WEIGHTS","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.377444Z","iopub.execute_input":"2023-12-02T13:55:52.377744Z","iopub.status.idle":"2023-12-02T13:55:52.388486Z","shell.execute_reply.started":"2023-12-02T13:55:52.377719Z","shell.execute_reply":"2023-12-02T13:55:52.387674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Create Fold🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"skf = StratifiedGroupKFold(n_splits=CONFIG['n_fold'], shuffle=True, random_state=CONFIG[\"seed\"])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label, groups=df.image_id)):\n df.loc[val_ , \"kfold\"] = int(fold)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:52.392138Z","iopub.execute_input":"2023-12-02T13:55:52.392442Z","iopub.status.idle":"2023-12-02T13:55:53.005876Z","shell.execute_reply.started":"2023-12-02T13:55:52.392412Z","shell.execute_reply":"2023-12-02T13:55:53.005027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['T_max'] = df[df[\"kfold\"]!=CONFIG[\"fold\"]].shape[0] * CONFIG['epochs'] // CONFIG['train_batch_size']\nCONFIG['T_max']","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.007018Z","iopub.execute_input":"2023-12-02T13:55:53.007342Z","iopub.status.idle":"2023-12-02T13:55:53.018048Z","shell.execute_reply.started":"2023-12-02T13:55:53.007315Z","shell.execute_reply":"2023-12-02T13:55:53.017158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

📁Dataset Class📁

\n
\n","metadata":{}},{"cell_type":"code","source":"class UBCDataset(Dataset):\n def __init__(self, df, transforms=None):\n self.df = df\n self.file_names = df['file_path'].values\n self.labels = df['label'].values\n self.transforms = transforms\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n img_path = self.file_names[index]\n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n label = self.labels[index]\n \n if self.transforms:\n img = self.transforms(image=img)[\"image\"]\n \n return {\n 'image': img,\n 'label': torch.tensor(label, dtype=torch.long)\n }","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.019453Z","iopub.execute_input":"2023-12-02T13:55:53.019774Z","iopub.status.idle":"2023-12-02T13:55:53.027592Z","shell.execute_reply.started":"2023-12-02T13:55:53.019743Z","shell.execute_reply":"2023-12-02T13:55:53.026695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🎨Augmentations🎨

\n
\n","metadata":{}},{"cell_type":"code","source":"# ✅Calc mean and std for Normalize\n# If you don't use mean and std of this dataset for normalize, you don't need to execute this cell.\n\nif False:\n df_train = df[df.kfold != fold].reset_index(drop=True)\n train_dataset = UBCDataset(df_train,\n transforms=A.Compose([\n A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n ToTensorV2()], p=1.)\n )\n train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n num_workers=2, pin_memory=True, drop_last=True)\n mean = 0.0\n std = 0.0\n total_samples = 0\n\n for batch_label in tqdm(train_loader):\n batch = batch_label['image']\n batch_size = batch.size(0)\n data = batch.view(batch_size, batch.size(1), -1).float()\n mean += data.mean(2).sum(0)\n std += data.std(2).sum(0)\n total_samples += batch_size\n\n mean /= total_samples\n std /= total_samples\n\n print(\"Calculated Mean:\", mean/255)\n print(\"Calculated Std:\", std/255)\n\n# Result as follows:\n# Calculated Mean: tensor([0.4682, 0.4063, 0.4651])\n# Calculated Std: tensor([0.3842, 0.3438, 0.3825])","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.028849Z","iopub.execute_input":"2023-12-02T13:55:53.029126Z","iopub.status.idle":"2023-12-02T13:55:53.041199Z","shell.execute_reply.started":"2023-12-02T13:55:53.029103Z","shell.execute_reply":"2023-12-02T13:55:53.040429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n \"train\": A.Compose([\n A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n A.Flip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.RandomRotate90(p=1.0),\n A.ShiftScaleRotate(shift_limit=0.1, \n scale_limit=0.15, \n rotate_limit=60, \n p=0.5),\n A.HueSaturationValue(\n hue_shift_limit=0.2,\n sat_shift_limit=0.2,\n val_shift_limit=0.2,\n p=0.5\n ),\n A.RandomBrightnessContrast(\n brightness_limit=(-0.1, 0.1), \n contrast_limit=(-0.1, 0.1), \n p=0.5\n ),\n A.Normalize(\n mean=[0.468, 0.406, 0.465], \n std=[0.384, 0.344, 0.383],\n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.),\n \n \"valid\": A.Compose([\n A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n A.Normalize(\n mean=[0.468, 0.406, 0.465], \n std=[0.384, 0.344, 0.383],\n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.042287Z","iopub.execute_input":"2023-12-02T13:55:53.042569Z","iopub.status.idle":"2023-12-02T13:55:53.053206Z","shell.execute_reply.started":"2023-12-02T13:55:53.042513Z","shell.execute_reply":"2023-12-02T13:55:53.052333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩GeM Pooling🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"class GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM, self).__init__()\n self.p = nn.Parameter(torch.ones(1)*p)\n self.eps = eps\n\n def forward(self, x):\n return self.gem(x, p=self.p, eps=self.eps)\n \n def gem(self, x, p=3, eps=1e-6):\n return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n \n def __repr__(self):\n return self.__class__.__name__ + \\\n '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.054258Z","iopub.execute_input":"2023-12-02T13:55:53.054515Z","iopub.status.idle":"2023-12-02T13:55:53.066203Z","shell.execute_reply.started":"2023-12-02T13:55:53.054493Z","shell.execute_reply":"2023-12-02T13:55:53.065382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Model Class🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"class UBCModel(nn.Module):\n def __init__(self, model_name, num_classes, pretrained=True, checkpoint_path=None):\n super(UBCModel, self).__init__()\n self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n\n in_features = self.model.classifier.in_features\n self.model.classifier = nn.Identity()\n self.model.global_pool = nn.Identity()\n self.pooling = GeM()\n self.linear = nn.Linear(in_features, num_classes)\n self.softmax = nn.Softmax(dim=1)\n\n def forward(self, images):\n features = self.model(images)\n pooled_features = self.pooling(features).flatten(1)\n output = self.linear(pooled_features)\n return output\n\n \nmodel = UBCModel(CONFIG['model_name'], CONFIG['num_classes'], checkpoint_path=CONFIG['checkpoint_path'])\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:53.067441Z","iopub.execute_input":"2023-12-02T13:55:53.067935Z","iopub.status.idle":"2023-12-02T13:55:56.861226Z","shell.execute_reply.started":"2023-12-02T13:55:53.067902Z","shell.execute_reply":"2023-12-02T13:55:56.860301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if \"pretrained\" in CONFIG:\n model.load_state_dict( torch.load(CONFIG[\"pretrained\"]))","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:56.862612Z","iopub.execute_input":"2023-12-02T13:55:56.862976Z","iopub.status.idle":"2023-12-02T13:55:57.110215Z","shell.execute_reply.started":"2023-12-02T13:55:56.862942Z","shell.execute_reply":"2023-12-02T13:55:57.109447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Loss Functions🔩

\n
\n","metadata":{}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n def __init__(self, alpha=1, gamma=2, logits=True, reduce=True):\n super(FocalLoss, self).__init__()\n self.alpha = alpha\n self.gamma = gamma\n self.logits = logits\n self.reduce = reduce\n\n def forward(self, inputs, targets):\n ce_loss = nn.functional.cross_entropy(inputs, targets, reduction='none' if self.reduce else 'mean')\n pt = torch.exp(-ce_loss)\n focal_loss = self.alpha * (1-pt)**self.gamma * ce_loss\n if self.reduce:\n return torch.mean(focal_loss)\n else:\n return focal_loss\n \nclass WeightedCrossEntropyLoss(nn.Module):\n def __init__(self, weights):\n super(WeightedCrossEntropyLoss, self).__init__()\n self.weights = weights\n\n def forward(self, inputs, targets):\n ce_loss = nn.CrossEntropyLoss(reduction='none')(inputs, targets)\n weighted_ce_loss = ce_loss * self.weights[targets]\n return torch.mean(weighted_ce_loss)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.111493Z","iopub.execute_input":"2023-12-02T13:55:57.111779Z","iopub.status.idle":"2023-12-02T13:55:57.121183Z","shell.execute_reply.started":"2023-12-02T13:55:57.111754Z","shell.execute_reply":"2023-12-02T13:55:57.120078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomLoss(nn.Module):\n def __init__(self, loss_type='cross_entropy', weight=None, **kwargs):\n super(CustomLoss, self).__init__()\n\n self.loss_type = loss_type\n self.weight = weight\n\n if self.loss_type == 'cross_entropy':\n self.criterion = nn.CrossEntropyLoss(weight=weight, **kwargs)\n elif self.loss_type == 'weighted_cross_entropy':\n self.criterion = WeightedCrossEntropyLoss(weight, **kwargs) \n elif self.loss_type == 'focal':\n self.criterion = FocalLoss(**kwargs)\n else:\n raise ValueError(f\"Unsupported loss type: {self.loss_type}\")\n\n def forward(self, outputs, labels):\n return self.criterion(outputs, labels)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.122451Z","iopub.execute_input":"2023-12-02T13:55:57.12272Z","iopub.status.idle":"2023-12-02T13:55:57.132048Z","shell.execute_reply.started":"2023-12-02T13:55:57.122696Z","shell.execute_reply":"2023-12-02T13:55:57.131294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(outputs, labels):\n return CustomLoss(loss_type=CONFIG[\"loss_type\"],\n weight=torch.tensor(CLASS_WEIGHTS).cuda(),\n **CONFIG[\"loss_params\"])(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.133051Z","iopub.execute_input":"2023-12-02T13:55:57.133353Z","iopub.status.idle":"2023-12-02T13:55:57.145492Z","shell.execute_reply.started":"2023-12-02T13:55:57.133329Z","shell.execute_reply":"2023-12-02T13:55:57.144506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Training Function🔩

\n
","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n model.train()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n running_recall = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar:\n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n \n outputs = model(images)\n loss = criterion(outputs, labels) \n loss = loss / CONFIG['n_accumulate']\n \n loss.backward()\n \n if (step + 1) % CONFIG['n_accumulate'] == 0:\n optimizer.step()\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n if scheduler is not None:\n scheduler.step()\n \n _, predicted = torch.max(model.softmax(outputs), 1)\n acc = torch.sum( predicted == labels )\n recall_nn = torchmetrics.Recall(task=\"multiclass\", average='macro', num_classes=CONFIG[\"num_classes\"]).cuda()\n recall = recall_nn(predicted, labels)\n \n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n running_recall += (recall.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n epoch_recall = running_recall / dataset_size\n \n bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss, Train_Acc=epoch_acc, Train_Recall=epoch_recall,\n LR=optimizer.param_groups[0]['lr'])\n gc.collect()\n \n return epoch_loss, epoch_acc, epoch_recall","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.146755Z","iopub.execute_input":"2023-12-02T13:55:57.147081Z","iopub.status.idle":"2023-12-02T13:55:57.159068Z","shell.execute_reply.started":"2023-12-02T13:55:57.147055Z","shell.execute_reply":"2023-12-02T13:55:57.158021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Validation Function🔩

\n
","metadata":{}},{"cell_type":"code","source":"@torch.inference_mode()\ndef valid_one_epoch(model, dataloader, device, epoch):\n model.eval()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n running_recall = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar: \n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n\n outputs = model(images)\n loss = criterion(outputs, labels)\n\n _, predicted = torch.max(model.softmax(outputs), 1)\n acc = torch.sum( predicted == labels )\n \n recall_nn = torchmetrics.Recall(task=\"multiclass\", average='macro', num_classes=CONFIG[\"num_classes\"]).cuda()\n recall = recall_nn(predicted, labels)\n\n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n running_recall += (recall.item() * batch_size)\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n epoch_recall = running_recall / dataset_size\n \n bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss, Valid_Acc=epoch_acc, Valid_Recall=epoch_recall,\n LR=optimizer.param_groups[0]['lr']) \n \n gc.collect()\n \n return epoch_loss, epoch_acc, epoch_recall","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.160207Z","iopub.execute_input":"2023-12-02T13:55:57.160543Z","iopub.status.idle":"2023-12-02T13:55:57.172975Z","shell.execute_reply.started":"2023-12-02T13:55:57.160518Z","shell.execute_reply":"2023-12-02T13:55:57.172117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩Run Training🔩

\n
","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n if torch.cuda.is_available():\n print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n \n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_epoch_recall = -np.inf\n history = defaultdict(list)\n \n for epoch in range(1, num_epochs + 1): \n gc.collect()\n train_epoch_loss, train_epoch_acc, train_epoch_recall = train_one_epoch(model, optimizer, scheduler, \n dataloader=train_loader, \n device=CONFIG['device'], epoch=epoch)\n \n val_epoch_loss, val_epoch_acc, val_epoch_recall = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n epoch=epoch)\n \n history['Train Loss'].append(train_epoch_loss)\n history['Valid Loss'].append(val_epoch_loss)\n history['Train Accuracy'].append(train_epoch_acc)\n history['Valid Accuracy'].append(val_epoch_acc)\n history['Train Recall'].append(train_epoch_recall)\n history['Valid Recall'].append(val_epoch_recall)\n history['lr'].append( scheduler.get_lr()[0] )\n \n # deep copy the model\n if best_epoch_recall <= val_epoch_recall:\n print(f\"{b_}Validation Recall Improved ({best_epoch_recall} ---> {val_epoch_recall})\")\n best_epoch_recall = val_epoch_recall\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"Recall{:.4f}_Acc{:.4f}_Loss{:.4f}_epoch{:.0f}.bin\".format(val_epoch_recall, val_epoch_acc, val_epoch_loss, epoch)\n torch.save(model.state_dict(), PATH)\n # Save a model file from the current directory\n print(f\"Model Saved{sr_}\")\n \n print()\n \n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best Recall: {:.4f}\".format(best_epoch_recall))\n \n # load best model weights\n model.load_state_dict(best_model_wts)\n \n return model, history","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.174399Z","iopub.execute_input":"2023-12-02T13:55:57.174707Z","iopub.status.idle":"2023-12-02T13:55:57.18754Z","shell.execute_reply.started":"2023-12-02T13:55:57.174683Z","shell.execute_reply":"2023-12-02T13:55:57.186704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CONFIG['scheduler'] == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == None:\n return None\n \n return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.188669Z","iopub.execute_input":"2023-12-02T13:55:57.189469Z","iopub.status.idle":"2023-12-02T13:55:57.200561Z","shell.execute_reply.started":"2023-12-02T13:55:57.189442Z","shell.execute_reply":"2023-12-02T13:55:57.199699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n df_train = df[df.kfold != fold].reset_index(drop=True)\n df_valid = df[df.kfold == fold].reset_index(drop=True)\n \n train_dataset = UBCDataset(df_train, transforms=data_transforms[\"train\"])\n valid_dataset = UBCDataset(df_valid, transforms=data_transforms[\"valid\"])\n\n train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n num_workers=2, shuffle=False, pin_memory=True)\n \n return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.201713Z","iopub.execute_input":"2023-12-02T13:55:57.202335Z","iopub.status.idle":"2023-12-02T13:55:57.213112Z","shell.execute_reply.started":"2023-12-02T13:55:57.202299Z","shell.execute_reply":"2023-12-02T13:55:57.212283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare Dataloaders","metadata":{}},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=CONFIG[\"fold\"])","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.214603Z","iopub.execute_input":"2023-12-02T13:55:57.214934Z","iopub.status.idle":"2023-12-02T13:55:57.226255Z","shell.execute_reply.started":"2023-12-02T13:55:57.214903Z","shell.execute_reply":"2023-12-02T13:55:57.225288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ✅If you want to check data augmentation, please use this cell.\n\ndef show_batch(batch):\n # batch is (num_batch, 3, width, height)\n num_batch = batch.shape[0]\n \n for i in range(num_batch):\n img = np.transpose(batch[i, :, :, :], (1, 2, 0))\n plt.imshow(img)\n plt.show()\n\ndataiter = iter(train_loader)\nshow_batch(next(dataiter)['image'].numpy())","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:55:57.22749Z","iopub.execute_input":"2023-12-02T13:55:57.227815Z","iopub.status.idle":"2023-12-02T13:56:00.653688Z","shell.execute_reply.started":"2023-12-02T13:55:57.227785Z","shell.execute_reply":"2023-12-02T13:56:00.652627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define Optimizer and Scheduler","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:56:00.661264Z","iopub.execute_input":"2023-12-02T13:56:00.661739Z","iopub.status.idle":"2023-12-02T13:56:00.671106Z","shell.execute_reply.started":"2023-12-02T13:56:00.66169Z","shell.execute_reply":"2023-12-02T13:56:00.669805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Start Training","metadata":{}},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n device=CONFIG['device'],\n num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2023-12-02T13:56:00.672172Z","iopub.execute_input":"2023-12-02T13:56:00.672457Z","iopub.status.idle":"2023-12-02T14:15:41.985041Z","shell.execute_reply.started":"2023-12-02T13:56:00.672433Z","shell.execute_reply":"2023-12-02T14:15:41.983977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = pd.DataFrame.from_dict(history)\nhistory.to_csv(\"history.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:15:41.986925Z","iopub.execute_input":"2023-12-02T14:15:41.987802Z","iopub.status.idle":"2023-12-02T14:15:41.997736Z","shell.execute_reply.started":"2023-12-02T14:15:41.987759Z","shell.execute_reply":"2023-12-02T14:15:41.996775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

📈Visualize Training Process📈

\n
","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3, figsize=(20, 5))\n\n# Loss\nax = axes[0]\nax.plot(range(history.shape[0]), history[\"Train Loss\"].values, label=\"Train Loss\")\nax.plot(range(history.shape[0]), history[\"Valid Loss\"].values, label=\"Valid Loss\")\nax.set_xlabel(\"epochs\")\nax.set_ylabel(\"Loss\")\nax.set_title(\"Loss\")\nax.grid(True)\nax.legend()\n\n# Recall\nax = axes[1]\nax.plot( range(history.shape[0]), history[\"Train Recall\"].values, label=\"Train Recall\")\nax.plot( range(history.shape[0]), history[\"Valid Recall\"].values, label=\"Valid Recall\")\nax.set_xlabel(\"epochs\")\nax.set_ylabel(\"Recall\")\nax.set_title(\"Recall\")\nax.grid(True)\nax.legend()\n\n# Accuracy\nax = axes[2]\nax.plot( range(history.shape[0]), history[\"Train Accuracy\"].values, label=\"Train Accuracy\")\nax.plot( range(history.shape[0]), history[\"Valid Accuracy\"].values, label=\"Valid Accuracy\")\nax.set_xlabel(\"epochs\")\nax.set_ylabel(\"Accuracy\")\nax.set_title(\"Accuracy\")\nplt.grid(True)\nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:17:30.544344Z","iopub.execute_input":"2023-12-02T14:17:30.544647Z","iopub.status.idle":"2023-12-02T14:17:31.388259Z","shell.execute_reply.started":"2023-12-02T14:17:30.544622Z","shell.execute_reply":"2023-12-02T14:17:31.387299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"lr\"].values, label=\"lr\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"lr\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:15:42.719214Z","iopub.execute_input":"2023-12-02T14:15:42.719628Z","iopub.status.idle":"2023-12-02T14:15:43.021981Z","shell.execute_reply.started":"2023-12-02T14:15:42.719593Z","shell.execute_reply":"2023-12-02T14:15:43.020967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

🔩OOF🔩

\n
","metadata":{}},{"cell_type":"code","source":"oof_dataset = UBCDataset(df, transforms=data_transforms[\"valid\"])\noof_loader = DataLoader(oof_dataset, batch_size=CONFIG['valid_batch_size'], num_workers=2, shuffle=False, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:15:43.023484Z","iopub.execute_input":"2023-12-02T14:15:43.024145Z","iopub.status.idle":"2023-12-02T14:15:43.029681Z","shell.execute_reply.started":"2023-12-02T14:15:43.024105Z","shell.execute_reply":"2023-12-02T14:15:43.028665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_preds_conf = []\ntotal_preds_label = []\nwith torch.no_grad():\n bar = tqdm(enumerate(oof_loader), total=len(oof_loader))\n for step, data in bar: \n images = data['image'].to(CONFIG[\"device\"], dtype=torch.float) \n batch_size = images.size(0)\n outputs = model(images)\n preds_conf = model.softmax(outputs)\n _, preds_label = torch.max(preds_conf, 1)\n total_preds_conf.append( preds_conf.detach().cpu().numpy() )\n total_preds_label.append( preds_label.detach().cpu().numpy() )\ntotal_preds_conf = np.concatenate(total_preds_conf, axis=0)\ntotal_preds_label = np.concatenate(total_preds_label).flatten()\ntotal_preds_label2 = encoder.inverse_transform( total_preds_label )","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:15:43.031402Z","iopub.execute_input":"2023-12-02T14:15:43.03199Z","iopub.status.idle":"2023-12-02T14:17:30.474213Z","shell.execute_reply.started":"2023-12-02T14:15:43.031952Z","shell.execute_reply":"2023-12-02T14:17:30.473075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(total_preds_conf.shape[-1]):\n df[f\"pred_conf_{i}\"] = total_preds_conf[:, i]\ndf[\"pred_label\"] = total_preds_label\ndf[\"pred_label_str\"] = total_preds_label2\ndf.to_csv(\"oof.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:17:30.476115Z","iopub.execute_input":"2023-12-02T14:17:30.477099Z","iopub.status.idle":"2023-12-02T14:17:30.50547Z","shell.execute_reply.started":"2023-12-02T14:17:30.477056Z","shell.execute_reply":"2023-12-02T14:17:30.504456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-12-02T14:17:30.506794Z","iopub.execute_input":"2023-12-02T14:17:30.507122Z","iopub.status.idle":"2023-12-02T14:17:30.540528Z","shell.execute_reply.started":"2023-12-02T14:17:30.507094Z","shell.execute_reply":"2023-12-02T14:17:30.539509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}