{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#

Import Required Libraries 📚

","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torchvision\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\n# For Image Models\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom PIL import Image\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:32.223495Z","iopub.execute_input":"2023-10-31T17:12:32.223838Z","iopub.status.idle":"2023-10-31T17:12:39.175321Z","shell.execute_reply.started":"2023-10-31T17:12:32.223807Z","shell.execute_reply":"2023-10-31T17:12:39.174486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Training Configuration ⚙️

","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n \"seed\": 42,\n \"epochs\": 30,\n \"img_size\": 512,\n \"model_name\": \"tf_efficientnet_b0_ns\",\n \"checkpoint_path\" : \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n \"num_classes\": 5,\n \"train_batch_size\": 10,\n \"valid_batch_size\": 16,\n \"learning_rate\": 1e-4,\n \"scheduler\": 'CosineAnnealingLR',\n \"min_lr\": 1e-6,\n \"T_max\": 500,\n \"weight_decay\": 1e-6,\n \"fold\" : 0,\n \"n_fold\": 5,\n \"n_accumulate\": 1,\n \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.177283Z","iopub.execute_input":"2023-10-31T17:12:39.177922Z","iopub.status.idle":"2023-10-31T17:12:39.209312Z","shell.execute_reply.started":"2023-10-31T17:12:39.177883Z","shell.execute_reply":"2023-10-31T17:12:39.208229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Set Seed for Reproducibility

","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.210902Z","iopub.execute_input":"2023-10-31T17:12:39.211571Z","iopub.status.idle":"2023-10-31T17:12:39.23843Z","shell.execute_reply.started":"2023-10-31T17:12:39.211537Z","shell.execute_reply":"2023-10-31T17:12:39.237504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_thumbnails'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.240543Z","iopub.execute_input":"2023-10-31T17:12:39.240857Z","iopub.status.idle":"2023-10-31T17:12:39.248595Z","shell.execute_reply.started":"2023-10-31T17:12:39.240832Z","shell.execute_reply":"2023-10-31T17:12:39.247777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(image_id):\n return f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"\n# return f\"{TRAIN_DIR}/{image_id}.png\"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.249554Z","iopub.execute_input":"2023-10-31T17:12:39.249806Z","iopub.status.idle":"2023-10-31T17:12:39.259322Z","shell.execute_reply.started":"2023-10-31T17:12:39.249785Z","shell.execute_reply":"2023-10-31T17:12:39.258502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Read the Data 📖

","metadata":{}},{"cell_type":"code","source":"train_images = sorted(glob.glob(f\"{TRAIN_DIR}/*.png\"))","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.26031Z","iopub.execute_input":"2023-10-31T17:12:39.260605Z","iopub.status.idle":"2023-10-31T17:12:39.324544Z","shell.execute_reply.started":"2023-10-31T17:12:39.260563Z","shell.execute_reply":"2023-10-31T17:12:39.323551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf['file_path'] = df['image_id'].apply(get_train_file_path)\ndf = df[ df[\"file_path\"].isin(train_images) ].reset_index(drop=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.325952Z","iopub.execute_input":"2023-10-31T17:12:39.326756Z","iopub.status.idle":"2023-10-31T17:12:39.372482Z","shell.execute_reply.started":"2023-10-31T17:12:39.32672Z","shell.execute_reply":"2023-10-31T17:12:39.371579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ndf['n_label'] = encoder.fit_transform(df['label'])\n\nwith open(\"label_encoder.pkl\", \"wb\") as fp:\n joblib.dump(encoder, fp)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.373878Z","iopub.execute_input":"2023-10-31T17:12:39.37419Z","iopub.status.idle":"2023-10-31T17:12:39.380748Z","shell.execute_reply.started":"2023-10-31T17:12:39.374164Z","shell.execute_reply":"2023-10-31T17:12:39.379838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['T_max'] = df.shape[0] * (CONFIG[\"n_fold\"]-1) * CONFIG['epochs'] // CONFIG['train_batch_size'] // CONFIG[\"n_fold\"]\nCONFIG['T_max']","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.382243Z","iopub.execute_input":"2023-10-31T17:12:39.382594Z","iopub.status.idle":"2023-10-31T17:12:39.392443Z","shell.execute_reply.started":"2023-10-31T17:12:39.382563Z","shell.execute_reply":"2023-10-31T17:12:39.391547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Create Folds

","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n df.loc[val_ , \"kfold\"] = int(fold)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.396169Z","iopub.execute_input":"2023-10-31T17:12:39.396445Z","iopub.status.idle":"2023-10-31T17:12:39.409751Z","shell.execute_reply.started":"2023-10-31T17:12:39.396419Z","shell.execute_reply":"2023-10-31T17:12:39.408732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Dataset Class

","metadata":{}},{"cell_type":"code","source":"class UBCDataset(Dataset):\n def __init__(self, df, transforms=None,image_size=512):\n self.df = df\n self.file_names = df['file_path'].values\n self.labels = df['n_label'].values\n self.transforms = transforms\n self.image_size = image_size \n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n img_path = self.file_names[index]\n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n \n if self.transforms:\n img = self.transforms(image=img)[\"image\"]\n \n return {\n 'image': img,\n 'label': torch.tensor(self.labels[index], dtype=torch.long)\n }","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.411064Z","iopub.execute_input":"2023-10-31T17:12:39.411388Z","iopub.status.idle":"2023-10-31T17:12:39.419575Z","shell.execute_reply.started":"2023-10-31T17:12:39.411362Z","shell.execute_reply":"2023-10-31T17:12:39.418728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Augmentations

","metadata":{}},{"cell_type":"code","source":"data_transforms = {\n \"train\": A.Compose([ \n A.Resize(512, 512),\n A.HorizontalFlip(p=0.5),\n A.VerticalFlip(p=0.5),\n A.RandomBrightnessContrast(p=0.75),\n A.ShiftScaleRotate(p=0.75),\n A.OneOf([A.GaussNoise(var_limit=[10, 50]),A.GaussianBlur(),A.MotionBlur(),], p=0.4),\n A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.5),\n A.CoarseDropout(max_holes=1, max_width=int(512* 0.3), max_height=int(512* 0.3),mask_fill_value=0, p=0.5),\n A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n ToTensorV2()], p=1.),\n \n \"valid\": A.Compose([\n A.Resize(512, 512),\n A.Normalize(\n mean=[0.485, 0.456, 0.406], \n std=[0.229, 0.224, 0.225], \n max_pixel_value=255.0, \n p=1.0\n ),\n ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.420697Z","iopub.execute_input":"2023-10-31T17:12:39.420983Z","iopub.status.idle":"2023-10-31T17:12:39.435098Z","shell.execute_reply.started":"2023-10-31T17:12:39.42096Z","shell.execute_reply":"2023-10-31T17:12:39.434146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=UBCDataset(df,data_transforms['train'])","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.436217Z","iopub.execute_input":"2023-10-31T17:12:39.43648Z","iopub.status.idle":"2023-10-31T17:12:39.445717Z","shell.execute_reply.started":"2023-10-31T17:12:39.436457Z","shell.execute_reply":"2023-10-31T17:12:39.444974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

GeM Pooling

\n\nCode taken from GeM Pooling Explained\n\n![](https://i.imgur.com/thTgYWG.jpg)","metadata":{}},{"cell_type":"markdown","source":"class GeM(nn.Module):\n def __init__(self, p=3, eps=1e-6):\n super(GeM, self).__init__()\n self.p = nn.Parameter(torch.ones(1)*p)\n self.eps = eps\n\n def forward(self, x):\n return self.gem(x, p=self.p, eps=self.eps)\n \n def gem(self, x, p=3, eps=1e-6):\n return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n \n def __repr__(self):\n return self.__class__.__name__ + \\\n '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n ', ' + 'eps=' + str(self.eps) + ')'\n \nclass UBCModel(nn.Module):\n def __init__(self, model_name, num_classes, pretrained=True, checkpoint_path=None):\n super(UBCModel, self).__init__()\n self.model = timm.create_model(model_name, pretrained=pretrained, checkpoint_path=checkpoint_path)\n\n in_features = self.model.classifier.in_features\n self.model.classifier = nn.Identity()\n self.model.global_pool = nn.Identity()\n self.pooling = GeM()\n self.linear = nn.Linear(in_features, num_classes)\n self.softmax = nn.Softmax(dim=1)\n\n def forward(self, images):\n features = self.model(images)\n pooled_features = self.pooling(features).flatten(1)\n output = self.linear(pooled_features)\n return output\n\n \nmodel = UBCModel(CONFIG['model_name'], CONFIG['num_classes'], checkpoint_path=CONFIG['checkpoint_path'])\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2023-10-28T04:03:26.613281Z","iopub.execute_input":"2023-10-28T04:03:26.613626Z","iopub.status.idle":"2023-10-28T04:03:26.62744Z","shell.execute_reply.started":"2023-10-28T04:03:26.613596Z","shell.execute_reply":"2023-10-28T04:03:26.626695Z"}}},{"cell_type":"markdown","source":"#

Create Model

","metadata":{}},{"cell_type":"code","source":"model=torchvision.models.efficientnet_v2_m(weights=\"IMAGENET1K_V1\")\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:39.446875Z","iopub.execute_input":"2023-10-31T17:12:39.447153Z","iopub.status.idle":"2023-10-31T17:12:41.623964Z","shell.execute_reply.started":"2023-10-31T17:12:39.447129Z","shell.execute_reply":"2023-10-31T17:12:41.623022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## model=torchvision.models.resnet50(weights=\"IMAGENET1K_V1\")\n#model.fc = nn.Sequential(nn.Linear(2048,1024),nn.Dropout(0.2), nn.Linear(1024, 5))\n#model.fc = nn.Linear(2048,5)\nmodel.classifier = nn.Sequential(nn.Dropout(0.2,inplace=True), nn.Linear(1280, 5))","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:41.625453Z","iopub.execute_input":"2023-10-31T17:12:41.626153Z","iopub.status.idle":"2023-10-31T17:12:41.633918Z","shell.execute_reply.started":"2023-10-31T17:12:41.626114Z","shell.execute_reply":"2023-10-31T17:12:41.632617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.to(CONFIG['device'])\nmodel.softmax=torch.nn.Softmax(dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:41.636511Z","iopub.execute_input":"2023-10-31T17:12:41.637257Z","iopub.status.idle":"2023-10-31T17:12:44.477187Z","shell.execute_reply.started":"2023-10-31T17:12:41.637224Z","shell.execute_reply":"2023-10-31T17:12:44.476166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Loss Function

","metadata":{}},{"cell_type":"code","source":"l1_lambda = 0.05\nl2_lambda = 0.05\n\ncel = nn.CrossEntropyLoss() # for validation\nclass_weights = [5.0 / count for count in [94,119,217,42,41]]\nclass_weights = torch.Tensor(class_weights).to('cuda')\ncel1 = nn.CrossEntropyLoss(weight=class_weights) # for Training\nl1_loss = nn.L1Loss()\nl2_loss = nn.MSELoss()\n\ndef criterion(outputs, targets):\n original_loss = cel1(outputs, targets)\n l1_regularization = 0\n l2_regularization = 0\n\n # Calculate L1 and L2 regularization for each model parameter\n for param in model.parameters():\n l1_regularization += l1_loss(param, torch.zeros_like(param))\n l2_regularization += l2_loss(param, torch.zeros_like(param))\n\n # Add L1 and L2 regularization to the original loss\n loss = original_loss + l1_lambda * l1_regularization + l2_lambda * l2_regularization\n\n return loss","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.47865Z","iopub.execute_input":"2023-10-31T17:12:44.479548Z","iopub.status.idle":"2023-10-31T17:12:44.488137Z","shell.execute_reply.started":"2023-10-31T17:12:44.479509Z","shell.execute_reply":"2023-10-31T17:12:44.486967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Training Function

","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n model.train()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar:\n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n \n outputs = model(images)\n loss = criterion(outputs, labels)\n loss = loss / CONFIG['n_accumulate']\n \n loss.backward()\n \n if (step + 1) % CONFIG['n_accumulate'] == 0:\n optimizer.step()\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n if scheduler is not None:\n scheduler.step()\n \n _, predicted = torch.max(model.softmax(outputs), 1)\n acc = torch.sum( predicted == labels )\n \n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n \n bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss, Train_Acc=epoch_acc,\n LR=optimizer.param_groups[0]['lr'])\n gc.collect()\n \n return epoch_loss, epoch_acc","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.489475Z","iopub.execute_input":"2023-10-31T17:12:44.489792Z","iopub.status.idle":"2023-10-31T17:12:44.500518Z","shell.execute_reply.started":"2023-10-31T17:12:44.489766Z","shell.execute_reply":"2023-10-31T17:12:44.499637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Validation Function

","metadata":{}},{"cell_type":"code","source":"@torch.inference_mode()\ndef valid_one_epoch(model, dataloader, device, epoch):\n model.eval()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar: \n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n\n outputs = model(images)\n loss = cel(outputs, labels)\n\n _, predicted = torch.max(model.softmax(outputs), 1)\n acc = torch.sum( predicted == labels )\n\n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n \n bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss, Valid_Acc=epoch_acc,\n LR=optimizer.param_groups[0]['lr']) \n \n gc.collect()\n \n return epoch_loss, epoch_acc","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.501857Z","iopub.execute_input":"2023-10-31T17:12:44.50274Z","iopub.status.idle":"2023-10-31T17:12:44.515308Z","shell.execute_reply.started":"2023-10-31T17:12:44.502674Z","shell.execute_reply":"2023-10-31T17:12:44.514394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Run Training

","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n if torch.cuda.is_available():\n print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n \n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_epoch_acc = -np.inf\n history = defaultdict(list)\n \n for epoch in range(1, num_epochs + 1): \n gc.collect()\n train_epoch_loss, train_epoch_acc = train_one_epoch(model, optimizer, scheduler, \n dataloader=train_loader, \n device=CONFIG['device'], epoch=epoch)\n \n val_epoch_loss, val_epoch_acc = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n epoch=epoch)\n \n history['Train Loss'].append(train_epoch_loss)\n history['Valid Loss'].append(val_epoch_loss)\n history['Train Accuracy'].append(train_epoch_acc)\n history['Valid Accuracy'].append(val_epoch_acc)\n history['lr'].append( scheduler.get_lr()[0] )\n \n # deep copy the model\n if best_epoch_acc <= val_epoch_acc:\n print(f\"{b_}Validation Accuracy Improved ({best_epoch_acc} ---> {val_epoch_acc})\")\n best_epoch_acc = val_epoch_acc\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"Acc{:.2f}_Loss{:.4f}_epoch{:.0f}.bin\".format(best_epoch_acc, val_epoch_loss, epoch)\n torch.save(model.state_dict(), PATH)\n # Save a model file from the current directory\n print(f\"Model Saved{sr_}\")\n \n print()\n \n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best Accuracy: {:.4f}\".format(best_epoch_acc))\n \n # load best model weights\n model.load_state_dict(best_model_wts)\n \n return model, history","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.516549Z","iopub.execute_input":"2023-10-31T17:12:44.516886Z","iopub.status.idle":"2023-10-31T17:12:44.531454Z","shell.execute_reply.started":"2023-10-31T17:12:44.516855Z","shell.execute_reply":"2023-10-31T17:12:44.530446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CONFIG['scheduler'] == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == None:\n return None\n \n return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.532654Z","iopub.execute_input":"2023-10-31T17:12:44.532968Z","iopub.status.idle":"2023-10-31T17:12:44.546376Z","shell.execute_reply.started":"2023-10-31T17:12:44.532943Z","shell.execute_reply":"2023-10-31T17:12:44.545235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n df_train = df[df.kfold != fold].reset_index(drop=True)\n df_valid = df[df.kfold == fold].reset_index(drop=True)\n \n train_dataset = UBCDataset(df_train, transforms=data_transforms[\"train\"])\n valid_dataset = UBCDataset(df_valid, transforms=data_transforms[\"valid\"])\n\n train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n num_workers=2, shuffle=False, pin_memory=True)\n \n return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.547718Z","iopub.execute_input":"2023-10-31T17:12:44.548948Z","iopub.status.idle":"2023-10-31T17:12:44.560333Z","shell.execute_reply.started":"2023-10-31T17:12:44.54892Z","shell.execute_reply":"2023-10-31T17:12:44.559525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare Dataloaders","metadata":{}},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=CONFIG[\"fold\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.561533Z","iopub.execute_input":"2023-10-31T17:12:44.561884Z","iopub.status.idle":"2023-10-31T17:12:44.573288Z","shell.execute_reply.started":"2023-10-31T17:12:44.561854Z","shell.execute_reply":"2023-10-31T17:12:44.572456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define Optimizer and Scheduler","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.574403Z","iopub.execute_input":"2023-10-31T17:12:44.574771Z","iopub.status.idle":"2023-10-31T17:12:44.588641Z","shell.execute_reply.started":"2023-10-31T17:12:44.574736Z","shell.execute_reply":"2023-10-31T17:12:44.587629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Start Training","metadata":{}},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n device=CONFIG['device'],\n num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2023-10-31T17:12:44.589861Z","iopub.execute_input":"2023-10-31T17:12:44.590126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Hello\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = pd.DataFrame.from_dict(history)\nhistory.to_csv(\"history.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#

Logs

","metadata":{}},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Loss\"].values, label=\"Train Loss\")\nplt.plot( range(history.shape[0]), history[\"Valid Loss\"].values, label=\"Valid Loss\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Loss\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Accuracy\"].values, label=\"Train Accuracy\")\nplt.plot( range(history.shape[0]), history[\"Valid Accuracy\"].values, label=\"Valid Accuracy\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Accuracy\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"lr\"].values, label=\"lr\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"lr\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}