{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

I made some changes from motono0223's notebook, including:\n \n- Revised the data augmentation.\n- Removed the GeM Pooling.\n- Switched to training the model from scratch for greater flexibility and ease of future adjustments.\n\n
\n

My first Kaggle code share - Your vote would mean the world to me! Thank you❤️

\n
\n","metadata":{}},{"cell_type":"markdown","source":"
\n

🌱 Setting Random Seeds for Reproducibility

\n
","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\nfrom PIL import Image\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torchvision\nimport torchvision.transforms as transforms\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\n# For Image Models\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:49.541344Z","iopub.execute_input":"2023-10-10T18:41:49.541656Z","iopub.status.idle":"2023-10-10T18:41:55.783844Z","shell.execute_reply.started":"2023-10-10T18:41:49.541629Z","shell.execute_reply":"2023-10-10T18:41:55.782935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Training Configuration

\n
\n","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n \"seed\": 42,\n \"epochs\": 20,\n \"img_size\": 512,\n \"model_name\": \"tf_efficientnet_b0_ns\",\n \"checkpoint_path\" : \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n \"num_classes\": 8,\n \"train_batch_size\": 32,\n \"valid_batch_size\": 64,\n \"learning_rate\": 1e-4,\n \"scheduler\": 'CosineAnnealingLR',\n \"min_lr\": 1e-6,\n \"T_max\": 500,\n \"weight_decay\": 1e-6,\n \"fold\" : 0,\n \"n_fold\": 5,\n \"n_accumulate\": 1,\n \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.785518Z","iopub.execute_input":"2023-10-10T18:41:55.786492Z","iopub.status.idle":"2023-10-10T18:41:55.816676Z","shell.execute_reply.started":"2023-10-10T18:41:55.786457Z","shell.execute_reply":"2023-10-10T18:41:55.815853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Set Seed for Reproducibility

\n
\n","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n '''Sets the seed of the entire notebook so results are the same every time we run.\n This is for REPRODUCIBILITY.'''\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.81824Z","iopub.execute_input":"2023-10-10T18:41:55.818947Z","iopub.status.idle":"2023-10-10T18:41:55.840637Z","shell.execute_reply.started":"2023-10-10T18:41:55.818916Z","shell.execute_reply":"2023-10-10T18:41:55.839764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_thumbnails'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.843335Z","iopub.execute_input":"2023-10-10T18:41:55.8439Z","iopub.status.idle":"2023-10-10T18:41:55.853483Z","shell.execute_reply.started":"2023-10-10T18:41:55.843849Z","shell.execute_reply":"2023-10-10T18:41:55.852567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_file_path(image_id):\n return f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"\n# return f\"{TRAIN_DIR}/{image_id}.png\"","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.854814Z","iopub.execute_input":"2023-10-10T18:41:55.855377Z","iopub.status.idle":"2023-10-10T18:41:55.866377Z","shell.execute_reply.started":"2023-10-10T18:41:55.855347Z","shell.execute_reply":"2023-10-10T18:41:55.865474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Read the Data

\n
\n","metadata":{}},{"cell_type":"code","source":"train_images = sorted(glob.glob(f\"{TRAIN_DIR}/*.png\"))","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.868101Z","iopub.execute_input":"2023-10-10T18:41:55.868851Z","iopub.status.idle":"2023-10-10T18:41:55.948493Z","shell.execute_reply.started":"2023-10-10T18:41:55.868816Z","shell.execute_reply":"2023-10-10T18:41:55.94768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf['file_path'] = df['image_id'].apply(get_train_file_path)\ndf = df[ df[\"file_path\"].isin(train_images) ].reset_index(drop=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.949652Z","iopub.execute_input":"2023-10-10T18:41:55.950479Z","iopub.status.idle":"2023-10-10T18:41:55.992739Z","shell.execute_reply.started":"2023-10-10T18:41:55.950448Z","shell.execute_reply":"2023-10-10T18:41:55.991843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ndf['label'] = encoder.fit_transform(df['label'])\n\nwith open(\"label_encoder.pkl\", \"wb\") as fp:\n joblib.dump(encoder, fp)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:55.993931Z","iopub.execute_input":"2023-10-10T18:41:55.994719Z","iopub.status.idle":"2023-10-10T18:41:56.001148Z","shell.execute_reply.started":"2023-10-10T18:41:55.994688Z","shell.execute_reply":"2023-10-10T18:41:56.000297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['T_max'] = df.shape[0] * (CONFIG[\"n_fold\"]-1) * CONFIG['epochs'] // CONFIG['train_batch_size'] // CONFIG[\"n_fold\"]\nCONFIG['T_max']","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:56.002416Z","iopub.execute_input":"2023-10-10T18:41:56.002961Z","iopub.status.idle":"2023-10-10T18:41:56.013117Z","shell.execute_reply.started":"2023-10-10T18:41:56.002932Z","shell.execute_reply":"2023-10-10T18:41:56.012164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Create Folds

\n
\n","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n df.loc[val_ , \"kfold\"] = int(fold)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:56.016888Z","iopub.execute_input":"2023-10-10T18:41:56.017574Z","iopub.status.idle":"2023-10-10T18:41:56.032032Z","shell.execute_reply.started":"2023-10-10T18:41:56.017546Z","shell.execute_reply":"2023-10-10T18:41:56.031212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Dataset Class

\n
\n","metadata":{}},{"cell_type":"code","source":"class UBCDataset(Dataset):\n def __init__(self, df, transforms=None):\n self.df = df\n self.file_names = df['file_path'].values\n self.labels = df['label'].values\n self.transforms = transforms\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n img_path = self.file_names[index]\n img = cv2.imread(img_path)\n img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n label = self.labels[index]\n \n # Convert the NumPy array to a PIL Image\n img = Image.fromarray(img)\n \n if self.transforms:\n img = self.transforms(img)\n \n \n return {\n 'image': img,\n 'label': torch.tensor(label, dtype=torch.long)\n }","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:56.033613Z","iopub.execute_input":"2023-10-10T18:41:56.034286Z","iopub.status.idle":"2023-10-10T18:41:56.041781Z","shell.execute_reply.started":"2023-10-10T18:41:56.034256Z","shell.execute_reply":"2023-10-10T18:41:56.040858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Augmentations

\n
\n","metadata":{}},{"cell_type":"code","source":"transform = transforms.Compose([\n transforms.RandomHorizontalFlip(),\n transforms.RandomCrop(32, padding=4),\n transforms.ToTensor(), \n transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261)) # mean and std\n])\n","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:56.043064Z","iopub.execute_input":"2023-10-10T18:41:56.043588Z","iopub.status.idle":"2023-10-10T18:41:56.058068Z","shell.execute_reply.started":"2023-10-10T18:41:56.043559Z","shell.execute_reply":"2023-10-10T18:41:56.057156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Create Model

\n
\n","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n def __init__(self):\n super(Net, self).__init__()\n self.conv1 = nn.Conv2d(3, 64, 3, padding=1)\n self.bn1 = nn.BatchNorm2d(64) # normalize the activations of each layer\n\n self.conv2 = nn.Conv2d(64, 64, 3, padding=1)\n self.bn2 = nn.BatchNorm2d(64)\n self.conv3 = nn.Conv2d(64, 64, 3, padding=1)\n self.bn3 = nn.BatchNorm2d(64)\n\n self.conv4 = nn.Conv2d(64, 64, 3, padding=1)\n self.bn4 = nn.BatchNorm2d(64)\n self.conv5 = nn.Conv2d(64, 64, 3, padding=1)\n self.bn5 = nn.BatchNorm2d(64)\n\n self.conv6 = nn.Conv2d(64, 128, 3, padding=1)\n self.bn6 = nn.BatchNorm2d(128)\n self.conv7 = nn.Conv2d(128, 128, 3, padding=1)\n self.bn7 = nn.BatchNorm2d(128)\n self.shortcut1 = nn.Sequential(\n nn.Conv2d(64, 128, 1, padding=0, bias=False),\n nn.BatchNorm2d(128)\n )\n self.pool1 = nn.MaxPool2d(2, 2)\n\n self.conv8 = nn.Conv2d(128, 128, 3, padding=1)\n self.bn8 = nn.BatchNorm2d(128)\n self.conv9 = nn.Conv2d(128, 128, 3, padding=1)\n self.bn9 = nn.BatchNorm2d(128)\n\n self.conv10 = nn.Conv2d(128, 128, 3, padding=1)\n self.bn10 = nn.BatchNorm2d(128)\n self.conv11 = nn.Conv2d(128, 128, 3, padding=1)\n self.bn11 = nn.BatchNorm2d(128)\n\n self.conv12 = nn.Conv2d(128, 256, 3, padding=1)\n self.bn12 = nn.BatchNorm2d(256)\n self.conv13 = nn.Conv2d(256, 256, 3, padding=1)\n self.bn13 = nn.BatchNorm2d(256)\n self.shortcut2 = nn.Sequential(\n nn.Conv2d(128, 256, 1, padding=0, bias=False),\n nn.BatchNorm2d(256)\n )\n self.pool2 = nn.MaxPool2d(2, 2)\n\n self.conv14 = nn.Conv2d(256, 256, 3, padding=1)\n self.bn14 = nn.BatchNorm2d(256)\n self.conv15 = nn.Conv2d(256, 256, 3, padding=1)\n self.bn15 = nn.BatchNorm2d(256)\n\n self.conv16 = nn.Conv2d(256, 256, 3, padding=1)\n self.bn16 = nn.BatchNorm2d(256)\n self.conv17 = nn.Conv2d(256, 256, 3, padding=1)\n self.bn17 = nn.BatchNorm2d(256)\n\n self.conv18 = nn.Conv2d(256, 512, 3, padding=1)\n self.bn18 = nn.BatchNorm2d(512)\n self.conv19 = nn.Conv2d(512, 512, 3, padding=1)\n self.bn19 = nn.BatchNorm2d(512)\n self.shortcut3 = nn.Sequential(\n nn.Conv2d(256, 512, 1, padding=0, bias=False),\n nn.BatchNorm2d(512)\n )\n\n self.pool3 = nn.MaxPool2d(2, 2)\n\n self.conv20 = nn.Conv2d(512, 512, 3, padding=1)\n self.bn20 = nn.BatchNorm2d(512)\n self.conv21 = nn.Conv2d(512, 512, 3, padding=1)\n self.bn21 = nn.BatchNorm2d(512)\n\n self.conv22 = nn.Conv2d(512, 512, 3, padding=1)\n self.bn22 = nn.BatchNorm2d(512)\n self.conv23 = nn.Conv2d(512, 512, 3, padding=1)\n self.bn23 = nn.BatchNorm2d(512)\n\n self.conv24 = nn.Conv2d(512, 1024, 3, padding=1)\n self.bn24 = nn.BatchNorm2d(1024)\n self.conv25 = nn.Conv2d(1024, 1024, 3, padding=1)\n self.bn25 = nn.BatchNorm2d(1024)\n self.shortcut4 = nn.Sequential(\n nn.Conv2d(512, 1024, 1, padding=0, bias=False),\n nn.BatchNorm2d(1024))\n self.pool4 = nn.AvgPool2d(4, 4)\n\n self.fc_net = nn.Linear(1024, CONFIG['num_classes']) \n\n self.softmax = nn.Softmax(dim=1)\n\n def forward(self, x):\n '''\n x = self.pool(F.relu(self.conv1(x)))\n # Output size = 28//2 x 28//2 = 14 x 14\n\n x = self.pool(F.relu(self.conv2(x)))\n # Output size = 10//2 x 10//2 = 5 x 5\n\n x = x.view(-1, 16 * 5 * 5)\n x = self.fc_net(x)\n '''\n # print('316:', x.size())\n x = F.relu(self.bn1(self.conv1(x)))\n\n tempx = F.relu(self.bn2(self.conv2(x)))\n tempx = F.relu(self.bn3(self.conv3(tempx)))\n x = tempx + x # Residual connections allow gradients to flow more easily during training, mitigating the vanishing gradient problem in deep networks\n # print('322:', x.size())\n\n tempx = F.relu(self.bn4(self.conv4(x)))\n tempx = F.relu(self.bn5(self.conv5(tempx)))\n x = tempx + x\n # print('327:', x.size())\n\n tempx = F.relu(self.bn6(self.conv6(x)))\n tempx = F.relu(self.bn7(self.conv7(tempx)))\n x = tempx + self.shortcut1(x)\n x = self.pool1(x)\n # print('333:', x.size())\n\n tempx = F.relu(self.bn8(self.conv8(x)))\n tempx = F.relu(self.bn9(self.conv9(tempx)))\n x = tempx + x\n # print('338:', x.size())\n\n tempx = F.relu(self.bn10(self.conv10(x)))\n tempx = F.relu(self.bn11(self.conv11(tempx)))\n x = tempx + x\n # print('343:', x.size())\n\n tempx = F.relu(self.bn12(self.conv12(x)))\n tempx = F.relu(self.bn13(self.conv13(tempx)))\n x = tempx + self.shortcut2(x)\n x = self.pool2(x)\n # print('349:', x.size())\n\n tempx = F.relu(self.bn14(self.conv14(x)))\n tempx = F.relu(self.bn15(self.conv15(tempx)))\n x = tempx + x\n # print('354:', x.size())\n\n tempx = F.relu(self.bn16(self.conv16(x)))\n tempx = F.relu(self.bn17(self.conv17(tempx)))\n x = tempx + x\n # print('359:', x.size())\n\n tempx = F.relu(self.bn18(self.conv18(x)))\n tempx = F.relu(self.bn19(self.conv19(tempx)))\n x = tempx + self.shortcut3(x)\n x = self.pool3(x)\n # print('365:', x.size())\n\n tempx = F.relu(self.bn20(self.conv20(x)))\n tempx = F.relu(self.bn21(self.conv21(tempx)))\n x = tempx + x\n # print('370:', x.size())\n\n tempx = F.relu(self.bn22(self.conv22(x)))\n tempx = F.relu(self.bn23(self.conv23(tempx)))\n x = tempx + x\n # print('375:', x.size())\n\n tempx = F.relu(self.bn24(self.conv24(x)))\n tempx = F.relu(self.bn25(self.conv25(tempx)))\n x = tempx + self.shortcut4(x)\n # print('380:', x.size())\n x = self.pool4(x)\n # print('382:', x.size())\n\n x = x.view(x.size(0), -1)\n # print('385:', x.size())\n x = self.fc_net(x)\n # print('387:', x.size())\n\n return x\n\n\n# Move the model to the GPU if available\nmodel = Net().to(CONFIG[\"device\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:56.05939Z","iopub.execute_input":"2023-10-10T18:41:56.059947Z","iopub.status.idle":"2023-10-10T18:41:59.672001Z","shell.execute_reply.started":"2023-10-10T18:41:56.059918Z","shell.execute_reply":"2023-10-10T18:41:59.671025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Loss Function

\n
\n","metadata":{}},{"cell_type":"code","source":"def criterion(outputs, labels):\n return nn.CrossEntropyLoss()(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.673475Z","iopub.execute_input":"2023-10-10T18:41:59.674078Z","iopub.status.idle":"2023-10-10T18:41:59.684338Z","shell.execute_reply.started":"2023-10-10T18:41:59.674046Z","shell.execute_reply":"2023-10-10T18:41:59.682921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Training Function

\n
\n","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n model.train()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar:\n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n \n outputs = model(images)\n loss = criterion(outputs, labels)\n loss = loss / CONFIG['n_accumulate']\n \n loss.backward()\n \n if (step + 1) % CONFIG['n_accumulate'] == 0:\n optimizer.step()\n\n # zero the parameter gradients\n optimizer.zero_grad()\n\n if scheduler is not None:\n scheduler.step()\n \n _, predicted = torch.max(torch.softmax(outputs, dim=1), 1) # Apply softmax here\n acc = torch.sum(predicted == labels)\n \n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n \n bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss, Train_Acc=epoch_acc,\n LR=optimizer.param_groups[0]['lr'])\n gc.collect()\n \n return epoch_loss, epoch_acc\n","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.687417Z","iopub.execute_input":"2023-10-10T18:41:59.688115Z","iopub.status.idle":"2023-10-10T18:41:59.702648Z","shell.execute_reply.started":"2023-10-10T18:41:59.688072Z","shell.execute_reply":"2023-10-10T18:41:59.701707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Validation Function

\n
\n","metadata":{}},{"cell_type":"code","source":"@torch.inference_mode()\ndef valid_one_epoch(model, dataloader, device, epoch):\n model.eval()\n \n dataset_size = 0\n running_loss = 0.0\n running_acc = 0.0\n \n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar: \n images = data['image'].to(device, dtype=torch.float)\n labels = data['label'].to(device, dtype=torch.long)\n \n batch_size = images.size(0)\n\n outputs = model(images)\n loss = criterion(outputs, labels)\n\n _, predicted = torch.max(model.softmax(outputs), 1)\n acc = torch.sum( predicted == labels )\n\n running_loss += (loss.item() * batch_size)\n running_acc += acc.item()\n dataset_size += batch_size\n \n epoch_loss = running_loss / dataset_size\n epoch_acc = running_acc / dataset_size\n \n bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss, Valid_Acc=epoch_acc,\n LR=optimizer.param_groups[0]['lr']) \n \n gc.collect()\n \n return epoch_loss, epoch_acc","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.703779Z","iopub.execute_input":"2023-10-10T18:41:59.704358Z","iopub.status.idle":"2023-10-10T18:41:59.724614Z","shell.execute_reply.started":"2023-10-10T18:41:59.704328Z","shell.execute_reply":"2023-10-10T18:41:59.723845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Run Training

\n
\n","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n if torch.cuda.is_available():\n print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n \n start = time.time()\n best_model_wts = copy.deepcopy(model.state_dict())\n best_epoch_acc = -np.inf\n history = defaultdict(list)\n \n for epoch in range(1, num_epochs + 1): \n gc.collect()\n train_epoch_loss, train_epoch_acc = train_one_epoch(model, optimizer, scheduler, \n dataloader=train_loader, \n device=CONFIG['device'], epoch=epoch)\n \n val_epoch_loss, val_epoch_acc = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n epoch=epoch)\n \n history['Train Loss'].append(train_epoch_loss)\n history['Valid Loss'].append(val_epoch_loss)\n history['Train Accuracy'].append(train_epoch_acc)\n history['Valid Accuracy'].append(val_epoch_acc)\n history['lr'].append( scheduler.get_lr()[0] )\n \n # deep copy the model\n if best_epoch_acc <= val_epoch_acc:\n print(f\"{b_}Validation Accuracy Improved ({best_epoch_acc} ---> {val_epoch_acc})\")\n best_epoch_acc = val_epoch_acc\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = \"Acc{:.2f}_Loss{:.4f}_epoch{:.0f}.bin\".format(best_epoch_acc, val_epoch_loss, epoch)\n torch.save(model.state_dict(), PATH)\n # Save a model file from the current directory\n print(f\"Model Saved{sr_}\")\n \n print()\n \n end = time.time()\n time_elapsed = end - start\n print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n print(\"Best Accuracy: {:.4f}\".format(best_epoch_acc))\n \n # load best model weights\n model.load_state_dict(best_model_wts)\n \n return model, history","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.725726Z","iopub.execute_input":"2023-10-10T18:41:59.726232Z","iopub.status.idle":"2023-10-10T18:41:59.755041Z","shell.execute_reply.started":"2023-10-10T18:41:59.726203Z","shell.execute_reply":"2023-10-10T18:41:59.754149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n if CONFIG['scheduler'] == 'CosineAnnealingLR':\n scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n eta_min=CONFIG['min_lr'])\n elif CONFIG['scheduler'] == None:\n return None\n \n return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.758922Z","iopub.execute_input":"2023-10-10T18:41:59.761076Z","iopub.status.idle":"2023-10-10T18:41:59.770917Z","shell.execute_reply.started":"2023-10-10T18:41:59.761046Z","shell.execute_reply":"2023-10-10T18:41:59.770023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n df_train = df[df.kfold != fold].reset_index(drop=True)\n df_valid = df[df.kfold == fold].reset_index(drop=True)\n \n train_dataset = UBCDataset(df_train, transforms=transform)\n valid_dataset = UBCDataset(df_valid, transforms=transform)\n\n train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n num_workers=1, shuffle=True, pin_memory=True, drop_last=True)\n valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n num_workers=1, shuffle=False, pin_memory=True)\n \n return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.775061Z","iopub.execute_input":"2023-10-10T18:41:59.777575Z","iopub.status.idle":"2023-10-10T18:41:59.786488Z","shell.execute_reply.started":"2023-10-10T18:41:59.777544Z","shell.execute_reply":"2023-10-10T18:41:59.785667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Prepare Dataloaders

\n
","metadata":{}},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=CONFIG[\"fold\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.790618Z","iopub.execute_input":"2023-10-10T18:41:59.792898Z","iopub.status.idle":"2023-10-10T18:41:59.803115Z","shell.execute_reply.started":"2023-10-10T18:41:59.792868Z","shell.execute_reply":"2023-10-10T18:41:59.802147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Define Optimizer and Scheduler

\n
","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.807308Z","iopub.execute_input":"2023-10-10T18:41:59.809281Z","iopub.status.idle":"2023-10-10T18:41:59.816341Z","shell.execute_reply.started":"2023-10-10T18:41:59.809251Z","shell.execute_reply":"2023-10-10T18:41:59.815397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Start Training

\n
","metadata":{}},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n device=CONFIG['device'],\n num_epochs=CONFIG['epochs'])","metadata":{"execution":{"iopub.status.busy":"2023-10-10T18:41:59.820431Z","iopub.execute_input":"2023-10-10T18:41:59.822719Z","iopub.status.idle":"2023-10-10T19:14:42.798984Z","shell.execute_reply.started":"2023-10-10T18:41:59.822689Z","shell.execute_reply":"2023-10-10T19:14:42.797946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = pd.DataFrame.from_dict(history)\nhistory.to_csv(\"history.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T19:14:42.800402Z","iopub.execute_input":"2023-10-10T19:14:42.801338Z","iopub.status.idle":"2023-10-10T19:14:42.810874Z","shell.execute_reply.started":"2023-10-10T19:14:42.801306Z","shell.execute_reply":"2023-10-10T19:14:42.810025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Logs

\n
","metadata":{}},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Loss\"].values, label=\"Train Loss\")\nplt.plot( range(history.shape[0]), history[\"Valid Loss\"].values, label=\"Valid Loss\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Loss\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-10T19:14:42.812183Z","iopub.execute_input":"2023-10-10T19:14:42.813124Z","iopub.status.idle":"2023-10-10T19:14:43.065973Z","shell.execute_reply.started":"2023-10-10T19:14:42.813087Z","shell.execute_reply":"2023-10-10T19:14:43.06508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Accuracy\"].values, label=\"Train Accuracy\")\nplt.plot( range(history.shape[0]), history[\"Valid Accuracy\"].values, label=\"Valid Accuracy\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Accuracy\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-10T19:14:43.067367Z","iopub.execute_input":"2023-10-10T19:14:43.067924Z","iopub.status.idle":"2023-10-10T19:14:43.318485Z","shell.execute_reply.started":"2023-10-10T19:14:43.067892Z","shell.execute_reply":"2023-10-10T19:14:43.317664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"lr\"].values, label=\"lr\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"lr\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-10T19:14:43.320018Z","iopub.execute_input":"2023-10-10T19:14:43.320616Z","iopub.status.idle":"2023-10-10T19:14:43.547955Z","shell.execute_reply.started":"2023-10-10T19:14:43.320583Z","shell.execute_reply":"2023-10-10T19:14:43.54706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

My first Kaggle code share - Your vote would mean the world to me! Thank you❤️

\n
\n=","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}