{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":3729,"sourceType":"modelInstanceVersion","modelInstanceId":2656}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Using [motono0223](https://www.kaggle.com/motono0223)'s notebook as baseline. I am adding fetaures such as \n* Class weights for cross entropy loss\n* adding aditional images thats isnt in thumbnails folder\n* balanced accuracy as metric for saving model as there is huge class imbalance\n","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\nimport torchvision\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\n# For Image Models\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:40:04.20736Z","iopub.execute_input":"2023-10-21T10:40:04.207634Z","iopub.status.idle":"2023-10-21T10:40:11.093815Z","shell.execute_reply.started":"2023-10-21T10:40:04.207609Z","shell.execute_reply":"2023-10-21T10:40:11.093003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 10,\n    \"epochs\": 25,\n    \"img_size\": 512,\n    \"model_name\": \"tf_efficientnetv2_l_in21ft1k\",\n    \"checkpoint_path\" : \"/kaggle/input/tf-efficientnet/pytorch/tf-efficientnet-b0/1/tf_efficientnet_b0_aa-827b6e33.pth\",\n    \"num_classes\": 5,\n    \"train_batch_size\": 8,\n    \"valid_batch_size\": 64,\n    \"learning_rate\": 1e-4,\n    \"scheduler\": 'CosineAnnealingLR',\n    \"min_lr\": 1e-6,\n    \"T_max\": 500,\n    \"weight_decay\": 1e-6,\n    \"fold\" : 0,\n    \"n_fold\": 5,\n    \"n_accumulate\": 1,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:44.907001Z","iopub.execute_input":"2023-10-21T10:45:44.907386Z","iopub.status.idle":"2023-10-21T10:45:44.913395Z","shell.execute_reply.started":"2023-10-21T10:45:44.907357Z","shell.execute_reply":"2023-10-21T10:45:44.912423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:45.537353Z","iopub.execute_input":"2023-10-21T10:45:45.537939Z","iopub.status.idle":"2023-10-21T10:45:45.547799Z","shell.execute_reply.started":"2023-10-21T10:45:45.537907Z","shell.execute_reply":"2023-10-21T10:45:45.546989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_thumbnails'\nALT_TEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_thumbnails'\nALT_TRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_images'","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:46.753274Z","iopub.execute_input":"2023-10-21T10:45:46.753713Z","iopub.status.idle":"2023-10-21T10:45:46.758431Z","shell.execute_reply.started":"2023-10-21T10:45:46.753685Z","shell.execute_reply":"2023-10-21T10:45:46.757346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this should load few extra images which arent thumbnails\ndef get_train_file_path(image_id):\n    if os.path.exists(f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"):\n        return f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"\n    else:\n        return f\"{ALT_TRAIN_DIR}/{image_id}.png\"\n#    return f\"{TRAIN_DIR}/{image_id}.png\"","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:47.578102Z","iopub.execute_input":"2023-10-21T10:45:47.57847Z","iopub.status.idle":"2023-10-21T10:45:47.583522Z","shell.execute_reply.started":"2023-10-21T10:45:47.578441Z","shell.execute_reply":"2023-10-21T10:45:47.582533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = sorted(glob.glob(f\"{TRAIN_DIR}/*.png\"))","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:48.47329Z","iopub.execute_input":"2023-10-21T10:45:48.473638Z","iopub.status.idle":"2023-10-21T10:45:48.50523Z","shell.execute_reply.started":"2023-10-21T10:45:48.473609Z","shell.execute_reply":"2023-10-21T10:45:48.504474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf['file_path'] = df['image_id'].apply(get_train_file_path)\n# df = df[ df[\"file_path\"].isin(train_images) ].reset_index(drop=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:48.758525Z","iopub.execute_input":"2023-10-21T10:45:48.759247Z","iopub.status.idle":"2023-10-21T10:45:48.815042Z","shell.execute_reply.started":"2023-10-21T10:45:48.759214Z","shell.execute_reply":"2023-10-21T10:45:48.814118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ndf['label'] = encoder.fit_transform(df['label'])\n\nwith open(\"label_encoder.pkl\", \"wb\") as fp:\n    joblib.dump(encoder, fp)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:49.27377Z","iopub.execute_input":"2023-10-21T10:45:49.274617Z","iopub.status.idle":"2023-10-21T10:45:49.280585Z","shell.execute_reply.started":"2023-10-21T10:45:49.274584Z","shell.execute_reply":"2023-10-21T10:45:49.279541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG['T_max'] = df.shape[0] * (CONFIG[\"n_fold\"]-1) * CONFIG['epochs'] // CONFIG['train_batch_size'] // CONFIG[\"n_fold\"]\nCONFIG['T_max']","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:49.713198Z","iopub.execute_input":"2023-10-21T10:45:49.713516Z","iopub.status.idle":"2023-10-21T10:45:49.720248Z","shell.execute_reply.started":"2023-10-21T10:45:49.713493Z","shell.execute_reply":"2023-10-21T10:45:49.719323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_class_weights(df, label_column):\n    \"\"\"\n    Compute class weights based on the inverse of class frequencies.\n    \n    Parameters:\n        df (pd.DataFrame): DataFrame containing the data.\n        label_column (str): Name of the column containing class labels.\n    \n    Returns:\n        class_weights (dict): Dictionary containing weights for each class.\n    \"\"\"\n    # Get the total number of samples\n    total_samples = len(df)\n    \n    # Get the number of classes\n    num_classes = df[label_column].nunique()\n    \n    # Get the count of each class\n    class_counts = df[label_column].value_counts().to_dict()\n    \n    # Compute class weights\n    class_weights = {class_label: total_samples / (num_classes * count) \n                     for class_label, count in class_counts.items()}\n    \n    return class_weights\n\n# Compute class weights for the provided dataset\nclass_weights = compute_class_weights(df, 'label')\nclass_weights = np.array([1.0868686868686868,0.867741935483871,0.4846846846846847,2.2893617021276595,2.3391304347826085])\nclass_weights_tensor = torch.tensor(class_weights, dtype=torch.float32)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:53.202606Z","iopub.execute_input":"2023-10-21T10:45:53.202955Z","iopub.status.idle":"2023-10-21T10:45:53.244949Z","shell.execute_reply.started":"2023-10-21T10:45:53.202926Z","shell.execute_reply":"2023-10-21T10:45:53.244016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=CONFIG['n_fold'])\n\nfor fold, ( _, val_) in enumerate(skf.split(X=df, y=df.label)):\n      df.loc[val_ , \"kfold\"] = int(fold)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:54.297502Z","iopub.execute_input":"2023-10-21T10:45:54.297836Z","iopub.status.idle":"2023-10-21T10:45:54.309457Z","shell.execute_reply.started":"2023-10-21T10:45:54.29781Z","shell.execute_reply":"2023-10-21T10:45:54.308569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.labels = df['label'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = self.file_names[index]\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.labels[index]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return {\n            'image': img,\n            'label': torch.tensor(label, dtype=torch.long)\n        }","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:45:55.161297Z","iopub.execute_input":"2023-10-21T10:45:55.162086Z","iopub.status.idle":"2023-10-21T10:45:55.169002Z","shell.execute_reply.started":"2023-10-21T10:45:55.162043Z","shell.execute_reply":"2023-10-21T10:45:55.168006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    \"train\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n         \n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomBrightnessContrast(p=0.75),\n        A.ShiftScaleRotate(p=0.75),\n        A.OneOf([\n                A.GaussNoise(var_limit=[10, 50]),\n                A.GaussianBlur(),\n                A.MotionBlur(),\n                ], p=0.4),\n        A.GridDistortion(num_steps=5, distort_limit=0.3, p=0.5),\n        A.CoarseDropout(max_holes=1, max_width=int(512 * 0.13), max_height=int(512 * 0.13), \n                        mask_fill_value=0, p=0.5),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.),\n    \n    \"valid\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:24.494021Z","iopub.execute_input":"2023-10-21T10:46:24.494389Z","iopub.status.idle":"2023-10-21T10:46:24.504352Z","shell.execute_reply.started":"2023-10-21T10:46:24.494358Z","shell.execute_reply":"2023-10-21T10:46:24.503474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:25.754592Z","iopub.execute_input":"2023-10-21T10:46:25.754944Z","iopub.status.idle":"2023-10-21T10:46:25.762537Z","shell.execute_reply.started":"2023-10-21T10:46:25.754915Z","shell.execute_reply":"2023-10-21T10:46:25.761593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCModel(nn.Module):\n    def __init__(self, model_name, num_classes, pretrained=True, checkpoint_path=None):\n        super(UBCModel, self).__init__()\n        self.model = timm.create_model('tf_efficientnetv2_m', pretrained=pretrained)\n\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.linear = nn.Linear(in_features, num_classes)\n        self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, images):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        output = self.linear(pooled_features)\n        return output\n\n    \nmodel = UBCModel(CONFIG['model_name'], CONFIG['num_classes'], checkpoint_path=CONFIG['checkpoint_path'])\nmodel.to(CONFIG['device']);","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:30.232613Z","iopub.execute_input":"2023-10-21T10:46:30.232977Z","iopub.status.idle":"2023-10-21T10:46:40.763422Z","shell.execute_reply.started":"2023-10-21T10:46:30.232947Z","shell.execute_reply":"2023-10-21T10:46:40.7624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = nn.CrossEntropyLoss(weight = class_weights_tensor.to(CONFIG['device']))\ndef criterion(outputs, labels):\n    return loss(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:40.765187Z","iopub.execute_input":"2023-10-21T10:46:40.766003Z","iopub.status.idle":"2023-10-21T10:46:40.771168Z","shell.execute_reply.started":"2023-10-21T10:46:40.765967Z","shell.execute_reply":"2023-10-21T10:46:40.770141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import balanced_accuracy_score\ndef train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    running_acc  = 0.0\n    all_preds=[]\n    all_labels=[]\n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        images = data['image'].to(device, dtype=torch.float)\n        labels = data['label'].to(device, dtype=torch.long)\n        \n        batch_size = images.size(0)\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss = loss / CONFIG['n_accumulate']\n            \n        loss.backward()\n    \n        if (step + 1) % CONFIG['n_accumulate'] == 0:\n            optimizer.step()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n                \n        _, predicted = torch.max(torch.nn.Softmax(dim=1)(outputs), 1)\n        acc = torch.sum( predicted == labels )\n        all_preds.extend(predicted.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n        running_loss += (loss.item() * batch_size)\n        running_acc  += acc.item()\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        epoch_acc = running_acc / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss, Train_Acc=epoch_acc,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect()\n    bl_accuracy_score = balanced_accuracy_score(all_labels, all_preds)\n    return epoch_loss, epoch_acc,bl_accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:42.189266Z","iopub.execute_input":"2023-10-21T10:46:42.191323Z","iopub.status.idle":"2023-10-21T10:46:42.206245Z","shell.execute_reply.started":"2023-10-21T10:46:42.191283Z","shell.execute_reply":"2023-10-21T10:46:42.204953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.inference_mode()\ndef valid_one_epoch(model, dataloader, device, epoch):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    running_acc = 0.0\n    all_preds=[]\n    all_labels=[]\n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:        \n        images = data['image'].to(device, dtype=torch.float)\n        labels = data['label'].to(device, dtype=torch.long)\n        \n        batch_size = images.size(0)\n\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n\n        _, predicted = torch.max(torch.nn.Softmax(dim=1)(outputs), 1)\n        all_preds.extend(predicted.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n        acc = torch.sum( predicted == labels )\n\n        running_loss += (loss.item() * batch_size)\n        running_acc  += acc.item()\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        epoch_acc = running_acc / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss, Valid_Acc=epoch_acc,\n                        LR=optimizer.param_groups[0]['lr'])   \n    \n    gc.collect()\n    bl_accuracy_score = balanced_accuracy_score(all_labels, all_preds)\n    return epoch_loss, epoch_acc,bl_accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:43.044235Z","iopub.execute_input":"2023-10-21T10:46:43.044594Z","iopub.status.idle":"2023-10-21T10:46:43.055642Z","shell.execute_reply.started":"2023-10-21T10:46:43.044565Z","shell.execute_reply":"2023-10-21T10:46:43.054808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_acc = -np.inf\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n        train_epoch_loss, train_epoch_acc, train_bl_accuracy_score = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=CONFIG['device'], epoch=epoch)\n        \n        val_epoch_loss, val_epoch_acc, val_bl_accuracy_score = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n                                         epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n        history['Valid Loss'].append(val_epoch_loss)\n        history['Train Accuracy'].append(train_epoch_acc)\n        history['Valid Accuracy'].append(val_epoch_acc)\n        history['Train balanced Accuracy'].append(train_bl_accuracy_score)\n        history['Valid balanced Accuracy'].append(val_bl_accuracy_score)\n        history['lr'].append( scheduler.get_lr()[0] )\n        \n        # deep copy the model\n        if best_epoch_acc <= val_bl_accuracy_score:\n            print(f\"{b_}Validation Balanced Accuracy Improved ({best_epoch_acc} ---> {val_bl_accuracy_score})\")\n            best_epoch_acc = val_bl_accuracy_score\n            best_model_wts = copy.deepcopy(model.state_dict())\n            PATH = \"Acc{:.2f}_Loss{:.4f}_epoch{:.0f}.bin\".format(best_epoch_acc, val_epoch_loss, epoch)\n            torch.save(model.state_dict(), PATH)\n            # Save a model file from the current directory\n            print(f\"Model Saved{sr_}\")\n            \n        print()\n    \n    end = time.time()\n    time_elapsed = end - start\n    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n    print(\"Best Accuracy: {:.4f}\".format(best_epoch_acc))\n    \n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:43.700851Z","iopub.execute_input":"2023-10-21T10:46:43.701236Z","iopub.status.idle":"2023-10-21T10:46:43.713365Z","shell.execute_reply.started":"2023-10-21T10:46:43.701205Z","shell.execute_reply":"2023-10-21T10:46:43.71217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if CONFIG['scheduler'] == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=CONFIG['T_max'], \n                                                   eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=CONFIG['T_0'], \n                                                             eta_min=CONFIG['min_lr'])\n    elif CONFIG['scheduler'] == None:\n        return None\n        \n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:44.346695Z","iopub.execute_input":"2023-10-21T10:46:44.347052Z","iopub.status.idle":"2023-10-21T10:46:44.353488Z","shell.execute_reply.started":"2023-10-21T10:46:44.34702Z","shell.execute_reply":"2023-10-21T10:46:44.352314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_loaders(df, fold):\n    df_train = df[df.kfold != fold].reset_index(drop=True)\n    df_valid = df[df.kfold == fold].reset_index(drop=True)\n    \n    train_dataset = UBCDataset(df_train, transforms=data_transforms[\"train\"])\n    valid_dataset = UBCDataset(df_valid, transforms=data_transforms[\"valid\"])\n\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], \n                              num_workers=2, shuffle=False, pin_memory=True)\n    \n    return train_loader, valid_loader","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:44.605054Z","iopub.execute_input":"2023-10-21T10:46:44.605455Z","iopub.status.idle":"2023-10-21T10:46:44.612683Z","shell.execute_reply.started":"2023-10-21T10:46:44.605426Z","shell.execute_reply":"2023-10-21T10:46:44.611659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader, valid_loader = prepare_loaders(df, fold=CONFIG[\"fold\"])","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:45.986334Z","iopub.execute_input":"2023-10-21T10:46:45.986725Z","iopub.status.idle":"2023-10-21T10:46:45.995228Z","shell.execute_reply.started":"2023-10-21T10:46:45.986686Z","shell.execute_reply":"2023-10-21T10:46:45.994135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n                       weight_decay=CONFIG['weight_decay'])\nscheduler = fetch_scheduler(optimizer)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:47.18901Z","iopub.execute_input":"2023-10-21T10:46:47.189946Z","iopub.status.idle":"2023-10-21T10:46:47.200841Z","shell.execute_reply.started":"2023-10-21T10:46:47.189913Z","shell.execute_reply":"2023-10-21T10:46:47.200011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = run_training(model, optimizer, scheduler,\n                              device=CONFIG['device'],\n                              num_epochs=30)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T10:46:48.196866Z","iopub.execute_input":"2023-10-21T10:46:48.197782Z","iopub.status.idle":"2023-10-21T11:18:01.758115Z","shell.execute_reply.started":"2023-10-21T10:46:48.197739Z","shell.execute_reply":"2023-10-21T11:18:01.757086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = pd.DataFrame.from_dict(history)\nhistory.to_csv(\"history.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T04:55:16.970628Z","iopub.execute_input":"2023-10-21T04:55:16.970905Z","iopub.status.idle":"2023-10-21T04:55:16.979948Z","shell.execute_reply.started":"2023-10-21T04:55:16.970879Z","shell.execute_reply":"2023-10-21T04:55:16.979217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Loss\"].values, label=\"Train Loss\")\nplt.plot( range(history.shape[0]), history[\"Valid Loss\"].values, label=\"Valid Loss\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Loss\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T04:55:46.46388Z","iopub.execute_input":"2023-10-21T04:55:46.464619Z","iopub.status.idle":"2023-10-21T04:55:46.753645Z","shell.execute_reply.started":"2023-10-21T04:55:46.464586Z","shell.execute_reply":"2023-10-21T04:55:46.752488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"Train Accuracy\"].values, label=\"Train Accuracy\")\nplt.plot( range(history.shape[0]), history[\"Valid Accuracy\"].values, label=\"Valid Accuracy\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"Accuracy\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T04:55:46.755536Z","iopub.execute_input":"2023-10-21T04:55:46.755981Z","iopub.status.idle":"2023-10-21T04:55:47.030525Z","shell.execute_reply.started":"2023-10-21T04:55:46.755945Z","shell.execute_reply":"2023-10-21T04:55:47.0296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot( range(history.shape[0]), history[\"lr\"].values, label=\"lr\")\nplt.xlabel(\"epochs\")\nplt.ylabel(\"lr\")\nplt.grid()\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T04:55:47.742799Z","iopub.execute_input":"2023-10-21T04:55:47.74316Z","iopub.status.idle":"2023-10-21T04:55:47.937804Z","shell.execute_reply.started":"2023-10-21T04:55:47.74313Z","shell.execute_reply":"2023-10-21T04:55:47.936811Z"},"trusted":true},"execution_count":null,"outputs":[]}]}