{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Simple Training Pipeline inspired from [this inferenece notebook](https://www.kaggle.com/code/khailashsanthakumar/ubc-pytorch-effb0-with-classweights-inference)","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport math\nimport copy\nimport time\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import StratifiedKFold\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.preprocessing import LabelEncoder\nfrom torch.cuda import amp\nimport torchvision\nimport torch\nimport torch.nn as nn\nfrom torch.optim import AdamW\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold\n\n# For Image Models\nimport timm\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nb_ = Fore.BLUE\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-10-19T09:43:15.96575Z","iopub.execute_input":"2023-10-19T09:43:15.966208Z","iopub.status.idle":"2023-10-19T09:43:22.723231Z","shell.execute_reply.started":"2023-10-19T09:43:15.966165Z","shell.execute_reply":"2023-10-19T09:43:22.722326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    \"seed\": 42,\n    \"img_size\": 512, #380\n    \"model_name\": 'efficientnetv2_rw_m',\n    \"num_classes\": 5,\n    \"batch_size\": 8,\n    \"test_batch_size\":1,\n    \"num_epochs\" : 3,\n    \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:22.724846Z","iopub.execute_input":"2023-10-19T09:43:22.7253Z","iopub.status.idle":"2023-10-19T09:43:22.754768Z","shell.execute_reply.started":"2023-10-19T09:43:22.725275Z","shell.execute_reply":"2023-10-19T09:43:22.753898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    \nset_seed(CONFIG['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:22.75624Z","iopub.execute_input":"2023-10-19T09:43:22.756493Z","iopub.status.idle":"2023-10-19T09:43:22.785668Z","shell.execute_reply.started":"2023-10-19T09:43:22.756472Z","shell.execute_reply":"2023-10-19T09:43:22.784805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTEST_DIR = '/kaggle/input/UBC-OCEAN/test_thumbnails'\nTRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_thumbnails'\nALT_TEST_DIR = '/kaggle/input/UBC-OCEAN/test_images'\nALT_TRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_images'\nLABEL_ENCODER_BIN = \"/kaggle/input/ubcpytorchwith-classweights-training-fold1of5/label_encoder.pkl\"","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:22.787928Z","iopub.execute_input":"2023-10-19T09:43:22.788237Z","iopub.status.idle":"2023-10-19T09:43:22.79298Z","shell.execute_reply.started":"2023-10-19T09:43:22.788217Z","shell.execute_reply":"2023-10-19T09:43:22.792019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_file_path(image_id):\n    if os.path.exists(f\"{TEST_DIR}/{image_id}_thumbnail.png\"):\n        return f\"{TEST_DIR}/{image_id}_thumbnail.png\"\n    else:\n        return f\"{ALT_TEST_DIR}/{image_id}.png\"\n\n\ndef get_train_file_path(image_id):\n    if os.path.exists(f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"):\n        return f\"{TRAIN_DIR}/{image_id}_thumbnail.png\"\n    else:\n        return f\"{ALT_TRAIN_DIR}/{image_id}.png\"","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:22.793975Z","iopub.execute_input":"2023-10-19T09:43:22.794229Z","iopub.status.idle":"2023-10-19T09:43:22.808707Z","shell.execute_reply.started":"2023-10-19T09:43:22.794209Z","shell.execute_reply":"2023-10-19T09:43:22.807773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\ndf['file_path'] = df['image_id'].apply(get_train_file_path)","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:22.809845Z","iopub.execute_input":"2023-10-19T09:43:22.810138Z","iopub.status.idle":"2023-10-19T09:43:23.257573Z","shell.execute_reply.started":"2023-10-19T09:43:22.810118Z","shell.execute_reply":"2023-10-19T09:43:23.256599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = joblib.load( LABEL_ENCODER_BIN )","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.258679Z","iopub.execute_input":"2023-10-19T09:43:23.258927Z","iopub.status.idle":"2023-10-19T09:43:23.266996Z","shell.execute_reply.started":"2023-10-19T09:43:23.258907Z","shell.execute_reply":"2023-10-19T09:43:23.266273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCDataset(Dataset):\n    def __init__(self, df, transforms=None):\n        self.df = df\n        self.file_names = df['file_path'].values\n        self.labels = df['label'].values\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        img_path = self.file_names[index]\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        label = self.labels[index]\n        \n        if self.transforms:\n            img = self.transforms(image=img)[\"image\"]\n            \n        return {\n            'image': img,\n            'label': torch.tensor(encoder.transform([label]), dtype=torch.long)\n        }","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.267979Z","iopub.execute_input":"2023-10-19T09:43:23.26831Z","iopub.status.idle":"2023-10-19T09:43:23.276056Z","shell.execute_reply.started":"2023-10-19T09:43:23.268283Z","shell.execute_reply":"2023-10-19T09:43:23.27509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    \"valid\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.) ,\n    \"train\": A.Compose([\n        A.Resize(CONFIG['img_size'], CONFIG['img_size']),\n        A.Normalize(\n                mean=[0.485, 0.456, 0.406], \n                std=[0.229, 0.224, 0.225], \n                max_pixel_value=255.0, \n                p=1.0\n            ),\n        ToTensorV2()], p=1.)\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.277094Z","iopub.execute_input":"2023-10-19T09:43:23.277335Z","iopub.status.idle":"2023-10-19T09:43:23.288524Z","shell.execute_reply.started":"2023-10-19T09:43:23.277315Z","shell.execute_reply":"2023-10-19T09:43:23.287818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6):\n        super(GeM, self).__init__()\n        self.p = nn.Parameter(torch.ones(1)*p)\n        self.eps = eps\n\n    def forward(self, x):\n        return self.gem(x, p=self.p, eps=self.eps)\n        \n    def gem(self, x, p=3, eps=1e-6):\n        return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1./p)\n        \n    def __repr__(self):\n        return self.__class__.__name__ + \\\n                '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \\\n                ', ' + 'eps=' + str(self.eps) + ')'","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.290521Z","iopub.execute_input":"2023-10-19T09:43:23.290769Z","iopub.status.idle":"2023-10-19T09:43:23.303781Z","shell.execute_reply.started":"2023-10-19T09:43:23.290737Z","shell.execute_reply":"2023-10-19T09:43:23.302855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCModel(nn.Module):\n    def __init__(self, model_name, num_classes, pretrained=True, checkpoint_path=None):\n        super(UBCModel, self).__init__()\n        self.model = timm.create_model(model_name, pretrained=pretrained)\n\n        in_features = self.model.classifier.in_features\n        self.model.classifier = nn.Identity()\n        self.model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        self.linear = nn.Linear(in_features, num_classes)\n        self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, images):\n        features = self.model(images)\n        pooled_features = self.pooling(features).flatten(1)\n        output = self.linear(pooled_features)\n        return output","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.304698Z","iopub.execute_input":"2023-10-19T09:43:23.304963Z","iopub.status.idle":"2023-10-19T09:43:23.314712Z","shell.execute_reply.started":"2023-10-19T09:43:23.304935Z","shell.execute_reply":"2023-10-19T09:43:23.313898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=CONFIG['seed'])  # you can choose another random state","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.315692Z","iopub.execute_input":"2023-10-19T09:43:23.316009Z","iopub.status.idle":"2023-10-19T09:43:23.326716Z","shell.execute_reply.started":"2023-10-19T09:43:23.315977Z","shell.execute_reply":"2023-10-19T09:43:23.326017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.327527Z","iopub.execute_input":"2023-10-19T09:43:23.327718Z","iopub.status.idle":"2023-10-19T09:43:23.35117Z","shell.execute_reply.started":"2023-10-19T09:43:23.327701Z","shell.execute_reply":"2023-10-19T09:43:23.350325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df.drop('label', axis=1)  # feature matrix\ny = df['label']  # target variable\n# Enumerate over each split index and train and test indices\nfor fold, (train_index, test_index) in enumerate(skf.split(X, y), 1):\n    df_train , df_valid = df.iloc[train_index],df.iloc[test_index]\n    # At this point, you can create and train your model with the training data\n    # And subsequently evaluate its performance with the test data\n    print(f'Fold: {fold}, Train set: {len(train_index)}, Test set: {len(test_index)}')\n    # Additional model training and evaluation code should be here\n    model = UBCModel(CONFIG['model_name'], CONFIG['num_classes'])\n    model.to(CONFIG['device']);\n    criterion = nn.CrossEntropyLoss()\n    optimizer = AdamW(model.parameters())\n    prev_loss = 1e6\n    train_dataset = UBCDataset(df_train, transforms=data_transforms[\"train\"])\n    train_loader = DataLoader(train_dataset, batch_size=CONFIG['batch_size'], \n                          num_workers=2, shuffle=True, pin_memory=True)\n    test_dataset = UBCDataset(df_valid, transforms=data_transforms[\"valid\"])\n    test_loader = DataLoader(test_dataset, batch_size=CONFIG['batch_size'], \n                          num_workers=2, shuffle=True, pin_memory=True)\n    for epoch in range(CONFIG[\"num_epochs\"]):\n        model.train()  # Set the model to training mode\n\n        running_loss = 0.0\n        correct = 0\n        total = 0\n        bar = tqdm(enumerate(train_loader), total=len(train_loader))\n        device = CONFIG['device']\n        for step, data in bar:\n            inputs, labels = data['image'] , data['label']\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            # Zero the parameter gradients\n            optimizer.zero_grad()\n\n            # Forward pass\n            outputs = model(inputs)\n            # Calculate the loss\n            loss = criterion(outputs, labels.view(-1))\n\n            # Backpropagation and optimization\n            loss.backward()\n            optimizer.step()\n\n            # Update the running loss\n            running_loss += loss.item()\n            _, predicted = torch.max(outputs, 1) # Class with the highest probability is our prediction\n            predicted = predicted.reshape(-1,1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n        # Print the average loss for this epoch\n        print(f'Epoch [{epoch+1}/{CONFIG[\"num_epochs\"]}] Training Loss: {running_loss/len(train_loader)}')\n        print(f'Training Accuracy: {(correct/total) * 100:.2f}%')\n        if running_loss < prev_loss : \n            prev_loss = running_loss \n            torch.save(model.state_dict(), 'best-model_{}.pt'.format(fold))\n        # Evaluation\n        model.eval() # Set the model to evaluation mode\n        correct = 0\n        total = 0\n        total_valid_loss=0\n        with torch.no_grad(): # No need to track the gradients\n            bar = tqdm(enumerate(test_loader), total=len(test_loader))\n            for step, data in bar:\n                inputs, labels = data['image'] , data['label']\n                inputs, labels = inputs.to(device), labels.to(device)\n                output = model(inputs) # Get the model's predictions\n                val_loss = criterion(output, labels.view(-1)) # Calculate the loss\n                total_valid_loss += val_loss.item()\n                _, predicted = torch.max(output, 1) # Class with the highest probability is our prediction\n                predicted = predicted.reshape(-1,1)\n                total += labels.size(0)\n                correct += (predicted == labels).sum().item() # Count correct predictions\n        accuracy = correct / total\n        total_valid_loss = total_valid_loss / len(test_loader)\n        print('Validation Loss : {}'.format(total_valid_loss))\n        print(f'Validation Accuracy: {accuracy * 100:.2f}%')","metadata":{"execution":{"iopub.status.busy":"2023-10-19T09:43:23.35243Z","iopub.execute_input":"2023-10-19T09:43:23.352989Z","iopub.status.idle":"2023-10-19T09:59:12.131532Z","shell.execute_reply.started":"2023-10-19T09:43:23.352954Z","shell.execute_reply":"2023-10-19T09:59:12.130556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}