{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom joblib import load\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim import Adam, lr_scheduler\nfrom sklearn.metrics import confusion_matrix, classification_report, balanced_accuracy_score\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\nimport torch\nfrom torchvision import models\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport wandb\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-15T10:19:52.957376Z","iopub.execute_input":"2023-10-15T10:19:52.957739Z","iopub.status.idle":"2023-10-15T10:19:53.1499Z","shell.execute_reply.started":"2023-10-15T10:19:52.95771Z","shell.execute_reply":"2023-10-15T10:19:53.148856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nwandb_api_key = user_secrets.get_secret(\"wandb_api_key\")\n\nwandb.login(key=wandb_api_key)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:14:30.561017Z","iopub.execute_input":"2023-10-15T10:14:30.561508Z","iopub.status.idle":"2023-10-15T10:14:35.381975Z","shell.execute_reply.started":"2023-10-15T10:14:30.561479Z","shell.execute_reply":"2023-10-15T10:14:35.381017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"efficientnet = models.efficientnet_v2_s(weights='DEFAULT'); efficientnet.name = 'EfficientNet_V2_small'\nefficientnet_in_features = efficientnet.classifier[-1].in_features\nefficientnet.classifier = nn.Sequential(*list(efficientnet.classifier.children())[:-1], nn.Linear(efficientnet_in_features, 5))\n\n# Function to freeze all model parameters\ndef freeze_parameters(model):\n    for param in model.parameters():\n        param.requires_grad = False\n\n# Function to unfreeze specific parameters\ndef unfreeze_layers(model, num_layers):\n    \"\"\"\n    Unfreeze the specified number of layers in the model starting from the end.\n    \n    Args:\n        model (nn.Module): The PyTorch model.\n        num_layers (int): The number of layers to unfreeze.\n    \"\"\"\n    # Get the total number of layers in the model\n    total_layers = len(list(model.parameters()))\n    \n    # Calculate the starting index of the layers to unfreeze from the end\n    start_index = total_layers - num_layers\n    \n    # Unfreeze the specified layers\n    for param in list(model.parameters())[start_index:]:\n        param.requires_grad = True\n\n\n# Freeze all parameters of each model\nfreeze_parameters(efficientnet)\n\n# Unfreeze the last layer for each model\nunfreeze_layers(efficientnet, num_layers=2)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:02:10.137697Z","iopub.execute_input":"2023-10-15T12:02:10.13807Z","iopub.status.idle":"2023-10-15T12:02:10.559401Z","shell.execute_reply.started":"2023-10-15T12:02:10.138042Z","shell.execute_reply":"2023-10-15T12:02:10.558321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n        'N_EPOCHS': 20,\n        'BATCH_SIZE': 32,\n        'LR': 1e-3,\n        'NUM_WORKERS': 6,\n        'CUDA': torch.cuda.is_available(),\n        'DEVICE': torch.device('cuda' if torch.cuda.is_available() else 'cpu'),\n        'architecture': efficientnet.name,\n        \"min_lr\": 1e-6,\n        \"T_max\": 500,\n        \"weight_decay\":1e-4\n    }","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:12:50.160855Z","iopub.execute_input":"2023-10-15T10:12:50.161497Z","iopub.status.idle":"2023-10-15T10:12:50.732287Z","shell.execute_reply.started":"2023-10-15T10:12:50.161469Z","shell.execute_reply":"2023-10-15T10:12:50.731349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = load('/kaggle/input/mymodels/UBCO/label_encoder.joblib')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image tiling\n### Code provided by [pjmathematician](https://www.kaggle.com/pjmathematician/code?userId=9507963&sortBy=dateCreated&tab=profile)\n\n<br>\nOriginal images are sliced into 256 segments.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\ntrain['tile_path'] = train['image_id'].apply(lambda x: f\"train/256_{x}\")\nprint(train.head())","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:05:45.313243Z","iopub.execute_input":"2023-10-15T10:05:45.314056Z","iopub.status.idle":"2023-10-15T10:05:45.331213Z","shell.execute_reply.started":"2023-10-15T10:05:45.314021Z","shell.execute_reply":"2023-10-15T10:05:45.33032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_image_path(image_id:int):\n    if 4 <= image_id <= 15188:\n        path = \"/kaggle/input/ucbo-tiles-256-1\"\n    elif 15209 <= image_id <= 30515:\n        path = \"/kaggle/input/ucbo-tiles-256-2\"\n    elif 30539 <= image_id <= 38687:\n        path = \"/kaggle/input/ucbo-tiles-256-3\"\n    elif 38849 <= image_id <= 65300:\n        path = \"/kaggle/input/ucbo-tiles-256-4\"\n    elif 65371 <= image_id <= 65533:\n        path = \"/kaggle/input/ucbo-tiles-256-5\"\n    return os.path.join(path, \"256_\"+str(image_id))","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:05:45.352864Z","iopub.execute_input":"2023-10-15T10:05:45.353224Z","iopub.status.idle":"2023-10-15T10:05:45.360018Z","shell.execute_reply.started":"2023-10-15T10:05:45.353193Z","shell.execute_reply":"2023-10-15T10:05:45.359177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['tile_path'] = train['image_id'].apply(lambda x: get_image_path(x))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:05:45.361275Z","iopub.execute_input":"2023-10-15T10:05:45.362101Z","iopub.status.idle":"2023-10-15T10:05:45.376156Z","shell.execute_reply.started":"2023-10-15T10:05:45.362072Z","shell.execute_reply":"2023-10-15T10:05:45.375234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['path_exists'] = train['tile_path'].apply(lambda x: os.path.exists(x))\ntrain = train[train['path_exists'] == True]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:05:45.332749Z","iopub.execute_input":"2023-10-15T10:05:45.333093Z","iopub.status.idle":"2023-10-15T10:05:45.350858Z","shell.execute_reply.started":"2023-10-15T10:05:45.333063Z","shell.execute_reply":"2023-10-15T10:05:45.349838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Expand our dataframe\nWe will create a row for each tile","metadata":{}},{"cell_type":"code","source":"# Expand the DataFrame\nexpanded_rows = []\nfor _, row in train.iterrows():\n    image_files = [f for f in os.listdir(row['tile_path']) if f.endswith(('.png', '.jpg', '.jpeg'))]\n    for img_file in image_files:\n        new_row = row.copy()\n        new_row['tile_path'] = os.path.join(row['tile_path'], img_file)\n        expanded_rows.append(new_row)\n\nexpanded_df = pd.DataFrame(expanded_rows)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:05:45.377538Z","iopub.execute_input":"2023-10-15T10:05:45.377823Z","iopub.status.idle":"2023-10-15T10:08:05.22664Z","shell.execute_reply.started":"2023-10-15T10:05:45.377795Z","shell.execute_reply":"2023-10-15T10:08:05.225655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"expanded_df['label'] = expanded_df['label'].apply(lambda x: encoder.transform([x])[0])\nexpanded_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:08:31.694763Z","iopub.execute_input":"2023-10-15T10:08:31.695144Z","iopub.status.idle":"2023-10-15T10:08:31.702034Z","shell.execute_reply.started":"2023-10-15T10:08:31.695117Z","shell.execute_reply":"2023-10-15T10:08:31.700981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"execution":{"iopub.status.busy":"2023-10-12T09:08:04.025525Z","iopub.execute_input":"2023-10-12T09:08:04.025884Z","iopub.status.idle":"2023-10-12T09:08:04.320885Z","shell.execute_reply.started":"2023-10-12T09:08:04.025858Z","shell.execute_reply":"2023-10-12T09:08:04.319093Z"}}},{"cell_type":"code","source":"class UBCODataset(Dataset):\n    def __init__(self, df, transforms=None, inference=False):\n        self.df = df\n        self.transforms = transforms\n        self.inference = inference\n\n    def __getitem__(self, ndx):\n        img_path = self.df['tile_path'][ndx]\n        img = cv2.imread(img_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)    \n        if self.transforms:\n            img = self.transforms(img)\n\n        if self.inference:\n            return img\n        else:\n            label = self.df['label'][ndx]\n            return img, label\n\n    def __len__(self):\n        return self.df.shape[0]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:09:27.792824Z","iopub.execute_input":"2023-10-15T10:09:27.793483Z","iopub.status.idle":"2023-10-15T10:09:27.799569Z","shell.execute_reply.started":"2023-10-15T10:09:27.793454Z","shell.execute_reply":"2023-10-15T10:09:27.798478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(expanded_df, test_size=0.2, random_state=1, stratify=expanded_df['label'])\ntrain_df = train_df.reset_index(drop=True)\nval_df = val_df.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:09:29.743368Z","iopub.execute_input":"2023-10-15T10:09:29.744112Z","iopub.status.idle":"2023-10-15T10:09:29.957191Z","shell.execute_reply.started":"2023-10-15T10:09:29.744077Z","shell.execute_reply":"2023-10-15T10:09:29.956031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_counts = train_df['label'].value_counts().to_dict()\nclass_weights = {cls: (1 / count) for cls, count in class_counts.items()}\n\n# Assign each sample the weight of its class\nsample_weights = train_df['label'].map(class_weights).values\n\nsampler = torch.utils.data.sampler.WeightedRandomSampler(\n                                    weights=sample_weights,\n                                    num_samples=len(train_df),\n                                    replacement=True\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:09:29.962658Z","iopub.execute_input":"2023-10-15T10:09:29.964876Z","iopub.status.idle":"2023-10-15T10:09:29.992851Z","shell.execute_reply.started":"2023-10-15T10:09:29.964842Z","shell.execute_reply":"2023-10-15T10:09:29.991991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(10),\n    transforms.RandomResizedCrop(256, scale=(0.8, 1.0)),  # Random cropping\n    transforms.RandomGrayscale(p=0.1),  # Occasionally transform to grayscale\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nval_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Resize((256, 256), antialias=True),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:03:37.734232Z","iopub.execute_input":"2023-10-15T12:03:37.734578Z","iopub.status.idle":"2023-10-15T12:03:37.748862Z","shell.execute_reply.started":"2023-10-15T12:03:37.734553Z","shell.execute_reply":"2023-10-15T12:03:37.747547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = UBCODataset(train_df, transforms=train_transforms)\n\ntrain_dl = DataLoader(\n                train_ds,\n                batch_size=config['BATCH_SIZE'],\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['CUDA'],\n                sampler=sampler,\n                drop_last=True\n                    )\n\n\nval_ds = UBCODataset(val_df, transforms=val_transforms)\n\nval_dl = DataLoader(\n                val_ds,\n                batch_size=config['BATCH_SIZE'],\n                num_workers=config['NUM_WORKERS'],\n                pin_memory=config['CUDA'],\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:13:14.56933Z","iopub.execute_input":"2023-10-15T10:13:14.56998Z","iopub.status.idle":"2023-10-15T10:13:14.575313Z","shell.execute_reply.started":"2023-10-15T10:13:14.569922Z","shell.execute_reply":"2023-10-15T10:13:14.57414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Utility function to save model checkpoints\ndef save_checkpoint(model, optimizer, epoch, filename):\n    torch.save({\n        'epoch': epoch,\n        'model_state_dict': model.state_dict(),\n        'optimizer_state_dict': optimizer.state_dict(),\n    }, filename)\n\n\ndef train_one_epoch(model, train_dl, optimizer, loss_func):\n    model.train()\n    train_loss = 0.0\n    train_labels, train_predictions = [], []\n\n    for inputs, labels in train_dl:\n        inputs, labels = inputs.to(config['DEVICE']), labels.to(config['DEVICE'])\n        \n        optimizer.zero_grad()\n\n        output_logits = model(inputs)\n        loss = loss_func(output_logits, labels)\n\n        loss.backward()\n        optimizer.step()\n\n        output_probs = F.softmax(output_logits, dim=1)\n        predicted_labels = output_probs.argmax(dim=1)\n\n        # Accumulate loss and predictions\n        train_loss += loss.item()\n        train_labels.extend(labels.cpu().tolist())\n        train_predictions.extend(predicted_labels.cpu().tolist())\n\n        # Clear up memory\n        del inputs, labels, output_logits\n        torch.cuda.empty_cache()\n\n    avg_train_loss = train_loss / len(train_dl)\n    train_bal_acc = balanced_accuracy_score(train_labels, train_predictions)\n    \n    return avg_train_loss, train_bal_acc","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:14:47.88248Z","iopub.execute_input":"2023-10-15T10:14:47.883004Z","iopub.status.idle":"2023-10-15T10:14:47.89113Z","shell.execute_reply.started":"2023-10-15T10:14:47.882974Z","shell.execute_reply":"2023-10-15T10:14:47.890245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate_model(model, val_dl, loss_func):\n    model.eval()\n    val_loss = 0.0\n    val_labels, val_predictions = [], []\n\n    with torch.no_grad():\n        for inputs, labels in val_dl:\n            inputs, labels = inputs.to(config['DEVICE']), labels.to(config['DEVICE'])\n            \n            # Model inference\n            output_logits = model(inputs)\n            loss = loss_func(output_logits, labels)\n\n            output_prob = F.softmax(output_logits, dim=1)\n            predicted_label = output_prob.argmax(dim=1)\n\n            # Accumulate loss and predictions\n            val_loss += loss.item()\n            val_labels.extend(labels.cpu().tolist())\n            val_predictions.extend(predicted_label.cpu().tolist())\n\n            # Clear up memory\n            del inputs, labels, output_logits\n            torch.cuda.empty_cache()\n\n    avg_val_loss = val_loss / len(val_dl)\n    val_bal_acc = balanced_accuracy_score(val_labels, val_predictions)\n    \n    return avg_val_loss, val_bal_acc","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:14:48.184299Z","iopub.execute_input":"2023-10-15T10:14:48.184626Z","iopub.status.idle":"2023-10-15T10:14:48.191627Z","shell.execute_reply.started":"2023-10-15T10:14:48.1846Z","shell.execute_reply":"2023-10-15T10:14:48.190694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_model(model, config, optimizer=None ):\n    loss_func = nn.CrossEntropyLoss()\n    if optimizer == None:\n        optimizer = Adam(model.parameters(), lr=config['LR'], weight_decay=config['weight_decay'])\n    scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=config['T_max'],  eta_min=config['min_lr'])\n    model = model.to(config['DEVICE'])\n    \n    wandb.init(\n        project=\"UBCO-efficientnetv2\", \n        config={\n            \"learning_rate\": optimizer.param_groups[0]['lr'],\n            'batch_size': config['BATCH_SIZE'],\n            \"architecture\": config['architecture'],\n            \"epochs\": config['N_EPOCHS'],\n            \"device\": config['DEVICE'],\n            }\n    )\n    wandb.watch(model, criterion=loss_func, log=\"all\", log_freq=20)\n\n    best_val_loss = float('inf')\n    \n    for epoch_ndx in range(1, config['N_EPOCHS'] + 1):\n        print(f\"\\nEpoch {epoch_ndx}/{config['N_EPOCHS']}\")\n        print(\".\"*10)\n        \n        train_loss, train_bal_acc = train_one_epoch(model, train_dl, optimizer, loss_func)\n        val_loss, val_bal_acc = validate_model(model, val_dl, loss_func)\n        \n        # Update the learning rate\n        scheduler.step()\n        new_lr = scheduler.get_lr\n        # Log metrics\n        wandb.log({'epoch': epoch_ndx, 'train_loss': train_loss, 'train_bal_acc': train_bal_acc,\n                   'val_loss': val_loss, 'val_bal_acc': val_bal_acc, 'learning_rate':new_lr})\n\n        print(f\"Train Loss: {train_loss:.4f}, Balanced Acc: {train_bal_acc:.4f}\")\n        print(f\"Validation Loss: {val_loss:.4f}, Balanced Acc: {val_bal_acc:.4f}\")\n\n        # Save model checkpoint if current model has lowest validation loss\n        if val_loss < best_val_loss:\n            best_epoch_ndx = epoch_ndx\n            best_val_loss = val_loss\n            best_val_acc = val_bal_acc\n\n            save_checkpoint(model, optimizer, epoch_ndx, f\"models/checkpoint_{model.name}_epoch_{epoch_ndx}.pth\")\n        \n    # Save final model\n    torch.save(model.state_dict(), f'models/efficientnetv2/{model.name}_v1.pth')\n    wandb.save(f'models/{model.name}_v1.pth')\n    \n    dummy_input = torch.rand(1, 3, 256, 256).to(config['DEVICE'])\n    onnx_path = f\"models/{model.name}_v1.onnx\"\n    torch.onnx.export(model, dummy_input, onnx_path)\n    wandb.save(onnx_path)\n    \n    text = f'Run Complete.\\nBest Epoch: {best_epoch_ndx}, Validation Accuracy: {best_val_acc:.4f}\\n\\n{config}'\n    wandb.alert(title='Fitting Complete.', text=text)\n    wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:02:44.02766Z","iopub.execute_input":"2023-10-15T12:02:44.02809Z","iopub.status.idle":"2023-10-15T12:02:44.048902Z","shell.execute_reply.started":"2023-10-15T12:02:44.028055Z","shell.execute_reply":"2023-10-15T12:02:44.048116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fit_model(model=efficientnet, config=config)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:03:41.575757Z","iopub.execute_input":"2023-10-15T12:03:41.576107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}