{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":18647,"databundleVersionId":1126921,"sourceType":"competition"},{"sourceId":187731,"sourceType":"datasetVersion","datasetId":80814},{"sourceId":1153338,"sourceType":"datasetVersion","datasetId":636745}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# You might need to install einops and timm libraries before running in console\n#pip install einops timm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T00:11:10.039878Z","iopub.execute_input":"2024-12-07T00:11:10.040714Z","iopub.status.idle":"2024-12-07T00:11:10.044608Z","shell.execute_reply.started":"2024-12-07T00:11:10.040679Z","shell.execute_reply":"2024-12-07T00:11:10.043724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from git import Repo  # pip install gitpython\nrepo_dir = '/kaggle/working/AttentionMIL'\nRepo.clone_from(\"https://github.com/AIMLab-UBC/EC-p53abnlike-AIclassifier.git\", repo_dir)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:34.243139Z","iopub.execute_input":"2024-12-07T14:43:34.243518Z","iopub.status.idle":"2024-12-07T14:43:35.092518Z","shell.execute_reply.started":"2024-12-07T14:43:34.243483Z","shell.execute_reply":"2024-12-07T14:43:35.091696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sys\nsys.path.append('/kaggle/working/AttentionMIL') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:36.239083Z","iopub.execute_input":"2024-12-07T14:43:36.239443Z","iopub.status.idle":"2024-12-07T14:43:36.243658Z","shell.execute_reply.started":"2024-12-07T14:43:36.239412Z","shell.execute_reply":"2024-12-07T14:43:36.242664Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(sys.path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:37.035595Z","iopub.execute_input":"2024-12-07T14:43:37.036382Z","iopub.status.idle":"2024-12-07T14:43:37.040485Z","shell.execute_reply.started":"2024-12-07T14:43:37.036348Z","shell.execute_reply":"2024-12-07T14:43:37.039609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pickle\nimport enum\n\nimport torch\nimport numpy as np\nfrom PIL import Image\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\n\nfrom utils.utils import print_color\nfrom augmentation.ToTensor import ToTensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:37.876761Z","iopub.execute_input":"2024-12-07T14:43:37.877446Z","iopub.status.idle":"2024-12-07T14:43:40.950477Z","shell.execute_reply.started":"2024-12-07T14:43:37.877413Z","shell.execute_reply":"2024-12-07T14:43:40.949513Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\n\n# Load CSV files (if applicable)\ntrain_labels = pd.read_csv('/kaggle/input/prostate-cancer-grade-assessment/train.csv')\ntest_images_dir = '/kaggle/input/prostate-cancer-grade-assessment/test_images'\ntrain_images_dir = '/kaggle/input/prostate-cancer-grade-assessment/train_images'\n\n# Explore the dataset\nprint(train_labels.head())\nprint(f\"Train images path: {train_images_dir}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:40.952069Z","iopub.execute_input":"2024-12-07T14:43:40.95291Z","iopub.status.idle":"2024-12-07T14:43:41.260664Z","shell.execute_reply.started":"2024-12-07T14:43:40.952865Z","shell.execute_reply":"2024-12-07T14:43:41.259704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndef load_chunks(chunk_file_location, chunk_ids, patch_pattern):\n    if chunk_file_location.endswith('.csv'):\n        data = pd.read_csv(chunk_file_location)\n        patch_paths = []\n        for _, row in data.iterrows():\n            patch_paths.append({\n                \"image_id\": row[\"image_id\"],\n                \"isup_grade\": row[\"isup_grade\"],\n                # Add more fields as necessary\n            })\n        return patch_paths\n    else:\n        # Default JSON handling\n        with open(chunk_file_location) as f:\n            data = json.load(f)\n            chunks = data['chunks']\n            patch_paths = [chunks[i] for i in chunk_ids]\n        return patch_paths","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:42.187478Z","iopub.execute_input":"2024-12-07T14:43:42.187807Z","iopub.status.idle":"2024-12-07T14:43:42.193555Z","shell.execute_reply.started":"2024-12-07T14:43:42.18778Z","shell.execute_reply":"2024-12-07T14:43:42.192498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import openslide\nfrom torchvision import transforms\nimport torch\nfrom torch.utils.data import Dataset\nimport os\n\nclass ProstateCancerOpenSlideDataset(Dataset):\n    def __init__(self, csv_file, img_dir, patch_size=224, level=0, transform=None):\n        \"\"\"\n        Args:\n            csv_file (str): Path to the CSV file with image IDs and labels.\n            img_dir (str): Directory with the whole-slide images.\n            patch_size (int): Size of the patches to extract.\n            level (int): Level of the downsampled image to use.\n            transform (callable, optional): Optional transform to apply to patches.\n        \"\"\"\n        self.data = pd.read_csv(csv_file)\n        self.img_dir = img_dir\n        self.patch_size = patch_size\n        self.level = level\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        img_path = os.path.join(self.img_dir, f\"{row['image_id']}.tiff\")\n        label = row['isup_grade']  # Update based on the target column\n\n        # Open the WSI\n        slide = openslide.OpenSlide(img_path)\n\n        # Get the dimensions of the selected level\n        dimensions = slide.level_dimensions[self.level]\n\n        # Randomly select a region to extract as a patch\n        # For a more structured approach, use pre-determined coordinates\n        x = torch.randint(0, dimensions[0] - self.patch_size, (1,)).item()\n        y = torch.randint(0, dimensions[1] - self.patch_size, (1,)).item()\n\n        # Extract the patch\n        patch = slide.read_region((x, y), self.level, (self.patch_size, self.patch_size)).convert('RGB')\n\n        # Apply transformations\n        if self.transform:\n            patch = self.transform(patch)\n\n        return patch, label\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:43.037077Z","iopub.execute_input":"2024-12-07T14:43:43.037761Z","iopub.status.idle":"2024-12-07T14:43:43.104602Z","shell.execute_reply.started":"2024-12-07T14:43:43.037726Z","shell.execute_reply":"2024-12-07T14:43:43.103734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:43.77112Z","iopub.execute_input":"2024-12-07T14:43:43.771881Z","iopub.status.idle":"2024-12-07T14:43:43.776887Z","shell.execute_reply.started":"2024-12-07T14:43:43.771843Z","shell.execute_reply":"2024-12-07T14:43:43.775989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_data_set(cfg, chunk_id, state=None, slide_id=None, training_set=False):\n    \"\"\"\n    Custom logic to create dataset from Kaggle data.\n    \"\"\"\n    csv_data = pd.read_csv(cfg[\"chunk_file_location\"])\n    patch_dataset = ProstateCancerDataset(\n        csv_file=cfg[\"chunk_file_location\"],\n        img_dir=\"/kaggle/input/prostate-cancer-grade-assessment/train_images\",\n        transform=transform\n    )\n    return patch_dataset, csv_data[\"isup_grade\"].tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:44.455468Z","iopub.execute_input":"2024-12-07T14:43:44.455794Z","iopub.status.idle":"2024-12-07T14:43:44.461802Z","shell.execute_reply.started":"2024-12-07T14:43:44.455767Z","shell.execute_reply":"2024-12-07T14:43:44.461119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from utils.utils import my_collate\nfrom torch.utils.data import DataLoader\n\nclass Data_Loader(object):\n    def __init__(self,\n                 cfg: dict,\n                 state: str):\n        self.cfg = cfg\n        self.state = state\n        chunk, training_set = self.handle_chunk()\n\n    def handle_chunk(self) -> dict | bool:\n        if self.state == 'train':\n            chunk = self.cfg[\"training_chunks\"]\n            training_set = True\n        elif self.state == 'validation':\n            chunk = self.cfg[\"validation_chunks\"]\n            training_set = False\n        elif self.state == 'test':\n            chunk = self.cfg[\"test_chunks\"]\n            training_set = False\n        elif self.state == 'external':\n            chunk = self.cfg[\"external_chunks\"]\n            training_set = False\n        else:\n            raise ValueError(f'{state} should be either train, validation, test or external!')\n        return chunk, training_set\n\n    def run(self):\n        raise NotImplementedError()\n\n\nclass RepresentationDataset(Data_Loader):\n    def __init__(self,\n                 cfg: dict,\n                 state: str,\n                 slide_id: int) -> None:\n        self.cfg = cfg\n        self.state = state\n        chunk, _ = self.handle_chunk()\n        self.patch_dataset, _ = create_data_set(cfg, chunk, state=state,\n                                                slide_id=slide_id)\n\n    def run(self):\n        batch_size = self.cfg[\"eval_batch_size\"]\n        return DataLoader(self.patch_dataset, batch_size=batch_size,\n                      shuffle=False, pin_memory=True,\n                      num_workers=self.cfg[\"num_patch_workers\"])\n\nclass Dataset(Data_Loader):\n    def __init__(self,\n                 cfg: dict,\n                 state: str = 'train') -> None:\n        self.cfg = cfg\n        self.state = state\n        chunk, training_set = self.handle_chunk()\n        self.patch_dataset, self.labels = create_data_set(cfg, chunk, state=state,\n                                                          training_set=training_set)\n\n    def run(self):\n        batch_size = self.cfg[\"batch_size\"] if self.state=='train' else \\\n                     self.cfg[\"eval_batch_size\"]\n\n        return DataLoader(self.patch_dataset, batch_size=batch_size,\n                      shuffle=True, pin_memory=True, collate_fn=my_collate,\n                      num_workers=self.cfg[\"num_patch_workers\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:45.063005Z","iopub.execute_input":"2024-12-07T14:43:45.063372Z","iopub.status.idle":"2024-12-07T14:43:45.074231Z","shell.execute_reply.started":"2024-12-07T14:43:45.06331Z","shell.execute_reply":"2024-12-07T14:43:45.073166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"csv_file = '/kaggle/input/prostate-cancer-grade-assessment/train.csv'\nimg_dir = '/kaggle/input/prostate-cancer-grade-assessment/train_images'\n\n# Initialize dataset\ndataset = ProstateCancerOpenSlideDataset(\n    csv_file=csv_file,\n    img_dir=img_dir,\n    patch_size=224,\n    level=0,  # Use the highest resolution\n    transform=transform\n)\n\n# Initialize DataLoader\ndataloader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)\n\n# # Iterate through the DataLoader\n# for patches, labels in dataloader:\n#     print(patches.shape, labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:45.674228Z","iopub.execute_input":"2024-12-07T14:43:45.675036Z","iopub.status.idle":"2024-12-07T14:43:45.695141Z","shell.execute_reply.started":"2024-12-07T14:43:45.675002Z","shell.execute_reply":"2024-12-07T14:43:45.694462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import timm\nimport torch\nimport numpy as np\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.models as models\nfrom einops import rearrange, repeat\nfrom torchvision.models import ResNet50_Weights\nfrom torchvision.models import efficientnet_b7, EfficientNet_B7_Weights\n\n####################################################\nout_channel = {'alexnet': 256, 'vgg16': 512, 'vgg19': 512, 'vgg16_bn': 512, 'vgg19_bn': 512,\n               'resnet18': 512, 'resnet34': 512, 'resnet50': 2048, 'resnext50_32x4d': 2048,\n               'resnext101_32x8d': 2048, 'mobilenet_v2': 1280, 'mobilenet_v3_small': 576,\n               'mobilenet_v3_large': 960 ,'mnasnet1_3': 1280, 'shufflenet_v2_x1_5': 1024,\n               'squeezenet1_1': 512, 'efficientnet-b0': 1280, 'efficientnet-l2': 5504,\n               'efficientnet-b1': 1280, 'efficientnet-b2': 1408, 'efficientnet-b3': 1536,\n               'efficientnet-b4': 1792, 'efficientnet-b5': 2048, 'efficientnet-b6': 2304,\n               'efficientnet-b7': 2560, 'efficientnet-b8': 2816, 'vit_deit_small_patch16_224': 384}\n\nfeature_map = {'alexnet': -2, 'vgg16': -2,  'vgg19': -2, 'vgg16_bn': -2,  'vgg19_bn': -2,\n               'resnet18': -2, 'resnet34': -2, 'resnet50': -2, 'resnext50_32x4d': -2,\n               'resnext101_32x8d': -2, 'mobilenet_v2': 0, 'mobilenet_v3_large': -2,\n               'mobilenet_v3_small': -2, 'mnasnet1_3': 0, 'shufflenet_v2_x1_5': -1,\n               'squeezenet1_1': 0, 'vit_deit_small_patch16_224': 'inf'}\n####################################################\n\nclass VanillaModel(nn.Module):\n    def __init__(self,\n                 backbone: str) -> None:\n        super(VanillaModel, self).__init__()\n\n        self.backbone  = backbone\n        # Vision Transformer\n        if 'vit' in self.backbone:\n            model = timm.create_model(self.backbone, pretrained=False, num_classes=0)\n            self.feature_extract = model\n        else:\n            model = getattr(models, self.backbone)\n            if self.backbone == \"resnet50\":\n                model = model(weights=ResNet50_Weights.IMAGENET1K_V1)  # Pretrained weights\n            elif self.backbone == \"efficientnet_b7\":\n                weights = EfficientNet_B7_Weights.IMAGENET1K_V1  # Pretrained weights\n                model = efficientnet_b7(weights=weights)\n            else:\n                model = model(weights=None)  # No pretrained weights for other models\n            # Seperate feature and classifier layers\n            self.feature_extract = nn.Sequential(*list(model.children())[0]) if feature_map[self.backbone]==0 \\\n                                   else nn.Sequential(*list(model.children())[:feature_map[self.backbone]])\n\n    def forward(self,\n                x: torch.Tensor) -> torch.Tensor:\n        feature = self.feature_extract(x)\n        feature = F.adaptive_avg_pool2d(feature, 1)\n        out     = torch.flatten(feature, 1)\n        return out\n\nclass VarMIL(nn.Module):\n    \"\"\"\n    Our modified implementation of https://arxiv.org/abs/2107.09405\n    \"\"\"\n    def __init__(self,\n                 cfg: dict) -> None:\n        super().__init__()\n        dim = 128\n        self.device     = 'cuda' if torch.cuda.is_available() else 'cpu'\n        self.attention  = nn.Sequential(nn.Linear(out_channel[cfg['backbone']], dim),\n                                       nn.Tanh(),\n                                       nn.Linear(dim, 1))\n        self.classifier = nn.Sequential(nn.Linear(2*out_channel[cfg['backbone']], dim),\n                                       nn.ReLU(),\n                                       nn.Linear(dim, cfg['num_classes']))\n\n    def forward(self,\n                x: torch.Tensor) -> torch.Tensor | torch.Tensor:\n        \"\"\"\n        x   (input)            : B (batch size) x K (nb_patch) x out_channel\n        A   (attention weights): B (batch size) x K (nb_patch) x 1\n        M   (weighted mean)    : B (batch size) x out_channel\n        S   (std)              : B (batch size) x K (nb_patch) x out_channel\n        V   (weighted variance): B (batch size) x out_channel\n        nb_patch (nb of patch) : B (batch size)\n        M_V (concate M and V)  : B (batch size) x 2*out_channel\n        out (final output)     : B (batch size) x num_classes\n        \"\"\"\n        b, k, c = x.shape\n        A = self.attention(x)\n        A = A.masked_fill((x == 0).all(dim=2).reshape(A.shape), -9e15) # filter padded rows\n        A = F.softmax(A, dim=1)                                        # softmax over K\n        M = torch.einsum('b k d, b k o -> b o', A, x)                  # d is 1 here\n        S = torch.pow(x-M.reshape(b,1,c), 2)\n        V = torch.einsum('b k d, b k o -> b o', A, S)\n        nb_patch = (torch.tensor(k).expand(b)).to(self.device)\n        nb_patch = nb_patch - torch.sum((x == 0).all(dim=2), dim=1)    # filter padded rows\n        nb_patch = nb_patch / (nb_patch - 1)                           # I / I-1\n        nb_patch = torch.nan_to_num(nb_patch, posinf=1)                # for cases, when we have only 1 patch (inf)\n        V = V * nb_patch[:, None]                                      # broadcasting\n        M_V = torch.cat((M, V), dim=1)\n        out = self.classifier(M_V)\n        return A, out\n\n\n\nclass Attention(nn.Module):\n    def __init__(self, cfg: dict) -> None:\n        super().__init__()\n        self.feature_extractor = VanillaModel(cfg['backbone'])  # Feature extractor\n        if cfg['model'] == 'VarMIL':\n            self.model = VarMIL(cfg)\n        else:\n            raise NotImplementedError()\n\n    def trainable_parameters(self) -> None:\n        params = sum(p.numel() for p in self.model.parameters() if p.requires_grad)\n        print(f'Total trainable parameters are {params}.')\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor | torch.Tensor:\n        \"\"\"\n        x (input): B (batch size) x 3 x H x W\n        \"\"\"\n        # Extract features for each patch\n        b, c, h, w = x.shape\n        features = self.feature_extractor(x)  # Shape: B x out_channel\n        features = features.unsqueeze(1)     # Add a pseudo-patch dimension: B x 1 x out_channel\n\n        # Pass features to VarMIL\n        attention, out = self.model(features)\n        return attention, out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:46.486872Z","iopub.execute_input":"2024-12-07T14:43:46.487536Z","iopub.status.idle":"2024-12-07T14:43:47.820457Z","shell.execute_reply.started":"2024-12-07T14:43:46.487501Z","shell.execute_reply":"2024-12-07T14:43:47.819813Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\n\n# Define device based on GPU availability\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\ncfg = {\n    'backbone': 'resnet50',  # Replace with your desired backbone\n    'model': 'VarMIL',\n    'num_classes': 6         # Adjust based on your dataset\n}\n\n# Initialize the model\nmodel = Attention(cfg)\nmodel = model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:47.822097Z","iopub.execute_input":"2024-12-07T14:43:47.822452Z","iopub.status.idle":"2024-12-07T14:43:49.307347Z","shell.execute_reply.started":"2024-12-07T14:43:47.822414Z","shell.execute_reply":"2024-12-07T14:43:49.306621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import random_split, DataLoader\n\n# Define the lengths for train and validation splits\ntrain_size = int(0.8 * len(dataset))  # 80% for training\nval_size = len(dataset) - train_size  # Remaining 20% for validation\n\n# Split the dataset\ntrain_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n\n# Create DataLoaders for both subsets\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:49.308932Z","iopub.execute_input":"2024-12-07T14:43:49.309576Z","iopub.status.idle":"2024-12-07T14:43:49.320223Z","shell.execute_reply.started":"2024-12-07T14:43:49.309534Z","shell.execute_reply":"2024-12-07T14:43:49.319525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(dataset.data['isup_grade'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:50.199246Z","iopub.execute_input":"2024-12-07T14:43:50.199613Z","iopub.status.idle":"2024-12-07T14:43:50.210562Z","shell.execute_reply.started":"2024-12-07T14:43:50.199584Z","shell.execute_reply":"2024-12-07T14:43:50.209558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.utils.class_weight import compute_class_weight\n\nclass_weights = compute_class_weight(\n    class_weight='balanced',\n    classes=np.unique(dataset.data['isup_grade']),\n    y=dataset.data['isup_grade']\n)\nclass_weights = torch.tensor(class_weights, dtype=torch.float).to(device)\n\ncriterion = nn.CrossEntropyLoss(weight=class_weights)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:50.837638Z","iopub.execute_input":"2024-12-07T14:43:50.837975Z","iopub.status.idle":"2024-12-07T14:43:50.848317Z","shell.execute_reply.started":"2024-12-07T14:43:50.837946Z","shell.execute_reply":"2024-12-07T14:43:50.847481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter\nfrom sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score\n\n# Initialize SummaryWriter\nwriter = SummaryWriter(log_dir='runs/training_logs')  # Set a directory for logs","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:43:51.481453Z","iopub.execute_input":"2024-12-07T14:43:51.481814Z","iopub.status.idle":"2024-12-07T14:44:01.900227Z","shell.execute_reply.started":"2024-12-07T14:43:51.481784Z","shell.execute_reply":"2024-12-07T14:44:01.899554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = torch.optim.Adam(model.parameters(), lr=5e-5, weight_decay=4e-5)\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n\n\nfor epoch in range(10):  # Adjust number of epochs\n    model.train()\n    running_loss = 0.0\n    correct_predictions = 0\n    total_samples = 0\n    all_labels = []\n    all_predictions = []\n\n    for i, (inputs, labels) in enumerate(train_loader):\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n\n        # Forward pass\n        _, outputs = model(inputs)\n        loss = criterion(outputs, labels)\n\n        # Backward pass and optimization\n        loss.backward()\n        optimizer.step()\n\n        # Track loss and accuracy\n        running_loss += loss.item()\n        _, predictions = torch.max(outputs, 1)\n        correct_predictions += (predictions == labels).sum().item()\n        total_samples += labels.size(0)\n\n        # Store labels and predictions for metrics\n        all_labels.extend(labels.cpu().numpy())\n        all_predictions.extend(predictions.cpu().numpy())\n\n        # Log training loss per step\n        step = epoch * len(train_loader) + i\n        writer.add_scalar('Loss/train', loss.item(), step)\n\n    # Compute metrics\n    all_labels = np.array(all_labels)\n    all_predictions = np.array(all_predictions)\n    train_accuracy = 100 * correct_predictions / total_samples\n    train_precision = precision_score(all_labels, all_predictions, average='weighted', zero_division=0)\n    train_recall = recall_score(all_labels, all_predictions, average='weighted', zero_division=0)\n    train_f1 = f1_score(all_labels, all_predictions, average='weighted', zero_division=0)\n\n    # Log metrics\n    writer.add_scalar('Accuracy/train', train_accuracy, epoch)\n    writer.add_scalar('Precision/train', train_precision, epoch)\n    writer.add_scalar('Recall/train', train_recall, epoch)\n    writer.add_scalar('F1/train', train_f1, epoch)\n\n    print(f\"Epoch {epoch+1}, Training Loss: {running_loss / len(train_loader):.4f}, \"\n          f\"Accuracy: {train_accuracy:.2f}%, Precision: {train_precision:.4f}, \"\n          f\"Recall: {train_recall:.4f}, F1 Score: {train_f1:.4f}\")\n\n    # Validation phase\n    model.eval()\n    val_loss = 0.0\n    val_correct_predictions = 0\n    val_total_samples = 0\n    val_all_labels = []\n    val_all_predictions = []\n\n    with torch.no_grad():\n        for inputs, labels in val_loader:\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            # Forward pass\n            _, outputs = model(inputs)\n            loss = criterion(outputs, labels)\n\n            # Track validation loss and accuracy\n            val_loss += loss.item()\n            _, predictions = torch.max(outputs, 1)\n            val_correct_predictions += (predictions == labels).sum().item()\n            val_total_samples += labels.size(0)\n\n            # Store labels and predictions for metrics\n            val_all_labels.extend(labels.cpu().numpy())\n            val_all_predictions.extend(predictions.cpu().numpy())\n\n    # Compute validation metrics\n    val_all_labels = np.array(val_all_labels)\n    val_all_predictions = np.array(val_all_predictions)\n    val_accuracy = 100 * val_correct_predictions / val_total_samples\n    val_precision = precision_score(val_all_labels, val_all_predictions, average='weighted', zero_division=0)\n    val_recall = recall_score(val_all_labels, val_all_predictions, average='weighted', zero_division=0)\n    val_f1 = f1_score(val_all_labels, val_all_predictions, average='weighted', zero_division=0)\n\n    # AUC requires probabilities, not class predictions\n    # try:\n    #     val_auc = roc_auc_score(val_all_labels, outputs.softmax(dim=1).cpu().numpy(), multi_class='ovr')\n    # except ValueError:\n    #     val_auc = None  # Handle cases where AUC calculation is invalid\n\n    # Log validation metrics\n    writer.add_scalar('Accuracy/val', val_accuracy, epoch)\n    writer.add_scalar('Precision/val', val_precision, epoch)\n    writer.add_scalar('Recall/val', val_recall, epoch)\n    writer.add_scalar('F1/val', val_f1, epoch)\n    # if val_auc is not None:\n    #     writer.add_scalar('AUC/val', val_auc, epoch)\n\n    print(f\"Epoch {epoch+1}, Validation Loss: {val_loss / len(val_loader):.4f}, \"\n          f\"Accuracy: {val_accuracy:.2f}%, Precision: {val_precision:.4f}, \"\n          f\"Recall: {val_recall:.4f}, F1 Score: {val_f1:.4f}\")\n\n    # Step the scheduler\n    scheduler.step()\n    current_lr = scheduler.get_last_lr()[0]\n    writer.add_scalar('Learning Rate', current_lr, epoch)\n    print(f\"Epoch {epoch+1}, Learning Rate: {current_lr:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T14:44:01.901671Z","iopub.execute_input":"2024-12-07T14:44:01.902172Z","iopub.status.idle":"2024-12-07T16:20:01.972964Z","shell.execute_reply.started":"2024-12-07T14:44:01.902144Z","shell.execute_reply":"2024-12-07T16:20:01.971924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"writer.close()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-07T16:20:01.974201Z","iopub.execute_input":"2024-12-07T16:20:01.974529Z","iopub.status.idle":"2024-12-07T16:20:01.979368Z","shell.execute_reply.started":"2024-12-07T16:20:01.974499Z","shell.execute_reply":"2024-12-07T16:20:01.978383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}