{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":251539253,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install Dependencies","metadata":{"editable":false}},{"cell_type":"code","source":"# Installs the C library without showing output\n!apt-get update &> /dev/null && apt-get install -y libvips &> /dev/null\n# Installs the Python wrapper without showing output\n!pip install -q --no-cache-dir pyvips &> /dev/null","metadata":{"trusted":true,"editable":false,"execution":{"iopub.status.busy":"2025-07-26T20:48:52.193007Z","iopub.execute_input":"2025-07-26T20:48:52.193721Z","iopub.status.idle":"2025-07-26T20:49:49.920292Z","shell.execute_reply.started":"2025-07-26T20:48:52.193693Z","shell.execute_reply":"2025-07-26T20:49:49.918928Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Fast Patch Selection Algorithm","metadata":{"editable":false}},{"cell_type":"code","source":"import cv2\nimport numpy as np\n# from openslide import OpenSlide\nimport openslide\nfrom pathlib import Path\nimport glob\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom multiprocessing import Pool\nfrom tqdm import tqdm\nimport os\nimport pandas as pd\nimport random\nfrom sklearn.neighbors import KernelDensity\nfrom PIL import Image\n\ndef get_sampled_points_density_proportional_KDE(points, desired_sample_size):\n    num_points = len(points)\n    if num_points <= desired_sample_size:\n        return points\n\n    points_arr = np.array(points)\n    \n    # Fit KDE model to the points\n    kde = KernelDensity(bandwidth=0.1)  # You can adjust the bandwidth\n    kde.fit(points_arr)\n\n    # Generate samples from the KDE\n    samples = kde.sample(desired_sample_size)\n    final_sample = samples.tolist()\n\n    return final_sample\n\n\ndef RGB2HSD(X):\n    eps = np.finfo(float).eps\n    X[np.where(X==0.0)] = eps\n    \n    OD = -np.log(X / 1.0)\n    D  = np.mean(OD,3)\n    D[np.where(D==0.0)] = eps\n    \n    cx = OD[:,:,:,0] / (D) - 1.0\n    cy = (OD[:,:,:,1]-OD[:,:,:,2]) / (np.sqrt(3.0)*D)\n    \n    D = np.expand_dims(D,3)\n    cx = np.expand_dims(cx,3)\n    cy = np.expand_dims(cy,3)\n            \n    X_HSD = np.concatenate((D,cx,cy),3)\n    return X_HSD\n\n\ndef clean_thumbnail(thumbnail):\n    # thumbnail_arr = np.asarray(thumbnail)\n    \n    # wthumbnail = np.zeros_like(thumbnail_arr)\n    # wthumbnail[:, :, :] = thumbnail_arr[:, :, :]\n\n    # thumbnail_std = np.std(wthumbnail, axis=2)\n    # wthumbnail[thumbnail_std<5] = (np.ones((1,3), dtype=\"uint8\")*255)\n    # thumbnail_HSD = RGB2HSD( np.array([wthumbnail.astype('float32')/255.]) )[0]\n    # kernel = np.ones((30,30),np.float32)/900\n    # thumbnail_HSD_mean = cv2.filter2D(thumbnail_HSD[:,:,2],-1,kernel)\n    # wthumbnail[thumbnail_HSD_mean<0.05] = (np.ones((1,3),dtype=\"uint8\")*255)\n    # return wthumbnail\n    \n    # Change np.asarray to np.array to create a writable copy\n    thumbnail_arr = np.array(thumbnail)\n\n    # Add a grayscale threshold to remove non-white background\n    gray_thumbnail = cv2.cvtColor(thumbnail_arr, cv2.COLOR_RGB2GRAY)\n    _, grayscale_mask = cv2.threshold(gray_thumbnail, 220, 255, cv2.THRESH_BINARY)\n    thumbnail_arr[grayscale_mask == 255] = [255, 255, 255]\n\n\n    wthumbnail = np.zeros_like(thumbnail_arr)\n    wthumbnail[:, :, :] = thumbnail_arr[:, :, :]\n\n    thumbnail_std = np.std(wthumbnail, axis=2)\n    wthumbnail[thumbnail_std < 5] = (np.ones((1, 3), dtype=\"uint8\") * 255)\n    thumbnail_HSD = RGB2HSD(np.array([wthumbnail.astype('float32') / 255.]))[0]\n    kernel = np.ones((30, 30), np.float32) / 900\n    thumbnail_HSD_mean = cv2.filter2D(thumbnail_HSD[:, :, 2], -1, kernel)\n    wthumbnail[thumbnail_HSD_mean < 0.05] = (np.ones((1, 3), dtype=\"uint8\") * 255)\n    return wthumbnail\n\n                \ndef is_far_enough(new_point, existing_points, min_distance):\n    for point in existing_points:\n        if np.sqrt((new_point[0] - point[0])**2 + (new_point[1] - point[1])**2) < min_distance:\n            return False\n    return True\n\n\ndef get_patch_locations(tissue_mask, cthumbnail,  mask_hratio, mask_wratio, tissue_threshold, stride):\n    contours, mm = cv2.findContours(tissue_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n    image_with_contours = cthumbnail.copy()\n    cv2.drawContours(image_with_contours, contours, -1, (0, 255, 0), 2)  # Draw contours on the image\n    \n    image_with_rectangles = cthumbnail.copy()\n    \n    # Calculate the step size for the grid based on the stride\n    step_w = int(mask_wratio * stride)\n    step_h = int(mask_hratio * stride)\n    \n    patch_locations = []\n    for contour in contours:\n        x, y, w, h = cv2.boundingRect(contour)\n        # plot the rectangles on the image_with_rectangles\n        cv2.rectangle(image_with_rectangles, (x, y), (x + w, y + h), (0, 255, 0), 2)\n        \n        if w >= mask_wratio and h >= mask_hratio:\n            for i in range(x, x + w - mask_wratio, step_w):\n                for j in range(y, y + h - mask_hratio, step_h):\n                    tissue_patch = tissue_mask[j:j + mask_hratio, i:i + mask_wratio]\n                    # if np.sum(tissue_patch) / (mask_hratio ** 2) > tissue_threshold:\n                    tissue_magnitude = np.count_nonzero(tissue_patch)/tissue_patch.size\n                    if tissue_magnitude  >= tissue_threshold:\n                        patch_locations.append(((i, j),tissue_magnitude))\n\n    return patch_locations, image_with_contours, image_with_rectangles\n\ndef process_wsi(wsi_obj, wsi_path, thumbnail_path, is_tma, output_patch_size=1000, tissue_percent=0.9, returnSamples=30, stride=1):\n    wsi_name = Path(wsi_path).stem + \".svs\"\n\n    if is_tma:\n        thumbnail = Image.open(wsi_path)\n        objective_power = 40\n    else:\n        thumbnail = Image.open(thumbnail_path)\n        objective_power = 20\n    \n    cthumbnail = clean_thumbnail(thumbnail)\n    tissue_mask = ((cthumbnail.mean(axis=2) != 255) * 255).astype(np.uint8)\n    # Add morphological operations to clean the mask\n    kernel = np.ones((5,5),np.uint8)\n    tissue_mask = cv2.morphologyEx(tissue_mask, cv2.MORPH_CLOSE, kernel, iterations = 2)\n    tissue_mask = cv2.morphologyEx(tissue_mask, cv2.MORPH_OPEN, kernel, iterations = 2)\n    \n    # try:\n    #     objective_power = int(wsi_obj.properties['openslide.objective-power'])\n    # except:\n    #     objective_power = 20\n         \n    w, h = wsi_obj.dimensions\n    mask_hratio = int((tissue_mask.shape[0] / h) * output_patch_size)\n    mask_wratio = int((tissue_mask.shape[1] / w) * output_patch_size)\n    # Ensure the step size is at least 1 pixel\n    if mask_hratio == 0:\n        mask_hratio = 1\n    if mask_wratio == 0:\n        mask_wratio = 1\n    # print(f\"mask_hratio is {mask_hratio} and mask_wratio is {mask_wratio}\")\n    # estimate the mask patch size given the size of the WSI, the size of the mask, and the output patch size\n    mask_patch_size = int(output_patch_size / mask_wratio)\n    \n    Mask_to_WSI_ratioW = int(w / tissue_mask.shape[1])\n    Mask_to_WSI_ratioH = int(h / tissue_mask.shape[0])\n    \n    patch_locations, image_with_contours, image_with_rectangles = get_patch_locations(tissue_mask, cthumbnail, mask_hratio, mask_wratio, tissue_percent, stride)\n    # print(f\"initially generated {len(patch_locations)} patch locations\")\n    min_distance = mask_hratio * 0.5  # Minimum distance between points\n\n    filtered_patch_locations = []\n    for (x, y), _ in patch_locations:\n        if is_far_enough((x, y), filtered_patch_locations, min_distance):\n            filtered_patch_locations.append((x, y))\n\n    # print(f\"after is_far_enough there are {len(filtered_patch_locations)} patch locations\")\n    filtered_patch_locations = get_sampled_points_density_proportional_KDE(filtered_patch_locations, returnSamples)\n\n    scaled_patch_coordinates = []\n    for (x, y) in filtered_patch_locations:\n        scaled_patch_coordinates.append((int(x * Mask_to_WSI_ratioW), int(y * Mask_to_WSI_ratioH)))\n\n    return scaled_patch_coordinates\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T20:49:49.922758Z","iopub.execute_input":"2025-07-26T20:49:49.923221Z","iopub.status.idle":"2025-07-26T20:49:51.680526Z","shell.execute_reply.started":"2025-07-26T20:49:49.923191Z","shell.execute_reply":"2025-07-26T20:49:51.679513Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Import, Patch Selection, and Tensor Computation","metadata":{"editable":false}},{"cell_type":"code","source":"import os\nimport torch\nimport torchvision.transforms as T\nimport openslide\nimport pyvips\nimport gc\nfrom contextlib import contextmanager\n\nclass SlidePatchExtractor:\n    def __init__(self, image_id, patch_size=224, mode='train', tissue_threshold=0.9, label='HGSC'):\n        \n        self.image_id = image_id\n        self.patch_size = patch_size\n        self.mode = mode\n        self.transform = T.Compose([\n            T.ToTensor(),\n            T.Resize((self.patch_size, self.patch_size), antialias=True),\n            # T.Normalize(mean=[0.2585, 0.2556, 0.2506], std=[0.229, 0.224, 0.225])\n            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])\n        self.tissue_threshold = tissue_threshold\n        if label == 'HGSC':\n            self.num_patches = 100\n        elif label == 'EC':\n            self.num_patches = 180\n        elif label == 'CC':\n            self.num_patches = 220\n        elif label == 'LGSC':\n            self.num_patches = 470\n        elif label == 'MC':\n            self.num_patches = 480\n        else:\n            self.num_patches = 256\n\n        # self.train_transform = T.Compose([\n        #     T.RandomHorizontalFlip(p=0.5),\n        #     T.RandomVerticalFlip(p=0.5),\n        #     T.RandomRotation(degrees=45),\n        #     T.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n        #     T.ToTensor(),\n        #     T.Resize((224, 224), antialias=True),\n        #     T.Normalize(mean=[0.2585, 0.2556, 0.2506], std=[0.229, 0.224, 0.225])\n        # ])\n\n        # Define paths for the source WSI and its thumbnail\n        self.source_path = os.path.join('/kaggle/input/UBC-OCEAN', f'{self.mode}_images', self.image_id + '.png')\n        self.thumbnail_path = os.path.join('/kaggle/input/UBC-OCEAN', f'{self.mode}_thumbnails', self.image_id + '_thumbnail.png')\n        \n        try:\n            temp_slide = openslide.open_slide(self.source_path)\n            self.width, self.height = temp_slide.dimensions\n            self.is_tma = self.width < 5000 and self.height < 5000\n            standard_magnification = 20\n            self.objective_power = 40 if self.is_tma else 20\n            magnification_factor = self.objective_power / standard_magnification\n            self.extraction_patch_size = int(self.patch_size * magnification_factor)\n            \n            self.stride = 1 if self.is_tma else 4\n            \n            self.patch_locations = process_wsi(\n                wsi_obj=temp_slide,\n                wsi_path=self.source_path,\n                thumbnail_path=self.thumbnail_path,\n                is_tma=self.is_tma,\n                output_patch_size=patch_size,\n                tissue_percent=tissue_threshold,\n                returnSamples=self.num_patches,\n                stride=self.stride\n            )\n        except openslide.OpenSlideError as e:\n            print(f\"Could not open slide {self.source_path}: {e}\")\n            self.patch_locations = []\n            return\n        finally:\n            if 'temp_slide' in locals():\n                temp_slide.close()\n                del temp_slide\n            # gc.collect()\n    \n    def __len__(self):\n        \"\"\"Returns the number of patches found for this slide.\"\"\"\n        return len(self.patch_locations)\n    \n    @contextmanager\n    def _get_pyvips_slide(self):\n        \"\"\"Context manager for proper PyVIPS resource management\"\"\"\n        pyvips_slide = None\n        try:\n            pyvips_slide = pyvips.Image.new_from_file(self.source_path)\n            yield pyvips_slide\n        finally:\n            if pyvips_slide is not None:\n                del pyvips_slide\n                gc.collect()\n    \n    def get_all_patch_tensors(self):\n        \"\"\"\n        Extracts all patches from the slide and returns them as a stacked tensor.\n        \"\"\"\n        patch_tensors = []\n        flat_feature_size = 3 * self.patch_size * self.patch_size\n        if not self.patch_locations:\n            # If no patches were found, return an empty tensor with the correct shape\n            return torch.empty((0, flat_feature_size))\n\n        with self._get_pyvips_slide() as pyvips_slide:\n            for (x, y) in self.patch_locations:\n                try:\n                    # patch_image = self.slide.read_region(\n                    #     (x, y), 0, (self.extraction_patch_size, self.extraction_patch_size)\n                    # ).convert('RGB')\n                    patch_image = pyvips_slide.crop(x, y, self.patch_size, self.patch_size).numpy()[..., :3]\n                    patch_tensor = self.transform(patch_image)\n                    patch_tensors.append(patch_tensor)\n                except Exception as e:\n                    print(f\"Error reading patch at ({x},{y}) for slide {self.image_id}: {e}\")\n                    continue\n        \n        if not patch_tensors:\n            return torch.empty((0, flat_feature_size))\n            \n        # Stack all patch tensors into a single 4D tensor (num_patches, 3, H, W)\n        stacked_patches = torch.stack(patch_tensors)\n        # Flatten the patch dimensions (3, H, W) into a single vector for each patch\n        flattened_patches = torch.flatten(stacked_patches, start_dim=1)\n\n        # Clean up intermediate tensors\n        del patch_tensors, stacked_patches\n        gc.collect()\n        \n        return flattened_patches\n    \n    def get_patch(self, idx):\n        x, y = self.patch_locations[idx]\n        with self._get_pyvips_slide() as pyvips_slide:\n            patch_image = pyvips_slide.crop(x, y, self.patch_size, self.patch_size).numpy()[..., :3]\n            # patch_image = self.slide.read_region((x, y), 0, (self.extraction_patch_size, self.extraction_patch_size)).convert('RGB')\n            patch_tensor = self.transform(patch_image)\n        return patch_tensor, patch_image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T20:49:51.681662Z","iopub.execute_input":"2025-07-26T20:49:51.682074Z","iopub.status.idle":"2025-07-26T20:50:02.081126Z","shell.execute_reply.started":"2025-07-26T20:49:51.682052Z","shell.execute_reply":"2025-07-26T20:50:02.08019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport openslide\nfrom PIL import Image\nimport numpy as np\nfrom tqdm import tqdm\nimport gc\n\nclass UBCDataset(Dataset):\n    \"\"\"\n    The main Dataset class for loading slides and their labels.\n    \"\"\"\n    def __init__(self, dataframe, label_map, mode='train', patch_size=224, tissue_threshold=0.9):\n        self.df = dataframe\n        self.label_map = label_map\n        self.mode = mode\n        self.patch_size = patch_size\n        self.tissue_threshold = tissue_threshold\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_id = str(row['image_id'])\n        \n        string_label = row['label']\n        # Use the label map to convert the string label to an integer\n        int_label = self.label_map[string_label]\n        # Create the tensor from the integer\n        label = torch.tensor(int_label, dtype=torch.long)\n\n        extractor = SlidePatchExtractor(\n            image_id=image_id,\n            mode=self.mode,\n            patch_size=self.patch_size,\n            tissue_threshold=self.tissue_threshold,\n            label=string_label\n        )\n        \n        patch_tensors = extractor.get_all_patch_tensors()\n\n        # Clean up the extractor\n        del extractor\n        gc.collect()\n\n        return {\"patches\": patch_tensors, \"label\": label, \"image_id\": image_id}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-23T23:42:07.066597Z","iopub.execute_input":"2025-07-23T23:42:07.067172Z","iopub.status.idle":"2025-07-23T23:42:07.075353Z","shell.execute_reply.started":"2025-07-23T23:42:07.067138Z","shell.execute_reply":"2025-07-23T23:42:07.074384Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Collate Function for the DataLoader ---\ndef collate_fn(batch):\n    patches_list = [item['patches'] for item in batch]\n    labels = torch.stack([item['label'] for item in batch])\n    image_ids = [item['image_id'] for item in batch]\n\n    return {\"patches\": patches_list, \"labels\": labels, \"image_ids\": image_ids}","metadata":{"trusted":true,"editable":false,"execution":{"iopub.status.busy":"2025-07-23T23:42:10.649905Z","iopub.execute_input":"2025-07-23T23:42:10.650287Z","iopub.status.idle":"2025-07-23T23:42:10.65557Z","shell.execute_reply.started":"2025-07-23T23:42:10.650258Z","shell.execute_reply":"2025-07-23T23:42:10.654588Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main Training Block","metadata":{"editable":false}},{"cell_type":"code","source":"import psutil\nimport shutil\nimport pyvips\nimport gc\nimport os\n\ndef force_clear_vips_cache():\n    \"\"\"\n    More aggressively clears pyvips caches and the dedicated temp subdirectory.\n    \"\"\"\n    # 1. Tell pyvips to drop its operation cache\n    pyvips.cache_set_max(0)\n    \n    # 2. Call Python's garbage collector\n    gc.collect()\n\n    # 3. Define the path to your dedicated temp directory\n    vips_temp_dir = '/kaggle/temp/my_vips_temp/'\n    \n    # 4. Use shutil to safely remove and recreate only your subdirectory\n    if os.path.exists(vips_temp_dir):\n        shutil.rmtree(vips_temp_dir)\n    os.makedirs(vips_temp_dir) # Recreate it for the next batch\n\n    # 5. Reset the pyvips cache to a normal size\n    pyvips.cache_set_max(100)\n\ndef print_memory_usage(prefix=\"\"):\n    \"\"\"Prints current RAM usage of the system.\"\"\"\n    process = psutil.Process(os.getpid())\n    memory_info = process.memory_info()\n    ram_usage_gb = memory_info.rss / (1024 ** 3)  # Resident Set Size in GB\n    \n    # Get total system memory\n    total_memory_gb = psutil.virtual_memory().total / (1024 ** 3)\n    \n    # Get percentage usage\n    memory_percent = process.memory_percent()\n    \n    print(\n        f\"{prefix} Memory Usage: \"\n        f\"{ram_usage_gb:.2f} GB / {total_memory_gb:.2f} GB \"\n        f\"({memory_percent:.2f}%)\"\n    )\n\ndef get_dir_size(path='.'):\n    \"\"\"\n    Calculates the total size of all files in a directory and its subdirectories.\n    \"\"\"\n    total = 0\n    # Use a try-except block in case of permission errors\n    try:\n        with os.scandir(path) as it:\n            for entry in it:\n                if entry.is_file():\n                    total += entry.stat().st_size\n                elif entry.is_dir():\n                    total += get_dir_size(entry.path)\n    except FileNotFoundError:\n        return 0 # If directory doesn't exist, its size is 0\n    return total\n\ndef print_disk_usage(path, prefix=\"\"):\n    \"\"\"\n    Prints the size of the specified directory's contents and the overall\n    usage of the disk partition it resides on.\n    \"\"\"\n    try:\n        # --- Overall Partition Usage ---\n        total_partition, used_partition, free_partition = shutil.disk_usage(path)\n        \n        # --- Specific Directory Size ---\n        dir_contents_size = get_dir_size(path)\n\n        # --- Convert to GB for printing ---\n        total_partition_gb = total_partition / (1024**3)\n        used_partition_gb = used_partition / (1024**3)\n        dir_contents_gb = dir_contents_size / (1024**3)\n        \n        # --- THE FIX IS HERE ---\n        # Strip trailing slashes from the path before getting the basename\n        dir_name = os.path.basename(path.rstrip('/'))\n        \n        print(\n            f\"{prefix} Dir '{dir_name}' Size: {dir_contents_gb:.2f} GB. \"\n            f\"Total Partition Usage: {used_partition_gb:.2f} GB / {total_partition_gb:.2f} GB\"\n        )\n        \n    except FileNotFoundError:\n        # This can happen right after cleanup, so we handle it gracefully\n        print(f\"{prefix} Dir '{os.path.basename(path.rstrip('/'))}' Size: 0.00 GB. (Directory removed during cleanup)\")\n    except Exception as e:\n        print(f\"An error occurred while checking disk usage for {path}: {e}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport gzip\nimport time\n\n# Disable the Decompression Bomb check\nImage.MAX_IMAGE_PIXELS = None\n\n# Create a dedicated directory for pyvips temporary files\nvips_temp_dir = '/kaggle/temp/my_vips_temp/'\nos.makedirs(vips_temp_dir, exist_ok=True)\n\n# Tell pyvips to use this directory\nos.environ['VIPS_TMPDIR'] = vips_temp_dir\n\n# Hyperparameters\nbatch_size = 1\npatch_size = 224\nnum_patches = 256\ntissue_threshold = 0.9\n\nbase_path = '/kaggle/input/UBC-OCEAN'\ntrain_df = pd.read_csv(os.path.join(base_path, 'train.csv'))\n\n# Create a mapping from string labels to integers\nunique_labels = sorted(train_df['label'].unique())\nlabel_to_int = {label: i for i, label in enumerate(unique_labels)}\nint_to_label = {i: label for label, i in label_to_int.items()}\n\n# testing the code, only take the top 20 reocrds in training file\ntrain_df = train_df.head(50)\n\n# create the dataset using PyTorch Dataset\nubc_dataset = UBCDataset(\n    dataframe=train_df,\n    label_map=label_to_int,\n    patch_size=patch_size,\n    tissue_threshold=tissue_threshold,\n    num_patches=num_patches\n)\n\n# create the training data loader, potentially change shuffle and num_workers\ntrain_loader = DataLoader(\n    ubc_dataset,\n    batch_size=batch_size,\n    shuffle=False,\n    num_workers=0,\n    collate_fn=collate_fn\n)\n\n# all_patch_tensors = {}\n\nstart_time = time.perf_counter()\n\nfor i, batch in enumerate(tqdm(train_loader, desc=\"Processing Batches\")):\n    print(f\"\\n--- Batch {i+1} ---\")\n    \n    patches_list = batch['patches']\n    labels = batch['labels']\n    image_ids = batch['image_ids']\n    \n    print(f\"Number of slides in this batch: {len(patches_list)}\")\n    print(f\"Labels for this batch (as integers): {labels.numpy()}\")\n    print(f\"Image IDs for this batch: {image_ids}\")\n\n    for slide_idx in range(len(image_ids)):\n        slide_id = image_ids[slide_idx]\n        slide_patches = patches_list[slide_idx]\n        slide_label_int = labels[slide_idx].item()\n\n        # file_name = slide_id + '_patches.pt.gz'\n        # torch.save(slide_patches, file_name)\n        # with gzip.open(file_name, 'wb') as f:\n        #     torch.save(slide_patches, f)\n        \n        print(f\"  - Slide ID: {slide_id}, Label: {int_to_label[slide_label_int]} ({slide_label_int}), Patches: {slide_patches.shape[0]}\")\n        \n        # all_patch_tensors[slide_id] = slide_patches\n\n    # Clean up batch variables at the end of each iteration\n    del patches_list, labels, image_ids, batch\n    gc.collect()\n    print_memory_usage(f\"End of Batch {i+1}:  \")\n    print_disk_usage('/kaggle/working', prefix=f\"End of Batch {i+1}:\")\n    print_disk_usage('/kaggle/temp/my_vips_temp/', prefix=f\"End of Batch {i+1}:   \") # Check the pyvips temp dir\n    print_disk_usage('/tmp', prefix=f\"End of Batch {i+1}:\")\n    force_clear_vips_cache()\n    print(\"Cleanup complete.\")\n    print_disk_usage('/kaggle/temp/my_vips_temp/', prefix=\"IMMEDIATELY AFTER CLEANUP:\")\n    \n    # if (i + 1) % 3 == 0:\n    #     print(f\"\\\\nCleaning up temporary directory at batch {i+1}...\")\n    #     # The '!' runs a shell command to remove all files in the directory\n    #     !rm -rf /kaggle/temp/*\n    #     print(\"Cleanup complete.\")\n\nend_time = time.perf_counter()\nduration = end_time - start_time\nprint(f\"The code block took {duration:.4f} seconds to execute.\")\n# print(f\"\\n--- Finished processing. Total slides with stored tensors: {len(all_patch_tensors)} ---\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# !rm /kaggle/working/*.gz","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Store Patch Tensors","metadata":{"editable":false}},{"cell_type":"code","source":"# import gzip\n\n# # torch.save(slide_patches, 'slide_patches.pt')\n# with gzip.open('slide_patches.pt.gz', 'wb') as f:\n#     torch.save(slide_patches, f)","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import gzip\n# import torch\n\n# with gzip.open('/kaggle/input/ovarian-caner-subtype-classification/1020_patches.pt.gz', 'rb') as f:\n#     my_tensor = torch.load(f)\n\n# print(my_tensor.shape)","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from joblib import dump\n\n# dump(all_patch_tensors, 'all_patch_tensors.joblib')","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from joblib import load\n# import os\n\n# input_path = '/kaggle/input/ovarian-caner-subtype-classification'\n# file_path = os.path.join(input_path, 'all_patch_tensors.joblib')\n# new_dict = load(file_path)","metadata":{"trusted":true,"editable":false},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Test the data class","metadata":{"editable":false}},{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom PIL import Image\nimport matplotlib.pyplot as plt\n\n# Hyperparameters\n# num_patches = 256\ntissue_threshold = 0.9\npatch_size = 224\n\n# Disable the Decompression Bomb check\nImage.MAX_IMAGE_PIXELS = None\n\nbase_path = '/kaggle/input/UBC-OCEAN'\ntrain_labels_df = pd.read_csv(os.path.join(base_path, 'train.csv'))\n# image_id = str(train_labels_df.loc[2, 'image_id'])\nimage_id = str(38669)\nwsi = SlidePatchExtractor(image_id=image_id, patch_size=patch_size, tissue_threshold=tissue_threshold, label='CC')\nprint(f'The image is {wsi.width} width and {wsi.height} height')\n# HGSC 100, EC 180, CC 220, LGSC 470, MC 480\n# MC: 5456, 21445\n# LGSC: 31300, 57162\n# CC: 38669\n# EC: 39269\n# HGSC: 39425","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T20:53:53.419102Z","iopub.execute_input":"2025-07-26T20:53:53.419462Z","iopub.status.idle":"2025-07-26T20:53:55.788736Z","shell.execute_reply.started":"2025-07-26T20:53:53.419438Z","shell.execute_reply":"2025-07-26T20:53:55.787785Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualize patches for sanity check","metadata":{"editable":false}},{"cell_type":"code","source":"print(len(wsi.patch_locations))","metadata":{"trusted":true,"editable":false,"execution":{"iopub.status.busy":"2025-07-26T20:53:57.95981Z","iopub.execute_input":"2025-07-26T20:53:57.960153Z","iopub.status.idle":"2025-07-26T20:53:57.965645Z","shell.execute_reply.started":"2025-07-26T20:53:57.960128Z","shell.execute_reply":"2025-07-26T20:53:57.964639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"patch_tensors = wsi.get_all_patch_tensors()","metadata":{"trusted":true,"editable":false,"execution":{"iopub.status.busy":"2025-07-24T00:32:14.96451Z","iopub.execute_input":"2025-07-24T00:32:14.964959Z","iopub.status.idle":"2025-07-24T00:33:06.912064Z","shell.execute_reply.started":"2025-07-24T00:32:14.964928Z","shell.execute_reply":"2025-07-24T00:33:06.911068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(patch_tensors.shape)","metadata":{"trusted":true,"editable":false,"execution":{"iopub.status.busy":"2025-07-24T00:31:10.17778Z","iopub.execute_input":"2025-07-24T00:31:10.178118Z","iopub.status.idle":"2025-07-24T00:31:10.183894Z","shell.execute_reply.started":"2025-07-24T00:31:10.178089Z","shell.execute_reply":"2025-07-24T00:31:10.182759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\ndel wsi\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T00:29:27.769568Z","iopub.execute_input":"2025-07-24T00:29:27.770595Z","iopub.status.idle":"2025-07-24T00:29:27.799925Z","shell.execute_reply.started":"2025-07-24T00:29:27.770553Z","shell.execute_reply":"2025-07-24T00:29:27.797527Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import time\nimport matplotlib.pyplot as plt\n\nstart_time = time.perf_counter()\n\npatch_tensor, patch_image = wsi.get_patch(50)\n\nend_time = time.perf_counter()\n\nduration = end_time - start_time\nprint(f\"The code block took {duration:.4f} seconds to execute.\")\n# print(patch_tensor)\n\nplt.imshow(patch_image)\nplt.show()\n\n# The code block took 154.5610 seconds to execute. openslide 66\n\n# The code block took 105.2772 seconds to execute. pyvips 2666\n# The code block took 140.6164 seconds to execute. openslide 2666","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T00:27:31.361648Z","iopub.execute_input":"2025-07-24T00:27:31.361996Z","iopub.status.idle":"2025-07-24T00:28:25.070801Z","shell.execute_reply.started":"2025-07-24T00:27:31.361972Z","shell.execute_reply":"2025-07-24T00:28:25.069373Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom PIL import Image\n\n# --- 1. Define Thumbnail and Get Image Dimensions ---\n# Get the full resolution image dimensions (width, height)\noriginal_width = wsi.width\noriginal_height = wsi.height\n\npatch_coordinates = wsi.patch_locations\npatch_size = 224\n\n# --- 2. Generate the Thumbnail ---\n# The get_thumbnail function maintains the aspect ratio,\n# creating an image that fits within the given size.\nif wsi.is_tma:\n    thumbnail = Image.open(wsi.source_path)\nelse:\n    thumbnail = Image.open(wsi.thumbnail_path)\n# Get the actual size of the generated thumbnail\nthumb_width, thumb_height = thumbnail.size\n\n# --- 3. Calculate Scaling Factors ---\n# These factors will scale coordinates from the original image to the thumbnail\nwidth_scale = thumb_width / original_width\nheight_scale = thumb_height / original_height\n\n# --- 4. Visualize the Thumbnail and Patches ---\n# Create a figure and axes for plotting\nfig, ax = plt.subplots(figsize=(10, 10))\n\n# Display the thumbnail image\nax.imshow(thumbnail)\n\n# Loop through each patch coordinate to draw it on the thumbnail\nfor x, y in patch_coordinates:\n    # Scale the patch's top-left corner coordinates\n    scaled_x = x * width_scale\n    scaled_y = y * height_scale\n\n    # Scale the patch's dimensions\n    scaled_patch_width = patch_size * width_scale\n    scaled_patch_height = patch_size * height_scale\n\n    # Create a rectangle patch with a red edge and no fill\n    rect = patches.Rectangle(\n        (scaled_x, scaled_y),\n        scaled_patch_width,\n        scaled_patch_height,\n        linewidth=1,\n        edgecolor='r',  # Red color for the patch border\n        facecolor='none'  # No fill\n    )\n\n    # Add the rectangle to the plot\n    ax.add_patch(rect)\n\n# --- 5. Finalize and Show the Plot ---\nax.set_title(\"WSI Thumbnail with Selected Patches\")\nplt.axis('off')  # Hide the axes ticks and labels\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T20:54:01.22219Z","iopub.execute_input":"2025-07-26T20:54:01.222531Z","iopub.status.idle":"2025-07-26T20:54:02.656102Z","shell.execute_reply.started":"2025-07-26T20:54:01.22251Z","shell.execute_reply":"2025-07-26T20:54:02.655111Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Auxiliary","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nbase_path = '/kaggle/input/UBC-OCEAN'\ntrain_df = pd.read_csv(os.path.join(base_path, 'train.csv'))\n\n# plot image dimensions\nplt.figure(figsize=(10, 5)) \nplt.scatter(train_df['image_width'], train_df['image_height'], c=train_df['is_tma'], cmap='viridis')\nplt.colorbar(label='is_tma')\nplt.xlabel('Image Width')\nplt.ylabel('Image Height')\nplt.title('Image Dimensions')\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T01:19:13.348402Z","iopub.execute_input":"2025-07-25T01:19:13.348716Z","iopub.status.idle":"2025-07-25T01:19:14.237636Z","shell.execute_reply.started":"2025-07-25T01:19:13.348693Z","shell.execute_reply":"2025-07-25T01:19:14.236556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot label distribution\nlabels_count = train_df.label.value_counts().to_dict() \ncategories = labels_count.keys()\nvalues = labels_count.values() \nplt.bar(categories, values) \nplt.title('Label Distribution')\nplt.xlabel('Labels')\nplt.ylabel('Count') \nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T01:19:31.912546Z","iopub.execute_input":"2025-07-25T01:19:31.912864Z","iopub.status.idle":"2025-07-25T01:19:32.068383Z","shell.execute_reply.started":"2025-07-25T01:19:31.91284Z","shell.execute_reply":"2025-07-25T01:19:32.067374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot label distribution\nlabels_count = train_df.label.value_counts().to_dict()\ncategories = labels_count.keys()\nvalues = list(labels_count.values()) # Using list() to ensure it's a list\n\n# Calculate the total number of labels for percentage calculation\ntotal = sum(values)\n\n# Create the bar plot and get the bar container object\nbars = plt.bar(categories, values)\n\n# Set the title and labels for the plot\nplt.title('Label Distribution')\nplt.xlabel('Labels')\nplt.ylabel('Percentage')\n\n# Iterate over each bar to add the percentage text\nfor bar in bars:\n    # Get the height of the bar\n    height = bar.get_height()\n    # Add text on top of the bar\n    plt.text(\n        bar.get_x() + bar.get_width() / 2.0,  # X position (center of the bar)\n        height,                               # Y position (top of the bar)\n        f'{height / total:.1%}',              # The text to display (formatted as percentage)\n        ha='center',                          # Horizontal alignment\n        va='bottom'                           # Vertical alignment\n    )\n\n# Display the plot\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-25T01:24:31.343766Z","iopub.execute_input":"2025-07-25T01:24:31.34406Z","iopub.status.idle":"2025-07-25T01:24:31.591342Z","shell.execute_reply.started":"2025-07-25T01:24:31.34404Z","shell.execute_reply":"2025-07-25T01:24:31.590173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}