{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":45867},{"sourceType":"datasetVersion","sourceId":6774553},{"sourceType":"datasetVersion","sourceId":18017557},{"sourceType":"datasetVersion","sourceId":19251079}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Feature extraction","metadata":{"id":"2pswp07zptWS"}},{"cell_type":"markdown","source":"### Imports","metadata":{"id":"YVz-Lx7Ep8k8"}},{"cell_type":"code","source":"!yes | dpkg -i --force-depends /kaggle/input/datasets/jirkaborovec/pyvips-python-and-deb-package-gpu/linux_packages/archives/*.deb\n!dpkg -i --force-depends /kaggle/input/datasets/i12tttt/libopenslide0-3-4-1-dfsg-5build1-amd64/libopenslide0_3.4.1dfsg-5build1_amd64.deb\n!pip install pyvips -f /kaggle/input/datasets/jirkaborovec/pyvips-python-and-deb-package-gpu/python_packages/ --no-index","metadata":{"id":"CIQQgis2psHF","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:48:14.567069Z","iopub.execute_input":"2026-09-06T13:48:14.567343Z","iopub.status.idle":"2026-09-06T13:49:36.033985Z","shell.execute_reply.started":"2026-09-06T13:48:14.567318Z","shell.execute_reply":"2026-09-06T13:49:36.033072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport gc\nimport logging\nimport random\nfrom typing import Any\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport pyvips\nfrom PIL import ImageFile, Image\nimport torchvision.transforms.functional as TF\nimport torchvision.transforms as T\n\nimport torch\n\nfrom timm.models.vision_transformer import VisionTransformer","metadata":{"id":"otUfWzDMp9uK","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:49:36.036108Z","iopub.execute_input":"2026-09-06T13:49:36.03642Z","iopub.status.idle":"2026-09-06T13:49:49.39482Z","shell.execute_reply.started":"2026-09-06T13:49:36.03639Z","shell.execute_reply":"2026-09-06T13:49:49.394014Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"logging.basicConfig(\n    level=logging.INFO,\n    format='%(asctime)s [%(levelname)s] %(message)s',\n    datefmt='%H:%M:%S',\n)\nlogger = logging.getLogger('ubc_ocean_feature_extraction')","metadata":{"id":"aZeiCKfhqGFo","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:49:49.395916Z","iopub.execute_input":"2026-09-06T13:49:49.396476Z","iopub.status.idle":"2026-09-06T13:49:49.401453Z","shell.execute_reply.started":"2026-09-06T13:49:49.396434Z","shell.execute_reply":"2026-09-06T13:49:49.400436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ImageFile.LOAD_TRUNCATED_IMAGES = True\nImage.MAX_IMAGE_PIXELS = 5_000_000_000\nos.environ['VIPS_CONCURRENCY'] = '4'\nos.environ['VIPS_DISC_THRESHOLD'] = '15gb'","metadata":{"id":"NuAJrrEXqH3m","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:49:49.403232Z","iopub.execute_input":"2026-09-06T13:49:49.403517Z","iopub.status.idle":"2026-09-06T13:49:49.431687Z","shell.execute_reply.started":"2026-09-06T13:49:49.403493Z","shell.execute_reply":"2026-09-06T13:49:49.430881Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Utils","metadata":{"id":"W2Ca-CQqqQz1"}},{"cell_type":"code","source":"def set_seed(seed: int) -> None:\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n        torch.backends.cudnn.deterministic = True\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\n\ndef ensure_dir(path: str) -> None:\n    if not os.path.exists(path):\n        os.makedirs(path)\n\ndef dir_size_bytes(path: str) -> int:\n    total = 0\n    for dirpath, _, filenames in os.walk(path):\n        for fname in filenames:\n            fpath = os.path.join(dirpath, fname)\n            total += os.path.getsize(fpath)\n    return total","metadata":{"id":"ih01TGBEqRVt","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:49:49.432943Z","iopub.execute_input":"2026-09-06T13:49:49.433394Z","iopub.status.idle":"2026-09-06T13:49:49.445229Z","shell.execute_reply.started":"2026-09-06T13:49:49.433366Z","shell.execute_reply":"2026-09-06T13:49:49.444112Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Config","metadata":{"id":"29C3D9GZqfwm"}},{"cell_type":"code","source":"class Config:\n    seed: int = 42\n\n    thumbnails_dir: str = '/kaggle/input/competitions/UBC-OCEAN/train_thumbnails'\n    images_dir: str = '/kaggle/input/competitions/UBC-OCEAN/train_images'\n    train_csv_path: str = '/kaggle/input/competitions/UBC-OCEAN/train.csv'\n\n    mode: str = 'p16' #'p8'\n\n    backbone_variants: dict[str, dict[str, Any]] = {\n        'p16': {\n            'patch_size': 16,\n            'checkpoint': '/kaggle/input/datasets/i12tttt/lunit-dino-vit-small/dino_vit_small_patch16_ep200.torch'\n        }\n    }\n\n    backbone_patch_size: int = backbone_variants[mode]['patch_size']\n    backbone_checkpoint: str = backbone_variants[mode]['checkpoint']\n\n    backbone_embed_dim: int = 384\n    backbone_num_heads: int = 6\n\n    backbone_batch_size: int = 512\n\n    use_amp: bool = True\n\n    device: torch.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n    patch_size: int = 512\n\n    tma_size_threshold: int = 5000\n    bg_dark_threshold: int = 20\n    bg_light_threshold: int = 235\n    bg_area_fraction: float = 0.7\n\n    wsi_size_medium_threshold: int = 40_000\n    wsi_size_large_threshold: int = 80_000\n    wsi_keep_ratio_small: float = 0.8\n    wsi_keep_ratio_medium: float = 0.6\n    wsi_keep_ratio_large: float = 0.5\n\n    norm_mean: list[float] = [0.70322989, 0.53606487, 0.66096631]\n    norm_std: list[float] = [0.21716536, 0.26081574, 0.20723464]\n    resize_to: tuple[int, int] = (224, 224)\n\n    aug_enabled: bool = True\n\n    rare_class_total_copies: dict[str, int] = {\n        'EC': 3, 'CC': 3, 'LGSC': 5, 'MC': 5\n    }\n\n    aug_color_jitter_brightness: float = 0.2\n    aug_color_jitter_contrast: float = 0.3\n    aug_color_jitter_saturation: float = 0.2\n\n    feature_dir: str = f'features_{mode}/'\n\n    augmented_csv_path: str = f'train_augmented_{mode}.csv'\n\n    start_row: int = 250\n    n_rows: int | None = 50\n\n    force_recompute: bool = False","metadata":{"id":"zCf8pInNqgrA","outputId":"b63a55ec-9731-4422-997e-beb25014c614","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:02.703358Z","iopub.execute_input":"2026-09-06T13:50:02.704175Z","iopub.status.idle":"2026-09-06T13:50:02.714181Z","shell.execute_reply.started":"2026-09-06T13:50:02.704141Z","shell.execute_reply":"2026-09-06T13:50:02.713368Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Tiling","metadata":{"id":"8zrG8DXKyEvP"}},{"cell_type":"code","source":"class WSITiler:\n    def __init__(self, config: Config, wsi_name: str, wsi: 'pyvips.Image', patch_size, is_tma: bool, mode: str='train'):\n        self.config = config\n        self.wsi_name = wsi_name\n        self.wsi = wsi\n        self.patch_size = patch_size\n        self.is_tma = is_tma\n        self.mode = mode\n\n        self.cor_list = self._build_cooridnate_list()\n\n    def _build_cooridnate_list(self) -> list[tuple[int, int]]:\n        thumbnail = self._load_thumbnail()\n        cor_list = self._candidate_coordinates(thumbnail)\n\n        if self.is_tma:\n            return cor_list\n\n        return self._subsample_by_wsi_size(cor_list)\n\n    def _load_thumbnail(self) -> 'pyvips.Image':\n        if self.is_tma:\n            return self.wsi\n\n        thumbnail_path = os.path.join(self.config.thumbnails_dir, self.wsi_name + '_thumbnail.png')\n\n        return pyvips.Image.new_from_file(thumbnail_path)\n\n    def _candidate_coordinates(self, thumbnail: 'pyvips.Image') -> list[tuple[int, int]]:\n        wsi_h, wsi_w = self.wsi.height, self.wsi.width\n        thu_h, thu_w = thumbnail.height, thumbnail.width\n        h_ratio, w_ratio = wsi_h / thu_h, wsi_w / thu_w\n\n        step_h = int(self.patch_size / h_ratio)\n        step_w = int(self.patch_size / w_ratio)\n\n        cor_list = []\n        for y in range(0, thu_h, step_h):\n            for x in range(0, thu_w, step_w):\n                if self._is_tissue_tile(thumbnail, x, y, step_w, step_h, thu_w, thu_h) or not cor_list:\n                    cor_list.append((int(x * w_ratio), int(y * h_ratio)))\n\n        return cor_list\n\n    def _is_tissue_tile(self, thumbnail: 'pyvips.Image', x: int, y: int, step_w: int, step_h: int, thu_w: int, thu_h: int) -> bool:\n        if self.is_tma:\n            return True\n\n        w = min(step_w, thu_w - x)\n        h = min(step_h, thu_h - y)\n\n        tile = thumbnail.crop(x, y, w, h).numpy()[...,: 3]\n        mean_intensity = np.mean(tile, axis=2)\n\n        black_bg = mean_intensity < self.config.bg_dark_threshold\n        tile[black_bg, :] = 255\n\n        white_bg = mean_intensity > self.config.bg_light_threshold\n\n        return np.sum(white_bg) < w * h * self.config.bg_area_fraction\n\n    def _subsample_by_wsi_size(self, cor_list: list[tuple[int, int]]) -> list[tuple[int, int]]:\n        if self.wsi.height < self.config.wsi_size_medium_threshold:\n            keep_ratio = self.config.wsi_keep_ratio_small\n        elif self.wsi.height < self.config.wsi_size_large_threshold:\n            keep_ratio = self.config.wsi_keep_ratio_medium\n        else:\n            keep_ratio = self.config.wsi_keep_ratio_large\n\n        random.shuffle(cor_list)\n        n_keep = max(int(len(cor_list) * keep_ratio), 1)\n        return cor_list[: n_keep]\n        ","metadata":{"id":"pASsUYqgq0_y","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:07.752299Z","iopub.execute_input":"2026-09-06T13:50:07.752835Z","iopub.status.idle":"2026-09-06T13:50:07.766008Z","shell.execute_reply.started":"2026-09-06T13:50:07.752802Z","shell.execute_reply":"2026-09-06T13:50:07.764846Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Backbone","metadata":{"id":"8FkaP-_S13_H"}},{"cell_type":"code","source":"def load_backbone(config: Config) -> VisionTransformer:\n    backbone = VisionTransformer(\n        patch_size=config.backbone_patch_size,\n        embed_dim=config.backbone_embed_dim,\n        num_heads=config.backbone_num_heads,\n        num_classes=0\n    )\n    state = torch.load(config.backbone_checkpoint, map_location='cpu')\n    backbone.load_state_dict(state)\n    backbone = backbone.to(config.device)\n    backbone.eval()\n\n    return backbone","metadata":{"id":"SC7Nze5n15J3","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:08.116036Z","iopub.execute_input":"2026-09-06T13:50:08.1171Z","iopub.status.idle":"2026-09-06T13:50:08.121988Z","shell.execute_reply.started":"2026-09-06T13:50:08.117065Z","shell.execute_reply":"2026-09-06T13:50:08.121193Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Crop tile","metadata":{"id":"EyxA6eUP4jc1"}},{"cell_type":"code","source":"def crop_tile(img: Image.Image, x: int, y: int, patch_size: int) -> Image.Image:\n    right = min(x + patch_size, img.width)\n    bottom = min(y + patch_size, img.height)\n    tile = img.crop((x, y, right, bottom))\n\n    if tile.size == (patch_size, patch_size):\n        return tile\n\n    padded = Image.new('RGB', (patch_size, patch_size), (0, 0, 0))\n    padded.paste(tile, (0, 0))\n    return padded","metadata":{"id":"WCXfA5ET4k7i","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:08.613099Z","iopub.execute_input":"2026-09-06T13:50:08.613811Z","iopub.status.idle":"2026-09-06T13:50:08.620777Z","shell.execute_reply.started":"2026-09-06T13:50:08.613778Z","shell.execute_reply":"2026-09-06T13:50:08.619219Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Augmentation","metadata":{"id":"JigWXeSe5SOl"}},{"cell_type":"code","source":"class TileAugmentor:\n    def __init__(self, config: Config) -> None:\n        self.color_jitter = T.ColorJitter(\n            brightness=config.aug_color_jitter_brightness,\n            contrast=config.aug_color_jitter_contrast,\n            saturation=config.aug_color_jitter_saturation\n        )\n\n    def __call__(self, tile: Image.Image) -> Image.Image:\n        tile = self._random_flip_rotate(tile)\n        tile = self.color_jitter(tile)\n        return tile\n\n    @staticmethod\n    def _random_flip_rotate(tile: Image.Image) -> Image.Image:\n        if random.random() < 0.5:\n            tile = TF.hflip(tile)\n\n        angle = random.choice([0, 90, 180, 270])\n        if angle:\n            tile = TF.rotate(tile, angle)\n\n        return tile","metadata":{"id":"QNLnZCs95TVh","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:09.226236Z","iopub.execute_input":"2026-09-06T13:50:09.226537Z","iopub.status.idle":"2026-09-06T13:50:09.233283Z","shell.execute_reply.started":"2026-09-06T13:50:09.226511Z","shell.execute_reply":"2026-09-06T13:50:09.23244Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Feature extraction for WSI","metadata":{"id":"eXYENSou7ldn"}},{"cell_type":"code","source":"@torch.no_grad()\ndef extract_features_for_image(config: Config, backbone: VisionTransformer, image_id: str, is_tma: bool, augmentor: 'TileAugmentor'=None) -> tuple[torch.Tensor, int]:\n    image_path = os.path.join(config.images_dir, f'{image_id}.png')\n\n    wsi = pyvips.Image.new_from_file(image_path)\n    img = Image.open(image_path)\n\n    tile_transform = T.Compose([\n        T.ToTensor(),\n        T.Resize(config.resize_to, antialias=True),\n        T.Normalize(mean=config.norm_mean, std=config.norm_std)\n    ])\n\n    try:\n        tiler = WSITiler(config, image_id, wsi, patch_size=config.patch_size, is_tma=is_tma, mode='train')\n        cor_list = tiler.cor_list\n        n_tiles = len(cor_list)\n\n        feature_chunks = []\n        for start in range(0, n_tiles, config.backbone_batch_size):\n            batch_coords = cor_list[start:start + config.backbone_batch_size]\n\n            tiles = [crop_tile(img, x, y, config.patch_size) for x, y in batch_coords]\n\n            if augmentor is not None:\n                tiles = [augmentor(t) for t in tiles]\n\n            batch_tensor = torch.stack([tile_transform(t) for t in tiles]).to(config.device)\n\n            if config.use_amp and config.device.type == 'cuda':\n                with torch.autocast(device_type='cuda'):\n                    feat = backbone(batch_tensor)\n            else:\n                feat = backbone(batch_tensor)\n\n            feature_chunks.append(feat.float().cpu())\n\n            del tiles, batch_tensor, feat\n        if feature_chunks:\n            features = torch.cat(feature_chunks, dim=0)\n        else:\n            features = torch.empty((0, config.backbone_embed_dim))\n\n        return features, n_tiles\n    finally:\n        del wsi, img\n","metadata":{"id":"OjHa8php7nrM","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:09.874565Z","iopub.execute_input":"2026-09-06T13:50:09.875577Z","iopub.status.idle":"2026-09-06T13:50:09.884807Z","shell.execute_reply.started":"2026-09-06T13:50:09.875541Z","shell.execute_reply":"2026-09-06T13:50:09.883787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def resolve_is_tma(config: Config, row: pd.Series) -> bool:\n    if 'is_tma' in row.index and pd.notna(row['is_tma']):\n        return bool(row['is_tma'])\n\n    image_path = os.path.join(config.images_dir, f\"{row['image_id']}.png\")\n    wsi = pyvips.Image.new_from_file(image_path)\n    is_tma = wsi.height < config.tma_size_threshold and wsi.width < config.tma_size_threshold\n    del wsi\n    return is_tma\n","metadata":{"id":"admKmmkC_MuV","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:10.222498Z","iopub.execute_input":"2026-09-06T13:50:10.22323Z","iopub.status.idle":"2026-09-06T13:50:10.228822Z","shell.execute_reply.started":"2026-09-06T13:50:10.223198Z","shell.execute_reply":"2026-09-06T13:50:10.227903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _n_copies_for_label(config: Config, row: pd.Series) -> int:\n    if not config.aug_enabled:\n        return 1\n\n    label = row.get('label')\n    return config.rare_class_total_copies.get(label, 1)\n\ndef _variant_image_id(image_id: str, aug_index: int) -> str:\n    return image_id if aug_index == 0 else f'{image_id}_aug{aug_index}'\n\ndef _variant_feature_path(config: Config, variant_id: str) -> str:\n    return os.path.join(config.feature_dir, f'{variant_id}.pt')\n\ndef _all_variants_cached(config: Config, image_id: str, n_copies: int) -> bool:\n    return all(\n        os.path.exists(_variant_feature_path(config, _variant_image_id(image_id, i)))\n        for i in range(n_copies)\n    )\n","metadata":{"id":"_yEXqML-AB6x","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:10.798337Z","iopub.execute_input":"2026-09-06T13:50:10.798665Z","iopub.status.idle":"2026-09-06T13:50:10.807273Z","shell.execute_reply.started":"2026-09-06T13:50:10.798638Z","shell.execute_reply":"2026-09-06T13:50:10.806128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _process_single_image(config: Config, backbone: VisionTransformer, row: pd.Series) -> dict[str, Any]:\n    image_id = str(row['image_id'])\n    is_tma = resolve_is_tma(config, row)\n    n_copies = _n_copies_for_label(config, row)\n\n    total_tiles = 0\n    n_new = 0\n    n_cached = 0\n    manifest_rows: list[dict] = []\n\n    for aug_idx in range(n_copies):\n        variant_id = _variant_image_id(image_id, aug_idx)\n        feature_path = _variant_feature_path(config, variant_id)\n\n        if not config.force_recompute and os.path.exists(feature_path):\n            n_cached += 1\n            manifest_rows.append({'image_id': variant_id, 'label': row['label'], 'is_tma': is_tma})\n            continue\n\n        augmentor = TileAugmentor(config) if aug_idx > 0 else None\n        features, n_tiles = extract_features_for_image(config, backbone, image_id, is_tma, augmentor)\n\n        torch.save(features, feature_path)\n        total_tiles += n_tiles\n        n_new += 1\n        manifest_rows.append({'image_id': variant_id, 'label': row['label'], 'is_tma': is_tma})\n\n    return {\n        'total_tiles': total_tiles,\n        'n_new': n_new,\n        'n_cached': n_cached,\n        'manifest_rows': manifest_rows,\n    }\n","metadata":{"id":"LuL2yPE5Br0S","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:11.045049Z","iopub.execute_input":"2026-09-06T13:50:11.045371Z","iopub.status.idle":"2026-09-06T13:50:11.05597Z","shell.execute_reply.started":"2026-09-06T13:50:11.045344Z","shell.execute_reply":"2026-09-06T13:50:11.054547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def run_tiling_and_feature_extraction(config: Config) -> dict[str, Any]:\n    set_seed(config.seed)\n    ensure_dir(config.feature_dir)\n\n    logger.info('Device: %s', config.device)\n    logger.info('mode: %s (backbone_patch_size=%s)', config.mode, config.backbone_patch_size)\n\n    train_csv = pd.read_csv(config.train_csv_path)\n\n    total_rows = len(train_csv)\n    start_row = max(config.start_row, 0)\n    end_row = total_rows if config.n_rows is None else min(start_row + config.n_rows, total_rows)\n    train_csv = train_csv.iloc[start_row:end_row].reset_index(drop=True)\n\n    logger.info(\n        'Processing rows [%s:%s) out of %s total rows in %s',\n        start_row, end_row, total_rows, config.train_csv_path,\n    )\n\n    logger.info('Backbone loading')\n    backbone = load_backbone(config)\n\n    n_success = 0\n    n_failed = 0\n    total_tiles = 0\n    total_new_copies = 0\n    total_cached_copies = 0\n    failed_ids: list[str] = []\n\n    manifest_path = config.augmented_csv_path\n    write_header = not os.path.exists(manifest_path)\n\n    for _, row in tqdm(train_csv.iterrows(), total=len(train_csv), desc='Feature extraction'):\n        image_id = str(row['image_id'])\n        try:\n            result = _process_single_image(config, backbone, row)\n            total_tiles += result['total_tiles']\n            total_new_copies += result['n_new']\n            total_cached_copies += result['n_cached']\n            n_success += 1\n\n            if result['manifest_rows']:\n                pd.DataFrame(\n                    result['manifest_rows'], columns=['image_id', 'label', 'is_tma']\n                ).to_csv(manifest_path, mode='a', header=write_header, index=False)\n                write_header = False\n        except Exception as e:\n            n_failed += 1\n            failed_ids.append(image_id)\n            logger.error('image_id=%s: %s', image_id, e, exc_info=True)\n        finally:\n            gc.collect()\n            if config.device.type == 'cuda':\n                torch.cuda.empty_cache()\n\n    if os.path.exists(manifest_path):\n        manifest_df = pd.read_csv(manifest_path)\n        manifest_df = manifest_df.drop_duplicates(subset='image_id', keep='last').reset_index(drop=True)\n        manifest_df.to_csv(manifest_path, index=False)\n        manifest_len = len(manifest_df)\n    else:\n        manifest_len = 0\n\n    logger.info('csv is saved: %s (%s rows total across all sessions)', manifest_path, manifest_len)\n\n    return {\n        'start_row': start_row,\n        'end_row': end_row,\n        'total_rows_in_csv': total_rows,\n        'n_success': n_success,\n        'n_failed': n_failed,\n        'total_tiles_new': total_tiles,\n        'total_new_copies': total_new_copies,\n        'total_cached_copies': total_cached_copies,\n        'failed_ids': failed_ids,\n        'manifest_path': manifest_path,\n        'manifest_rows_total': manifest_len,\n    }\n","metadata":{"id":"VbmoAf6EBsMa","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:13.841825Z","iopub.execute_input":"2026-09-06T13:50:13.842731Z","iopub.status.idle":"2026-09-06T13:50:13.854Z","shell.execute_reply.started":"2026-09-06T13:50:13.8427Z","shell.execute_reply":"2026-09-06T13:50:13.853105Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Main","metadata":{"id":"T8BT0svkD7F9"}},{"cell_type":"code","source":"config = Config()\n\nlogger.info('mode=%s, backbone_patch_size=%s', config.mode, config.backbone_patch_size)\nlogger.info('backbone_checkpoint=%s', config.backbone_checkpoint)\nlogger.info('PATCH_SIZE=%s, backbone_batch_size=%s', config.patch_size, config.backbone_batch_size)\nlogger.info('feature_dir=%s', config.feature_dir)\nlogger.info('device=%s', config.device)\nlogger.info('start_row=%s, n_rows=%s, force_recompute=%s', config.start_row, config.n_rows, config.force_recompute)\nstats = run_tiling_and_feature_extraction(config)\n","metadata":{"id":"6KRvUTEeD8rt","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:50:14.37047Z","iopub.execute_input":"2026-09-06T13:50:14.370804Z","iopub.status.idle":"2026-09-06T13:59:40.274208Z","shell.execute_reply.started":"2026-09-06T13:50:14.370777Z","shell.execute_reply":"2026-09-06T13:59:40.273521Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Stats","metadata":{"id":"m6tq9eIZEF_0"}},{"cell_type":"code","source":"feature_dir_bytes = dir_size_bytes(config.feature_dir)\nn_pt_files = len([f for f in os.listdir(config.feature_dir) if f.endswith('.pt')])\n\nlogger.info('Row range processed this session: [%s:%s) out of %s', stats['start_row'], stats['end_row'], stats['total_rows_in_csv'])\nlogger.info('Succes processed images (this session): %s', stats['n_success'])\nlogger.info('Failed processed images (this session): %s', stats['n_failed'])\nif stats['failed_ids']:\n    logger.warning('failed image_id: %s', stats['failed_ids'])\nlogger.info('New tiles computed (this session, with augmented copies): %s', stats['total_tiles_new'])\nlogger.info('New .pt copies written (this session): %s', stats['total_new_copies'])\nlogger.info('Copies skipped via cache (this session): %s', stats['total_cached_copies'])\nlogger.info('Manifest rows total (all sessions so far): %s', stats['manifest_rows_total'])\nlogger.info('Saved .pt-files total (all sessions so far): %s', n_pt_files)\nlogger.info(\n    'Features folder size (%s): %s bytes (%.2f MB)',\n    config.feature_dir, feature_dir_bytes, feature_dir_bytes / (1024 ** 2),\n)\n\nif stats['end_row'] < stats['total_rows_in_csv']:\n    logger.info(\n        'Not finished yet: next session should set start_row=%s (n_rows unchanged or None).',\n        stats['end_row'],\n    )\n","metadata":{"id":"2U7_PrD_EHIk","trusted":true,"execution":{"iopub.status.busy":"2026-09-06T13:59:40.276059Z","iopub.execute_input":"2026-09-06T13:59:40.27675Z","iopub.status.idle":"2026-09-06T13:59:40.294403Z","shell.execute_reply.started":"2026-09-06T13:59:40.27672Z","shell.execute_reply":"2026-09-06T13:59:40.293501Z"}},"outputs":[],"execution_count":null}]}