{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13735059,"sourceType":"datasetVersion","datasetId":8739150}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport glob\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport functools\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns  # <-- This will now work\nfrom typing import List, Tuple, Optional\nimport gc\nimport shutil\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold  # <-- This will now work\nfrom sklearn.metrics import roc_auc_score\nimport pydicom\nimport polars as pl\n\n# The RSNA inference server (needed for submission)\ntry:\n    import kaggle_evaluation.rsna_inference_server as rsna_inference_server\nexcept ImportError:\n    print(\"Inference server not found (OK for training)\")\n\nwarnings.filterwarnings('ignore')\n\nprint(f\"Imports successful. Numpy version: {np.__version__}\")\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:10.151233Z","iopub.execute_input":"2025-11-15T05:55:10.151494Z","iopub.status.idle":"2025-11-15T05:55:27.393734Z","shell.execute_reply.started":"2025-11-15T05:55:10.151468Z","shell.execute_reply":"2025-11-15T05:55:27.392977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# -------------------------\n# 2. GLOBAL CONFIGURATION\n# -------------------------\nclass Config:\n    # --- IMPORTANT: Path to your PNG dataset ---\n    # This path comes from your screenshot\n    DATA_DIR = \"/kaggle/input/another1\" \n    \n    # --- Paths derived from DATA_DIR ---\n    CVT_PNG_DIR = os.path.join(DATA_DIR, \"cvt_png\")\n    SERIES_MAPPING_PATH = os.path.join(DATA_DIR, \"series_index_mapping.csv\")\n    LOCALIZERS_PATH = os.path.join(DATA_DIR, \"train_localizers_with_relative.csv\")\n    \n    # --- Path to original competition data ---\n    ORIGINAL_DATA_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection\"\n    TRAIN_CSV_PATH = os.path.join(ORIGINAL_DATA_DIR, \"train.csv\")\n    ORIGINAL_SERIES_DIR = os.path.join(ORIGINAL_DATA_DIR, \"series\")\n    \n    # --- Model Hyperparameters ---\n    NUM_FRAMES = 8\n    IMAGE_SIZE = 224\n    NUM_CLASSES = 14\n    BATCH_SIZE = 6\n    NUM_EPOCHS = 10\n    LEARNING_RATE = 5e-5\n    MODEL_NAME_BACKBONE = \"tf_efficientnetv2_s.in1k\"\n    \n    # --- Feature Flags ---\n    USE_METADATA = True\n    USE_WINDOWING = True\n    USE_3CHANNEL_INPUT = True\n    USE_IMPROVED_LOSS = True\n    USE_CLAHE = True\n    USE_STRONG_AUGMENTATION = True\n    \n    # --- Dataloader & System ---\n    NUM_WORKERS = 2\n    PIN_MEMORY = True\n    PREFETCH_FACTOR = 2\n    PERSISTENT_WORKERS = True\n    \n    # --- CV & Training Loop ---\n    NUM_FOLDS = 5 # Set to 1 if you want to train only one model\n    FOLD = 0 # We will only train FOLD 0 if TRAIN_MODEL is True\n    ACCUMULATION_STEPS = 5\n    EARLY_STOPPING_PATIENCE = 5\n    USE_GROUP_CV = True\n    CACHE_SIZE = 100\n    OUTPUT_DIR = \"/kaggle/working\"\n    MODEL_NAME = \"eightframe_efficientnetv2s\"\n\nconfig = Config()\n\n\n\n# -------------------------\n# 4. GLOBAL TARGETS\n# -------------------------\nTARGET_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery', \n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery', \n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present'\n]\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:27.395409Z","iopub.execute_input":"2025-11-15T05:55:27.395698Z","iopub.status.idle":"2025-11-15T05:55:27.402997Z","shell.execute_reply.started":"2025-11-15T05:55:27.395681Z","shell.execute_reply":"2025-11-15T05:55:27.402092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# -------------------------\n# 3. GLOBAL SEED & DEVICE\n# -------------------------\ndef set_seed(seed: int = 42, deterministic: bool = False):\n    \"\"\"Set all random seeds for reproducibility.\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    if deterministic:\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n    else:\n        torch.backends.cudnn.deterministic = False\n        torch.backends.cudnn.benchmark = True\nset_seed(42, deterministic=False)\n\n# Device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\nif torch.cuda.is_available():\n    try:\n        print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    except Exception:\n        pass\n    torch.cuda.empty_cache()\nelif TRAIN_MODEL:\n    print(\"WARNING: CUDA not available. Training will be VERY slow.\")\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:27.403874Z","iopub.execute_input":"2025-11-15T05:55:27.404446Z","iopub.status.idle":"2025-11-15T05:55:27.556461Z","shell.execute_reply.started":"2025-11-15T05:55:27.404419Z","shell.execute_reply":"2025-11-15T05:55:27.555753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n\t#-------------------------\n# 5. GLOBAL HELPER FUNCTIONS\n# -------------------------\ndef get_windowing_params(modality: str) -> Tuple[float, float]:\n    windows = {\n        'CT': (40, 80), 'CTA': (50, 350), 'MRA': (600, 1200),\n        'MRI': (40, 80), 'MR': (40, 80)\n    }\n    return windows.get(modality, (40, 80))\n\ndef apply_dicom_windowing(img: np.ndarray, window_center: float, window_width: float) -> np.ndarray:\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    img = (img - img_min) / (img_max - img_min + 1e-7)\n    return (img * 255).astype(np.uint8)\n\ndef apply_clahe_normalization(img: np.ndarray, modality: str) -> np.ndarray:\n    if not config.USE_CLAHE:\n        return img.astype(np.uint8)\n        \n    img = img.astype(np.uint8)\n    if modality in ['CTA', 'MRA']:\n        clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img)\n        img_clahe = cv2.convertScaleAbs(img_clahe, alpha=1.1, beta=5)\n    elif modality in ['MRI', 'MR']:\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img)\n        img_clahe = np.power(img_clahe / 255.0, 0.9) * 255\n        img_clahe = img_clahe.astype(np.uint8)\n    else:\n        clahe = cv2.createCLAHE(clipLimit=2.5, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img)\n    return img_clahe\n\ndef robust_normalization(volume: np.ndarray) -> np.ndarray:\n    p1, p99 = np.percentile(volume.flatten(), [1, 99])\n    volume_norm = np.clip(volume, p1, p99)\n    if p99 > p1:\n        volume_norm = (volume_norm - p1) / (p99 - p1 + 1e-7)\n    else:\n        volume_norm = np.zeros_like(volume_norm)\n    return (volume_norm * 255).astype(np.uint8)\n\ndef create_3channel_input_8frame(volume: np.ndarray) -> np.ndarray:\n    if len(volume) == 0:\n        return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n    \n    middle_slice = volume[len(volume) // 2]\n    mip = np.max(volume, axis=0)\n    std_proj = np.std(volume, axis=0).astype(np.float32)\n    \n    if std_proj.max() > std_proj.min():\n        p1, p99 = np.percentile(std_proj, [5, 95])\n        std_proj = np.clip(std_proj, p1, p99)\n        std_proj = ((std_proj - p1) / (p99 - p1 + 1e-7) * 255).astype(np.uint8)\n    else:\n        std_proj = np.zeros_like(std_proj, dtype=np.uint8)\n        \n    return np.stack([middle_slice, mip, std_proj], axis=-1)\n\ndef smart_8_frame_sampling(volume_paths: List[str]) -> List[str]:\n    n = len(volume_paths)\n    if n == 0:\n        return []\n    if n <= 8:\n        result = volume_paths[:]\n        while len(result) < 8:\n            result.extend(volume_paths[:8-len(result)])\n        return result[:8]\n\n    start_idx = max(0, int(n * 0.1))\n    available_frames = n - start_idx\n    step = max(1, available_frames // 8)\n    indices = []\n    current_idx = start_idx\n    \n    while len(indices) < 8 and current_idx < n:\n        indices.append(current_idx)\n        current_idx += step\n        \n    while len(indices) < 8:\n        remaining = [i for i in range(n) if i not in indices]\n        if remaining:\n            indices.append(remaining[len(indices) % len(remaining)])\n        else:\n            indices.append(indices[-1])\n            \n    return [volume_paths[i] for i in indices[:8]]\n\ndef resolve_dicom_path(dicom_entry, series_uid):\n    \"\"\" Robustly find DICOM path. \"\"\"\n    if dicom_entry and os.path.exists(dicom_entry):\n        return dicom_entry\n        \n    # Try relative to original series dir\n    series_dir = os.path.join(config.ORIGINAL_SERIES_DIR, series_uid)\n    if dicom_entry:\n        possible = os.path.join(series_dir, os.path.basename(dicom_entry))\n        if os.path.exists(possible):\n            return possible\n\n    # Try searching series dir\n    if os.path.exists(series_dir):\n        candidates = sorted(glob.glob(os.path.join(series_dir, \"*.dcm\")))\n        if candidates:\n            # Fallback: try to find by SOPInstanceUID if entry is just a filename\n            if dicom_entry:\n                sop_uid = os.path.splitext(os.path.basename(dicom_entry))[0]\n                for cand_path in candidates:\n                    if sop_uid in cand_path:\n                        return cand_path\n            return candidates[0] # Return first if no match\n\n    raise FileNotFoundError(f\"No DICOM file found for series {series_uid}. Tried entry='{dicom_entry}'\")\n\n\n\n\n\n\n\n\n\n# -------------------------\n# 6. GLOBAL MODEL & LOSS DEFINITION\n# -------------------------\nclass ImprovedMultiFrameModel(nn.Module):\n    def __init__(self, num_frames=8, num_classes=14, pretrained=True):\n        super(ImprovedMultiFrameModel, self).__init__()\n        self.num_frames = num_frames\n        self.num_classes = num_classes\n        self.use_metadata = config.USE_METADATA\n        print(f\"Loading backbone: {config.MODEL_NAME_BACKBONE}\")\n        self.backbone = timm.create_model(\n            config.MODEL_NAME_BACKBONE,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        self.feature_dim = self.backbone.num_features\n        print(f\"Backbone {config.MODEL_NAME_BACKBONE}: {self.feature_dim} features\")\n        \n        if self.use_metadata:\n            self.meta_fc = nn.Sequential(\n                nn.Linear(2, 16),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(16, 32),\n                nn.ReLU()\n            )\n            classifier_input_dim = self.feature_dim + 32\n        else:\n            classifier_input_dim = self.feature_dim\n            \n        self.classifier = nn.Sequential(\n            nn.Linear(classifier_input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, num_classes)\n        )\n\n    def forward(self, x, meta=None):\n        features = self.backbone(x)\n        if self.use_metadata and meta is not None:\n            meta_features = self.meta_fc(meta)\n            features = torch.cat([features, meta_features], dim=1)\n        output = self.classifier(features)\n        return output\n\nclass FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n    def forward(self, inputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-bce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * bce_loss\n        return focal_loss.mean()\n\nclass ImprovedLoss(nn.Module):\n    def __init__(self, aneurysm_weight=3.0, focal_weight=0.3):\n        super(ImprovedLoss, self).__init__()\n        self.aneurysm_weight = aneurysm_weight\n        self.focal_weight = focal_weight\n        # Create weights on the correct device\n        weights = torch.ones(config.NUM_CLASSES)\n        weights[-1] = aneurysm_weight\n        self.register_buffer('weights', weights)\n        self.focal_loss = FocalLoss(alpha=1, gamma=2)\n        \n    def forward(self, outputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction='none')\n        weighted_bce = (bce_loss * self.weights).mean()\n        focal_loss_val = self.focal_loss(outputs, targets)\n        return (1 - self.focal_weight) * weighted_bce + self.focal_weight * focal_loss_val\n\n\t\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:27.557195Z","iopub.execute_input":"2025-11-15T05:55:27.557431Z","iopub.status.idle":"2025-11-15T05:55:27.583019Z","shell.execute_reply.started":"2025-11-15T05:55:27.557412Z","shell.execute_reply":"2025-11-15T05:55:27.582361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------------------------------------------\n# ------------------------- PHASE 1: TRAINING -------------------------\n# -------------------------------------------------------------------\n\nprint(\"=\"*40)\nprint(\"STARTING TRAINING MODE (Internet ON)\")\nprint(\"=\"*40)\n\n# -------------------------\n# 7. TRAINING: DATASET CLASS\n# -------------------------\ndef create_frame_paths_8frame_structured(train_df, series_mapping_df):\n    \"\"\"Return dict: series_uid -> {'paths': [...], 'is_dummy': bool}\"\"\"\n    frame_paths = {}\n    print(\"Creating 8-frame optimized structured paths from PNG dataset...\")\n    \n    # Pre-build a lookup for all PNGs\n    all_png_files = {}\n    print(f\"Scanning PNG directory: {config.CVT_PNG_DIR}\")\n    if not os.path.exists(config.CVT_PNG_DIR):\n         print(f\"WARNING: PNG Directory not found at {config.CVT_PNG_DIR}\")\n         print(\"The dataset will fall back to reading DICOMs, which will be much slower.\")\n    \n    for root, _, files in os.walk(config.CVT_PNG_DIR):\n        for file in files:\n            if file.endswith(\".png\"):\n                # Path: .../cvt_png/DISEASE_NAME/SERIES_UID/FRAME.png\n                try:\n                    series_uid = os.path.basename(os.path.dirname(root))\n                    if series_uid not in all_png_files:\n                        all_png_files[series_uid] = []\n                    all_png_files[series_uid].append(os.path.join(root, file))\n                except:\n                    pass\n    print(f\"Found PNGs for {len(all_png_files)} series.\")\n\n    for series_uid in tqdm(train_df['SeriesInstanceUID'].unique(), desc=\"Processing series\"):\n        series_data = series_mapping_df[series_mapping_df['SeriesInstanceUID'] == series_uid]\n        if series_data.empty:\n            frame_paths[series_uid] = {'paths': [], 'is_dummy': True}\n            continue\n\n        # Try to find pre-converted PNGs\n        found_paths = []\n        if series_uid in all_png_files:\n            # We found PNGs. Sort them numerically by filename (e.g., 0001.png, 0002.png)\n            # Use set() to remove duplicates if a series is in multiple disease folders\n            png_files = sorted(list(set(all_png_files[series_uid])), key=lambda x: os.path.basename(x))\n            if png_files:\n                found_paths = png_files\n        \n        # Fallback: Use DICOMs (marked as 'dummy' PNG path)\n        if not found_paths:\n            dicom_dir = os.path.join(config.ORIGINAL_SERIES_DIR, series_uid)\n            if os.path.exists(dicom_dir):\n                num_frames = len(series_data)\n                # Create dummy paths that signal DICOM loading\n                found_paths = [f\"dummy_dicom_path_{i:04d}.dcm\" for i in range(num_frames)]\n\n        if found_paths:\n            sampled = smart_8_frame_sampling(found_paths)\n            is_dummy = any(p.startswith('dummy_dicom_path') for p in sampled)\n            frame_paths[series_uid] = {'paths': sampled, 'is_dummy': is_dummy}\n        else:\n            frame_paths[series_uid] = {'paths': [], 'is_dummy': True}\n            \n    return frame_paths\n\n\nclass EightFrameDataset(Dataset):\n    def __init__(self, df, frame_paths_dict, series_mapping_df, num_frames=8, transform=None, is_training=True):\n        self.df = df.reset_index(drop=True)\n        self.frame_paths_dict = frame_paths_dict\n        self.series_mapping_df = series_mapping_df\n        self.num_frames = num_frames\n        self.transform = transform\n        self.is_training = is_training\n        self._cache = {}\n        self._cache_keys = []\n        self._max_cache_size = config.CACHE_SIZE\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        if idx in self._cache:\n            return self._cache[idx]\n        \n        row = self.df.iloc[idx]\n        series_uid = row['SeriesInstanceUID']\n        labels = torch.tensor(row[TARGET_COLS].values.astype(np.float32))\n        metadata = self._extract_metadata(row)\n        \n        try:\n            image = self._load_8frame_3channel_image(series_uid, row)\n        except Exception as e:\n            # print(f\"Error loading {series_uid}: {e}. Returning zeros.\") # Uncomment for debug\n            dummy_image = np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n            if self.transform:\n                image = self.transform(image=dummy_image)['image']\n            else:\n                image = torch.from_numpy(dummy_image).permute(2,0,1).float()\n\n        result = (image, labels, metadata)\n        self._update_cache(idx, result)\n        return result\n\n    def _update_cache(self, idx, data):\n        if len(self._cache) >= self._max_cache_size:\n            oldest_idx = self._cache_keys.pop(0)\n            if oldest_idx in self._cache:\n                del self._cache[oldest_idx]\n        self._cache[idx] = data\n        self._cache_keys.append(idx)\n\n    def _extract_metadata(self, row) -> torch.Tensor:\n        if not config.USE_METADATA:\n            return torch.tensor([0.0, 0.0], dtype=torch.float32)\n        \n        age = row.get('PatientAge', 50)\n        if pd.isna(age):\n            age = 50\n        elif isinstance(age, str):\n            age = int(''.join(filter(str.isdigit, age[:3])) or '50')\n        age = min(float(age), 100.0) / 100.0\n        \n        sex = row.get('PatientSex', 'M')\n        sex = 1.0 if sex == 'M' else 0.0\n        return torch.tensor([age, sex], dtype=torch.float32)\n\n    def _load_8frame_3channel_image(self, series_uid: str, row) -> torch.Tensor:\n        entry = self.frame_paths_dict.get(series_uid, {'paths': [], 'is_dummy': True})\n        paths = entry['paths']\n        is_dummy = entry.get('is_dummy', True)\n        \n        if len(paths) == 0:\n            raise FileNotFoundError(f\"No paths found for series {series_uid}\")\n            \n        if is_dummy:\n            # is_dummy=True means we must load from DICOM\n            volume = self._load_volume_from_dicom_8frame(series_uid, row)\n        else:\n            # We have PNGs, load them\n            volume = self._load_volume_from_png_8frame(paths)\n            \n        volume_norm = robust_normalization(volume)\n        image = create_3channel_input_8frame(volume_norm)\n        \n        if self.transform:\n            transformed = self.transform(image=image)\n            image = transformed['image']\n            \n        return image\n\n    def _load_volume_from_png_8frame(self, paths: List[str]) -> np.ndarray:\n        volume = []\n        for path in paths:\n            try:\n                img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n                if img is not None:\n                    img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n                    volume.append(img)\n                else:\n                    volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n            except Exception:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        if not volume:\n            return np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n        return np.array(volume)\n\n    def _load_volume_from_dicom_8frame(self, series_uid: str, row) -> np.ndarray:\n        series_data = self.series_mapping_df[self.series_mapping_df['SeriesInstanceUID'] == series_uid].sort_values('relative_index')\n        if series_data.empty:\n            return np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n            \n        modality = row.get('Modality', 'CT')\n        \n        # Smart sampling logic on the series_data dataframe\n        if len(series_data) <= 8:\n            sampled_data = series_data\n        else:\n            all_indices = list(range(len(series_data)))\n            sampled_indices_str = smart_8_frame_sampling([str(i) for i in all_indices])\n            sampled_indices = [int(i) for i in sampled_indices_str]\n            sampled_data = series_data.iloc[sampled_indices]\n            \n        volume = []\n        for _, dicom_row in sampled_data.iterrows():\n            dicom_entry = dicom_row.get('dicom_filename', None)\n            try:\n                dicom_path = resolve_dicom_path(dicom_entry, series_uid)\n                ds = pydicom.dcmread(dicom_path, force=True)\n                img = ds.pixel_array.astype(np.float32)\n                \n                if hasattr(ds, 'RescaleSlope') and hasattr(ds, 'RescaleIntercept'):\n                    img = img * float(ds.RescaleSlope) + float(ds.RescaleIntercept)\n                    \n                if config.USE_WINDOWING:\n                    window_center, window_width = get_windowing_params(modality)\n                    img = apply_dicom_windowing(img, window_center, window_width)\n                else: # Simple normalization\n                    img_min, img_max = img.min(), img.max()\n                    if img_max > img_min:\n                        img = ((img - img_min) / (img_max - img_min) * 255).astype(np.uint8)\n                    else:\n                        img = np.zeros_like(img, dtype=np.uint8)\n                        \n                img = apply_clahe_normalization(img, modality)\n                img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), interpolation=cv2.INTER_AREA)\n                volume.append(img)\n            except Exception as e:\n                # print(f\"Dicom read error for {series_uid}: {e}\") # Uncomment for debug\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        # Ensure 8 frames\n        while len(volume) < 8:\n            if volume:\n                volume.append(volume[-1])\n            else:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n                \n        return np.array(volume[:8])\n\n\n\n\n\n# -------------------------\n# 8. TRAINING: TRANSFORMS\n# -------------------------\n\n# FIX: Removed ElasticTransform and GridDistortion as they \n# are the specific augmentations that trigger the scipy import error.\n\nif config.USE_STRONG_AUGMENTATION:\n    print(\"Using strong augmentation (minus Elastic/GridDistortion)...\")\n    train_transform = A.Compose([\n        A.Rotate(limit=15, p=0.7),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=10, p=0.6),\n        \n        A.ElasticTransform(alpha=50, sigma=5, p=0.3),     # <-- REMOVED: This causes the scipy error\n        A.GridDistortion(num_steps=3, distort_limit=0.1, p=0.3), # <-- REMOVED: This also causes the error\n        \n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.6),\n        A.RandomGamma(gamma_limit=(80, 120), p=0.4),\n        A.GaussNoise(var_limit=(10, 80), p=0.4),\n        A.Blur(blur_limit=3, p=0.2),\n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\nelse:\n    print(\"Using standard augmentation...\")\n    train_transform = A.Compose([\n        A.Rotate(limit=10, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.3),\n        A.GaussNoise(var_limit=(10, 50), p=0.2),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\nval_transform = A.Compose([\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\n\n\n\n\n# -------------------------\n# 9. TRAINING: LOSS & METRIC\n# -------------------------\ndef get_loss_function():\n    if config.USE_IMPROVED_LOSS:\n        return ImprovedLoss(aneurysm_weight=3.0, focal_weight=0.3).to(device)\n    else:\n        weights = torch.ones(config.NUM_CLASSES, device=device)\n        weights[-1] = 3.0\n        return nn.BCEWithLogitsLoss(pos_weight=weights)\n\ndef calculate_competition_metric(y_true, y_pred):\n    individual_aucs = []\n    for i in range(13):\n        try:\n            if len(np.unique(y_true[:, i])) > 1:\n                auc = roc_auc_score(y_true[:, i], y_pred[:, i])\n            else:\n                auc = 0.5\n            individual_aucs.append(auc)\n        except:\n            individual_aucs.append(0.5)\n            \n    try:\n        if len(np.unique(y_true[:, 13])) > 1:\n            aneurysm_present_auc = roc_auc_score(y_true[:, 13], y_pred[:, 13])\n        else:\n            aneurysm_present_auc = 0.5\n    except:\n        aneurysm_present_auc = 0.5\n        \n    avg_individual = np.mean(individual_aucs)\n    final_score = (aneurysm_present_auc + avg_individual) / 2\n    return final_score, aneurysm_present_auc, avg_individual\n\n# -------------------------\n# 10. TRAINING: HELPERS\n# -------------------------\ndef save_checkpoint(model, optimizer, scheduler, epoch, best_score, val_loss, out_dir, model_name, fold):\n    os.makedirs(out_dir, exist_ok=True)\n    model_path = os.path.join(\n        out_dir,\n        f\"{model_name}_fold{fold}_epoch{epoch}_score{best_score:.6f}.pth\"\n    )\n    torch.save({\n        'epoch': epoch,\n        'model_state_dict': model.state_dict(),\n        'optimizer_state_dict': optimizer.state_dict(),\n        'scheduler_state_dict': scheduler.state_dict(),\n        'best_score': best_score,\n        'val_loss': val_loss,\n    }, model_path)\n    return model_path\n\n@functools.lru_cache(maxsize=5000)\ndef extract_dicom_patient_info(series_uid: str) -> Tuple[str, str]:\n    \"\"\"Reads PatientID from the original DICOM files.\"\"\"\n    try:\n        dicom_dir = os.path.join(config.ORIGINAL_SERIES_DIR, series_uid)\n        if os.path.exists(dicom_dir):\n            dcm_files = [f for f in os.listdir(dicom_dir) if f.endswith('.dcm') or f.endswith('.DCM')]\n            if dcm_files:\n                # Read only the first DICOM file header\n                ds = pydicom.dcmread(os.path.join(dicom_dir, dcm_files[0]), stop_before_pixels=True, force=True)\n                study_uid = getattr(ds, 'StudyInstanceUID', None)\n                patient_id = getattr(ds, 'PatientID', None)\n                return study_uid or f\"fallback_{series_uid[:32]}\", patient_id\n    except Exception:\n        pass\n    return f\"fallback_{series_uid[:32]}\", f\"fallback_{series_uid[:32]}\"\n\n@functools.lru_cache(maxsize=5000)\ndef get_patient_group_cached(series_uid: str) -> str:\n    \"\"\"Gets a unique ID for grouping patients.\"\"\"\n    study_uid, patient_id = extract_dicom_patient_info(series_uid)\n    # Prefer StudyUID if available, as PatientID can be non-unique\n    return study_uid if study_uid and not study_uid.startswith('fallback_') else patient_id\n\ndef create_robust_cv_split(train_df_local, n_splits=5):\n    \"\"\"\n    This is the original, correct function that reads DICOM headers\n    to get the PatientID for grouping.\n    \"\"\"\n    print(\"Creating patient-separated cross-validation split...\")\n    patient_groups = []\n    # This loop is slow, but it is robust and correct.\n    for series_uid in tqdm(train_df_local['SeriesInstanceUID'], desc=\"Reading Patient Info for CV Split\"):\n        patient_group = get_patient_group_cached(series_uid)\n        patient_groups.append(patient_group)\n        \n    train_df_local = train_df_local.copy()\n    train_df_local['patient_id'] = patient_groups\n    \n    n_groups = train_df_local['patient_id'].nunique()\n    print(f\"Total unique patients found: {n_groups}\")\n    \n    if n_groups < n_splits:\n        print(f\"Warning: Only {n_groups} patients. Falling back to StratifiedKFold.\")\n        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n        return list(skf.split(train_df_local, train_df_local['Aneurysm Present']))\n        \n    gkf = GroupKFold(n_splits=n_splits)\n    splits = list(gkf.split(train_df_local, groups=train_df_local['patient_id']))\n    return splits\n\n\n# -------------------------\n# 11. TRAINING: TRAIN/VAL LOOPS\n# -------------------------\ndef train_epoch_optimized(model, train_loader, criterion, optimizer, scaler, device, accumulation_steps):\n    model.train()\n    running_loss = 0.0\n    optimizer.zero_grad()\n    \n    for batch_idx, (images, targets, metadata) in enumerate(tqdm(train_loader, desc=\"Training\")):\n        images = images.to(device, non_blocking=True)\n        targets = targets.to(device, non_blocking=True)\n        metadata = metadata.to(device, non_blocking=True)\n        \n        with torch.cuda.amp.autocast():\n            outputs = model(images, metadata)\n            loss = criterion(outputs, targets)\n            loss = loss / accumulation_steps\n            \n        scaler.scale(loss).backward()\n        \n        if (batch_idx + 1) % accumulation_steps == 0 or (batch_idx + 1) == len(train_loader):\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            \n        running_loss += loss.item() * accumulation_steps\n        \n    return running_loss / len(train_loader)\n\ndef validate_epoch_optimized(model, val_loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    all_outputs = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for images, targets, metadata in tqdm(val_loader, desc=\"Validating\"):\n            images = images.to(device, non_blocking=True)\n            targets = targets.to(device, non_blocking=True)\n            metadata = metadata.to(device, non_blocking=True)\n            \n            with torch.cuda.amp.autocast():\n                logits = model(images, metadata)\n                loss = criterion(logits, targets)\n                \n            outputs = torch.sigmoid(logits)\n            running_loss += loss.item()\n            all_outputs.append(outputs.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n            \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    \n    final_score, aneurysm_auc, avg_individual = calculate_competition_metric(all_targets, all_outputs)\n    return running_loss / len(val_loader), final_score, aneurysm_auc, avg_individual\n\n\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:27.583876Z","iopub.execute_input":"2025-11-15T05:55:27.584064Z","iopub.status.idle":"2025-11-15T05:55:27.639691Z","shell.execute_reply.started":"2025-11-15T05:55:27.584048Z","shell.execute_reply":"2025-11-15T05:55:27.638819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# -------------------------\n# 12. TRAINING: MAIN EXECUTION\n# -------------------------\nprint(\"Loading main CSVs...\")\n\n# --- Load all data ---\ntry:\n    if not os.path.exists(config.TRAIN_CSV_PATH):\n        raise FileNotFoundError(f\"Original train.csv not found at {config.TRAIN_CSV_PATH}\")\n    if not os.path.exists(config.SERIES_MAPPING_PATH):\n         raise FileNotFoundError(f\"series_index_mapping.csv not found at {config.SERIES_MAPPING_PATH}\")\n         \n    train_df = pd.read_csv(config.TRAIN_CSV_PATH)\n    series_mapping_df = pd.read_csv(config.SERIES_MAPPING_PATH)\n    \nexcept FileNotFoundError as e:\n    print(f\"ERROR: {e}\")\n    print(f\"Please make sure your PNG dataset ('another1') and the original competition data are both added to the notebook.\")\n    # Stop training if files are missing\n    train_df = None\n\nif train_df is not None:\n    # Build frame paths dict ONCE\n    frame_paths_dict = create_frame_paths_8frame_structured(train_df, series_mapping_df)\n    \n    # Filter train_df to valid series ONCE\n    valid_series = [uid for uid, entry in frame_paths_dict.items() if len(entry['paths']) > 0]\n    train_df_filtered = train_df[train_df['SeriesInstanceUID'].isin(valid_series)].copy()\n    print(f\"Filtered train data shape (series with paths): {train_df_filtered.shape}\")\n    \n    # Create CV splits ONCE\n    cv_splits = create_robust_cv_split(train_df_filtered, config.NUM_FOLDS)\n    \n    # --- Train all 5 folds for the ensemble ---\n    print(f\"\\n===== STARTING {config.NUM_FOLDS}-FOLD TRAINING =====\")\n    \n    for fold in range(1):\n        print(f\"\\n--- Fold {fold}/{config.NUM_FOLDS - 1} ---\")\n        \n        # --- 1. Get Fold Data ---\n        train_indices, val_indices = cv_splits[fold]\n        train_fold_df = train_df_filtered.iloc[train_indices]\n        val_fold_df = train_df_filtered.iloc[val_indices]\n        print(f\"Train size: {len(train_fold_df)}, Val size: {len(val_fold_df)}\")\n\n        # --- 2. Create Datasets & Loaders ---\n        train_dataset = EightFrameDataset(train_fold_df, frame_paths_dict, series_mapping_df, \n                                          num_frames=config.NUM_FRAMES, transform=train_transform, is_training=True)\n        val_dataset = EightFrameDataset(val_fold_df, frame_paths_dict, series_mapping_df, \n                                        num_frames=config.NUM_FRAMES, transform=val_transform, is_training=False)\n        \n        train_loader = DataLoader(train_dataset, batch_size=config.BATCH_SIZE, shuffle=True, \n                                  num_workers=config.NUM_WORKERS, pin_memory=config.PIN_MEMORY, \n                                  drop_last=True, prefetch_factor=config.PREFETCH_FACTOR, \n                                  persistent_workers=config.PERSISTENT_WORKERS)\n        val_loader = DataLoader(val_dataset, batch_size=config.BATCH_SIZE, shuffle=False, \n                                num_workers=config.NUM_WORKERS, pin_memory=config.PIN_MEMORY, \n                                prefetch_factor=config.PREFETCH_FACTOR, \n                                persistent_workers=config.PERSISTENT_WORKERS)\n        print(f\"Train batches: {len(train_loader)}, Val batches: {len(val_loader)}\")\n\n        # --- 3. Initialize Model, Optimizer, etc. (Fresh for each fold) ---\n        model = ImprovedMultiFrameModel(num_frames=config.NUM_FRAMES, num_classes=config.NUM_CLASSES, pretrained=True)\n        model = model.to(device)\n        \n        criterion = get_loss_function()\n        optimizer = AdamW(model.parameters(), lr=config.LEARNING_RATE, weight_decay=1e-4)\n        scheduler = CosineAnnealingLR(optimizer, T_max=config.NUM_EPOCHS, eta_min=1e-6)\n        scaler = torch.cuda.amp.GradScaler()\n\n        # --- 4. Run Training Loop for this fold ---\n        best_score = 0.0\n        best_epoch = 0\n        patience_counter = 0\n        \n        for epoch in range(config.NUM_EPOCHS):\n            print(f\"\\nEpoch {epoch+1}/{config.NUM_EPOCHS}\")\n            \n            train_loss = train_epoch_optimized(model, train_loader, criterion, optimizer, scaler, device, config.ACCUMULATION_STEPS)\n            val_loss, val_score, aneurysm_auc, avg_individual = validate_epoch_optimized(model, val_loader, criterion, device)\n            scheduler.step()\n            \n            print(f\"Fold {fold} Epoch {epoch+1} - Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}, Val Score: {val_score:.6f}\")\n            print(f\"Aneurysm AUC: {aneurysm_auc:.4f}, Avg Location AUC: {avg_individual:.4f}\")\n            \n            if val_score > best_score:\n                best_score = val_score\n                best_epoch = epoch + 1\n                patience_counter = 0\n                ckpt_path = save_checkpoint(model, optimizer, scheduler, epoch+1, best_score, val_loss, \n                                            config.OUTPUT_DIR, config.MODEL_NAME, fold)\n                print(f\"Saved checkpoint (Best Score): {ckpt_path}\")\n            else:\n                patience_counter += 1\n                print(f\"No improvement. Patience: {patience_counter}/{config.EARLY_STOPPING_PATIENCE}\")\n                if patience_counter >= config.EARLY_STOPPING_PATIENCE:\n                    print(f\"Early stopping at epoch {epoch+1}\")\n                    break\n            \n            torch.cuda.empty_cache()\n            gc.collect()\n\n        print(f\"\\nFold {fold} finished. Best Score: {best_score:.6f} at epoch {best_epoch}\")\n        \n        # Clean up memory before next fold\n        del model, train_dataset, val_dataset, train_loader, val_loader, optimizer, scheduler, scaler\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    print(\"\\n===== ALL 5 FOLDS TRAINING COMPLETE =====\")\n    print(\"Your models are saved in the '/kaggle/working' directory.\")\n    print(\"Please download them, upload as a new dataset, and create a new notebook for submission.\")\n\nelse:\n    print(\"TRAIN_MODEL is False. Skipping training.\")\n    print(\"To train your models, set TRAIN_MODEL = True at the top of the script.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T05:55:27.640998Z","iopub.execute_input":"2025-11-15T05:55:27.641344Z","iopub.status.idle":"2025-11-15T08:42:27.39182Z","shell.execute_reply.started":"2025-11-15T05:55:27.641294Z","shell.execute_reply":"2025-11-15T08:42:27.390821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}