{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13603172,"sourceType":"datasetVersion","datasetId":8644155},{"sourceId":13670106,"sourceType":"datasetVersion","datasetId":8691937},{"sourceId":254737188,"sourceType":"kernelVersion"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install nnunet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:37:34.457921Z","iopub.execute_input":"2025-11-08T11:37:34.458193Z","iopub.status.idle":"2025-11-08T11:40:02.465973Z","shell.execute_reply.started":"2025-11-08T11:37:34.458174Z","shell.execute_reply":"2025-11-08T11:40:02.462394Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PreProcessing","metadata":{}},{"cell_type":"code","source":"from glob import glob\nimport pydicom as dicom #for dicom files\nimport nibabel as nib #for nii files\n\nimport os\nimport shutil\nimport gc\nfrom collections import defaultdict\nfrom typing import Tuple, List, Dict, Optional, Callable\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport polars as pl\nimport pydicom\nfrom tqdm.auto import tqdm\nfrom scipy import ndimage\nfrom sklearn.preprocessing import StandardScaler\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nimport json\nimport time\nfrom functools import partial\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.optim as optim\n\nimport kaggle_evaluation.rsna_inference_server\n\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:53:43.842987Z","iopub.execute_input":"2025-11-08T11:53:43.84327Z","iopub.status.idle":"2025-11-08T11:53:43.849441Z","shell.execute_reply.started":"2025-11-08T11:53:43.84325Z","shell.execute_reply":"2025-11-08T11:53:43.84883Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ID_COL = 'SeriesInstanceUID'\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\n# Paths - Kaggle-specific\nTRAIN_CSV_PATH = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\nSERIES_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series\"\nOUTPUT_DIR = \"/kaggle/working/preprocessed_data\"\n\n# Processing configuration\nTARGET_SIZE = (64, 64, 64)      # final (D,H,W)\nTARGET_SPACING_MM = 1.0         # isotropic resample\nCTA_WINDOW = (300.0, 700.0)     # (center, width) for CT (CTA)\nMRI_Z_CLIP = 3.0                # clip z-score to ±3σ\nMAX_SERIES = None               # Set to None for full dataset, or number for testing\nTEST_MODE = False               # Ultra-fast test mode (5 series, no progress bars)\n\n# Create output directory\nos.makedirs(OUTPUT_DIR, exist_ok=True)\nprint(f\"Preprocessing output will be saved to: {OUTPUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:53:46.922701Z","iopub.execute_input":"2025-11-08T11:53:46.923254Z","iopub.status.idle":"2025-11-08T11:53:46.929224Z","shell.execute_reply.started":"2025-11-08T11:53:46.92323Z","shell.execute_reply":"2025-11-08T11:53:46.928414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def _safe_zoom(volume: np.ndarray, zoom_factors: Tuple[float, ...], order: int = 1) -> np.ndarray:\n    \"\"\"Robust wrapper around ndimage.zoom to avoid rank mismatch and invalid factors.\"\"\"\n    volume = np.nan_to_num(volume, copy=False)\n    zf = tuple(float(max(1e-6, f)) for f in zoom_factors)  # avoid zeros/negatives\n    if len(zf) != volume.ndim:\n        if len(zf) > volume.ndim:\n            zf = zf[:volume.ndim]\n        else:\n            zf = (1.0,) * (volume.ndim - len(zf)) + zf\n    return ndimage.zoom(volume, zf, order=order)\n\ndef _resize_slice(arr: np.ndarray, out_h: int, out_w: int) -> np.ndarray:\n    \"\"\"Resize a 2D slice to (out_h, out_w) using safe zoom.\"\"\"\n    h, w = arr.shape\n    if h == out_h and w == out_w:\n        return arr.astype(np.float32, copy=False)\n    zy = out_h / max(h, 1)\n    zx = out_w / max(w, 1)\n    return _safe_zoom(arr, (zy, zx), order=1).astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:53:49.626012Z","iopub.execute_input":"2025-11-08T11:53:49.626808Z","iopub.status.idle":"2025-11-08T11:53:49.633975Z","shell.execute_reply.started":"2025-11-08T11:53:49.62678Z","shell.execute_reply":"2025-11-08T11:53:49.633186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class DICOMProcessor:\n    \"\"\"Process DICOM series into normalized 3D volumes.\"\"\"\n    \n    def __init__(\n        self,\n        target_size: Tuple[int, int, int] = TARGET_SIZE,\n        target_spacing_mm: float = TARGET_SPACING_MM,\n        cta_window: Tuple[float, float] = CTA_WINDOW,\n        mri_z_clip: float = MRI_Z_CLIP,\n    ):\n        self.target_size = target_size\n        self.target_spacing_mm = target_spacing_mm\n        self.cta_window = cta_window\n        self.mri_z_clip = mri_z_clip\n        \n        # Adjustment counters\n        self.slope_adjustments = 0\n        self.intercept_adjustments = 0\n        self.adaptive_windowing_count = 0\n        self.failed_series = []\n        self.modalities = {}  # Track modality for each series\n\n    def _validate_and_apply_rescale(self, sl: np.ndarray, ds) -> np.ndarray:\n        \"\"\"Validate slope/intercept values and apply rescaling.\"\"\"\n        slope = float(getattr(ds, \"RescaleSlope\", 1.0))\n        intercept = float(getattr(ds, \"RescaleIntercept\", 0.0))\n        \n        # Validate slope\n        if slope <= 0 or not np.isfinite(slope) or abs(slope) > 1000:\n            slope = 1.0\n            self.slope_adjustments += 1\n        \n        # Validate intercept with range clamping for extreme values\n        if not np.isfinite(intercept):\n            intercept = 0.0\n            self.intercept_adjustments += 1\n        elif abs(intercept) > 10000:\n            intercept = np.clip(intercept, -2000, 0)\n            self.intercept_adjustments += 1\n        \n        # Apply rescaling directly to array (no copy)\n        sl *= slope\n        sl += intercept\n        \n        # Validate result\n        if np.any(~np.isfinite(sl)):\n            sl = np.nan_to_num(sl, copy=False)\n        \n        # Post-rescale range check\n        min_val = sl.min()\n        max_val = sl.max()\n        if min_val < -5000 or max_val > 10000:\n            np.clip(sl, -3000, 5000, out=sl)\n        \n        return sl\n\n    def _log_adjustment_summary(self):\n        \"\"\"Log summary of adjustments made during processing.\"\"\"\n        print(f\"\\nProcessing adjustments summary:\")\n        print(f\"- Slope adjustments: {self.slope_adjustments}\")\n        print(f\"- Intercept adjustments: {self.intercept_adjustments}\")\n        print(f\"- Adaptive windowing: {self.adaptive_windowing_count}\")\n        print(f\"- Failed series: {len(self.failed_series)}\")\n        if self.failed_series:\n            print(\"\\nFirst 5 failed series:\")\n            for sid in self.failed_series[:5]:\n                print(f\"  {sid}\")\n\n    def _process_single_series(self, series_path: str, series_id: str) -> Tuple[Optional[np.ndarray], Optional[str]]:\n        \"\"\"Return (D,H,W) float32 volume in [0,1]. Returns None if processing fails.\"\"\"\n        try:\n            # Collect DICOM datasets\n            dicoms = []\n            for root, _, files in os.walk(series_path):\n                for f in files:\n                    if f.endswith(\".dcm\"):\n                        try:\n                            ds = pydicom.dcmread(os.path.join(root, f), force=True)\n                            if hasattr(ds, \"PixelData\"):\n                                dicoms.append(ds)\n                        except Exception as e:\n                            continue\n            if not dicoms:\n                raise ValueError(f\"No valid DICOM files with pixel data in {series_path}\")\n\n            dicoms = self._sort_slices(dicoms)\n            has_multiframe = any(getattr(ds, \"NumberOfFrames\", 1) > 1 for ds in dicoms)\n            spacing = self._get_spacing(dicoms, has_multiframe=has_multiframe)\n\n            # Choose base HxW\n            base_h, base_w = self._choose_base_shape(dicoms)\n\n            modality_tag = (getattr(dicoms[0], \"Modality\", \"\") or \"\").upper()\n            self.modalities[series_id] = modality_tag  # Track modality\n            \n            vol_slices = []\n            for ds in dicoms:\n                arr = ds.pixel_array\n                # standardize to (N,H,W) where N=number of frames (1 if 2D)\n                if arr.ndim >= 3:\n                    h, w = arr.shape[-2], arr.shape[-1]\n                    n = int(np.prod(arr.shape[:-2]))\n                    arr = arr.reshape(n, h, w)\n                    frames = arr\n                else:\n                    frames = arr[np.newaxis, ...]  # shape (1,H,W)\n\n                for sl in frames:\n                    sl = sl.astype(np.float32)\n\n                    # Handle MONOCHROME1 inversion\n                    if getattr(ds, \"PhotometricInterpretation\", \"MONOCHROME2\") == \"MONOCHROME1\":\n                        sl = sl.max() - sl\n\n                    # Apply validated rescaling\n                    sl = self._validate_and_apply_rescale(sl, ds)\n\n                    sl = _resize_slice(sl, base_h, base_w)\n                    vol_slices.append(sl)\n\n            if len(vol_slices) == 0:\n                raise ValueError(\"No valid slices extracted.\")\n\n            volume = np.stack(vol_slices, axis=0).astype(np.float32)  # (D,H,W)\n\n            # Normalize by modality -> [0,1]\n            volume = self._normalize_by_modality(volume, modality_tag)\n\n            # Isotropic resample (mm-based)\n            if self.target_spacing_mm is not None:\n                dz, dy, dx = spacing\n                z, y, x = volume.shape\n                newD = max(1, int(round(z * dz / self.target_spacing_mm)))\n                newH = max(1, int(round(y * dy / self.target_spacing_mm)))\n                newW = max(1, int(round(x * dx / self.target_spacing_mm)))\n                volume = _safe_zoom(volume, (newD / z, newH / y, newW / x), order=1)\n\n            # Resize to target grid\n            tz, ty, tx = self.target_size\n            z, y, x = volume.shape\n            volume = _safe_zoom(volume, (tz / z, ty / y, tx / x), order=1).astype(np.float32)\n            \n            # Quality check - reject near-zero volumes\n            if np.mean(volume) < 0.01 or np.std(volume) < 0.01:\n                raise ValueError(\"Volume has near-zero intensity\")\n                \n            return volume, modality_tag\n\n        except Exception as e:\n            self.failed_series.append(series_id)\n            return None, None\n    \n    def process_series(self, series_path: str, series_id: str) -> Tuple[Optional[np.ndarray], Optional[str]]:\n        \"\"\"Wrapper for parallel processing - returns (series_id, volume)\"\"\"\n        volume, modality = self._process_single_series(series_path, series_id)\n        return (series_id, volume, modality)\n    \n    def _sort_slices(self, ds_list: List[pydicom.dataset.FileDataset]) -> List[pydicom.dataset.FileDataset]:\n        try:\n            orient = np.array(ds_list[0].ImageOrientationPatient, dtype=np.float32)\n            row = orient[:3]; col = orient[3:]\n            normal = np.cross(row, col)\n            def sort_key(ds):\n                ipp = np.array(getattr(ds, \"ImagePositionPatient\", [0, 0, 0]), dtype=np.float32)\n                return float(np.dot(ipp, normal))\n            return sorted(ds_list, key=sort_key)\n        except Exception:\n            return sorted(ds_list, key=lambda ds: getattr(ds, \"InstanceNumber\", 0))\n\n    def _get_spacing(self, ds_sorted: List[pydicom.dataset.FileDataset], has_multiframe: bool = False) -> Tuple[float, float, float]:\n        try:\n            dy, dx = map(float, ds_sorted[0].PixelSpacing)\n        except Exception:\n            ps = getattr(ds_sorted[0], \"PixelSpacing\", [1.0, 1.0])\n            dy, dx = float(ps[0]), float(ps[1])\n\n        if has_multiframe:\n            dz = float(getattr(ds_sorted[0], \"SpacingBetweenSlices\", getattr(ds_sorted[0], \"SliceThickness\", 1.0)))\n        else:\n            zs = []\n            for i in range(1, len(ds_sorted)):\n                p0 = np.array(getattr(ds_sorted[i-1], \"ImagePositionPatient\", [0, 0, 0]), dtype=np.float32)\n                p1 = np.array(getattr(ds_sorted[i], \"ImagePositionPatient\", [0, 0, 0]), dtype=np.float32)\n                d = np.linalg.norm(p1 - p0)\n                if d > 0:\n                    zs.append(d)\n            if zs:\n                dz = float(np.median(zs))\n            else:\n                dz = float(getattr(ds_sorted[0], \"SliceThickness\", 1.0))\n\n        dz = dz if (dz > 0 and np.isfinite(dz)) else 1.0\n        dy = dy if (dy > 0 and np.isfinite(dy)) else 1.0\n        dx = dx if (dx > 0 and np.isfinite(dx)) else 1.0\n        return (dz, dy, dx)\n\n    def _choose_base_shape(self, ds_list: List[pydicom.dataset.FileDataset]) -> Tuple[int, int]:\n        shapes = []\n        for ds in ds_list:\n            try:\n                h, w = int(ds.Rows), int(ds.Columns)\n            except Exception:\n                arr = ds.pixel_array\n                h, w = arr.shape[-2], arr.shape[-1]\n            shapes.append((h, w))\n        vals, counts = np.unique(shapes, return_counts=True, axis=0)\n        base = tuple(vals[counts.argmax()])\n        return int(base[0]), int(base[1])\n\n    def _normalize_by_modality(self, volume: np.ndarray, modality_tag: str) -> np.ndarray:\n        \"\"\"CT: adaptive windowing for extreme ranges; MR: z-score -> clip -> [0,1].\"\"\"\n        volume = np.nan_to_num(volume, copy=False)\n        \n        if modality_tag == \"CT\":\n            min_val, max_val = volume.min(), volume.max()\n            \n            # Check if values are in normal CT range\n            if min_val >= -2000 and max_val <= 4000:\n                # Normal range: use standard windowing\n                c, w = self.cta_window\n                lo, hi = c - w / 2.0, c + w / 2.0\n            else:\n                # Extreme range: use adaptive windowing\n                self.adaptive_windowing_count += 1\n                \n                # Percentile-based adaptive window\n                p1, p99 = np.percentile(volume, [1, 99])\n                margin = (p99 - p1) * 0.1\n                lo = p1 - margin\n                hi = p99 + margin\n                \n                # Ensure minimum window width\n                if hi - lo < 100:\n                    center = (hi + lo) / 2\n                    lo = center - 50\n                    hi = center + 50\n            \n            # In-place clipping and normalization\n            np.clip(volume, lo, hi, out=volume)\n            volume = (volume - lo) / (hi - lo + 1e-6)\n            return volume.astype(np.float32, copy=False)\n        else:\n            # MRI processing\n            mean = float(volume.mean())\n            std = float(volume.std() + 1e-6)\n            \n            # Validate statistics\n            if std < 1e-6 or not np.isfinite(mean) or not np.isfinite(std):\n                return np.full_like(volume, 0.5, dtype=np.float32)\n            \n            # Check dynamic range\n            min_val, max_val = volume.min(), volume.max()\n            if max_val - min_val < 1e-6:\n                return np.full_like(volume, 0.5, dtype=np.float32)\n            \n            # In-place operations\n            volume -= mean\n            volume /= std\n            zc = float(self.mri_z_clip)\n            np.clip(volume, -zc, zc, out=volume)\n            volume = (volume + zc) / (2.0 * zc)\n            return volume.astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:53:52.310511Z","iopub.execute_input":"2025-11-08T11:53:52.310808Z","iopub.status.idle":"2025-11-08T11:53:52.340341Z","shell.execute_reply.started":"2025-11-08T11:53:52.310786Z","shell.execute_reply":"2025-11-08T11:53:52.339746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_single_series_wrapper(args):\n    \"\"\"Wrapper for parallel processing to handle the processor instance correctly.\"\"\"\n    processor, series_path, series_id = args\n    return processor.process_series(series_path, series_id)\n    \ndef main():\n    # Initialize processor\n    processor = DICOMProcessor(\n        target_size=TARGET_SIZE,\n        target_spacing_mm=TARGET_SPACING_MM,\n        cta_window=CTA_WINDOW,\n        mri_z_clip=MRI_Z_CLIP,\n    )\n    \n    # Load training labels\n    train_df = pd.read_csv(TRAIN_CSV_PATH)\n    print(f\"Loaded {len(train_df)} series from train.csv\")\n    \n    # CORRECTED SAMPLING LOGIC - Replace the existing block with this\n    if MAX_SERIES is not None and len(train_df) > MAX_SERIES:\n        # Stratified sampling to maintain class distribution\n        from sklearn.model_selection import StratifiedShuffleSplit\n        sss = StratifiedShuffleSplit(n_splits=1, test_size=MAX_SERIES, random_state=42)\n        _, test_idx = next(sss.split(train_df, train_df['Aneurysm Present']))  # Get TEST indices\n        train_df = train_df.iloc[test_idx].reset_index(drop=True)\n        print(f\"Reduced to {len(train_df)} series for testing (stratified)\")\n\n    # TEST_MODE for ultra-fast validation (5 series)\n    if TEST_MODE:\n        print(\"\\n⚠️ TEST MODE ACTIVE - Processing only 5 series for validation\")\n        train_df = train_df.head(5).copy()\n        MAX_WORKERS = 2  # Fewer workers for small test\n    else:\n        # Determine optimal number of workers (leave 1 core free)\n        MAX_WORKERS = max(1, os.cpu_count() - 1)\n        \n    # Clear output directory for the selected subset to force reprocessing\n    if os.path.exists(OUTPUT_DIR):\n        shutil.rmtree(OUTPUT_DIR)\n        os.makedirs(OUTPUT_DIR, exist_ok=True)\n        print(f\"Cleared output directory to force reprocessing of {len(train_df)} series\")\n    \n    # Prepare processing tasks\n    tasks = []\n    for _, row in train_df.iterrows():\n        series_id = row[ID_COL]\n        series_path = os.path.join(SERIES_DIR, series_id)\n        tasks.append((processor, series_path, series_id))\n    \n    # Parallel processing\n    print(f\"\\nStarting preprocessing of {len(train_df)} series using {MAX_WORKERS} workers...\")\n    start_time = time.time()\n    processed_volumes = {}\n    series_modalities = {}  # Will store modality for each series\n    \n    # Use ProcessPoolExecutor for parallel processing\n    with ProcessPoolExecutor(max_workers=MAX_WORKERS) as executor:\n        # Submit all tasks\n        future_to_series = {\n            executor.submit(process_single_series_wrapper, task): task[2] \n            for task in tasks\n        }\n        \n        # Progress tracking\n        completed = 0\n        pbar = tqdm(total=len(tasks), disable=TEST_MODE)  # Hide progress in test mode\n        \n        for future in as_completed(future_to_series):\n            result = future.result()\n            series_id = result[0]\n            volume = result[1]\n            modality = result[2] if len(result) > 2 else None\n            \n            if volume is not None:\n                output_path = os.path.join(OUTPUT_DIR, f\"{series_id}.npy\")\n                np.save(output_path, volume)\n                processed_volumes[series_id] = volume\n                series_modalities[series_id] = modality  # Store modality\n            \n            completed += 1\n            pbar.update(1)\n            pbar.set_postfix(processed=completed, failed=len(tasks)-completed)\n            \n            # Memory cleanup\n            if completed % 10 == 0:\n                gc.collect()\n    \n    pbar.close()\n    total_time = time.time() - start_time\n    print(f\"\\nPreprocessing completed in {total_time:.1f} seconds\")\n    print(f\"Average time per series: {total_time/max(len(train_df),1):.2f} seconds\")\n    \n    # Save metadata\n    metadata = {\n        'series_ids': list(processed_volumes.keys()),\n        'modalities': series_modalities,\n        'config': {\n            'target_size': TARGET_SIZE,\n            'target_spacing_mm': TARGET_SPACING_MM,\n            'cta_window': CTA_WINDOW,\n            'mri_z_clip': MRI_Z_CLIP,\n        },\n        'processing_time': total_time,\n        'series_count': len(processed_volumes),\n        'failed_series_count': len(train_df) - len(processed_volumes)\n    }\n    \n    with open(os.path.join(OUTPUT_DIR, 'metadata.json'), 'w') as f:\n        json.dump(metadata, f, indent=2)\n    \n    # Final report\n    processor._log_adjustment_summary()\n\n    # Additional modality breakdown\n    if series_modalities:\n        modality_counts = {}\n        for m in series_modalities.values():\n            if m:\n                modality_counts[m] = modality_counts.get(m, 0) + 1\n        \n        print(\"\\nModality distribution in processed data:\")\n        for mod, count in modality_counts.items():\n            print(f\"- {mod}: {count} series ({count/len(series_modalities)*100:.1f}%)\")\n            \n    print(f\"\\nPreprocessing complete! Successfully processed {len(processed_volumes)}/{len(train_df)} series\")\n    print(f\"Preprocessed volumes saved to: {OUTPUT_DIR}\")\n    print(f\"Metadata saved to: {os.path.join(OUTPUT_DIR, 'metadata.json')}\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T11:53:59.069093Z","iopub.execute_input":"2025-11-08T11:53:59.069656Z","iopub.status.idle":"2025-11-08T16:25:50.32319Z","shell.execute_reply.started":"2025-11-08T11:53:59.069631Z","shell.execute_reply":"2025-11-08T16:25:50.314521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MR Model","metadata":{}},{"cell_type":"code","source":"!pip install monai --quiet","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:31:00.634175Z","iopub.execute_input":"2025-11-08T16:31:00.634854Z","iopub.status.idle":"2025-11-08T16:31:12.473016Z","shell.execute_reply.started":"2025-11-08T16:31:00.634826Z","shell.execute_reply":"2025-11-08T16:31:12.472023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nimport json\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport random\nfrom scipy.ndimage import rotate, zoom, gaussian_filter\n\nfrom monai.networks.nets import DenseNet121\n# Check if we can use GPU (makes training faster)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Reproducibility (so results are consistent)\ntorch.manual_seed(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:34:23.128102Z","iopub.execute_input":"2025-11-08T16:34:23.12887Z","iopub.status.idle":"2025-11-08T16:34:54.449166Z","shell.execute_reply.started":"2025-11-08T16:34:23.128838Z","shell.execute_reply":"2025-11-08T16:34:54.448518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Path to our preprocessed data (from the previous notebook)\nPREPROCESSED_DIR = \"/kaggle/working/preprocessed_data\"\nTRAIN_CSV_PATH = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\n\n# Load metadata to see what we have\nwith open(os.path.join(PREPROCESSED_DIR, 'metadata.json'), 'r') as f:\n    metadata = json.load(f)\n\nprint(f\"Found {metadata['series_count']} successfully processed brain scans\")\nprint(f\"Failed to process: {metadata['failed_series_count']} scans\")\n\n# Load the training labels (which scans have aneurysms)\ntrain_df = pd.read_csv(TRAIN_CSV_PATH)\n\n# Keep only the series we successfully processed\nprocessed_ids = metadata['series_ids']\ntrain_df = train_df[train_df['SeriesInstanceUID'].isin(processed_ids)].copy()\n\n# Filter out only CT\nprocessed_ids = [sid for sid, mod in metadata['modalities'].items() if mod == \"MR\"]\ntrain_df = train_df[train_df['SeriesInstanceUID'].isin(processed_ids)].copy()\nprint(f\"Using {len(train_df)} scans for training\")\n\n# Define our target columns (the aneurysm locations)\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',  # This is the main label we care about most\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:35:10.636609Z","iopub.execute_input":"2025-11-08T16:35:10.63792Z","iopub.status.idle":"2025-11-08T16:35:10.673328Z","shell.execute_reply.started":"2025-11-08T16:35:10.637888Z","shell.execute_reply":"2025-11-08T16:35:10.672541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a simple dataset class (like a photo album for our brain scans)\nclass BrainScanDataset(Dataset):\n    def __init__(self, series_ids, labels_df, data_dir):\n        self.series_ids = series_ids\n        self.labels_df = labels_df.set_index('SeriesInstanceUID')\n        self.data_dir = data_dir\n    \n    def __len__(self):\n        return len(self.series_ids)\n    \n    def __getitem__(self, idx):\n        series_id = self.series_ids[idx]\n        \n        # Load the preprocessed brain scan\n        volume = np.load(os.path.join(self.data_dir, f\"{series_id}.npy\"))\n        \n        # Get the labels (which locations have aneurysms)\n        labels = self.labels_df.loc[series_id][LABEL_COLS].values.astype(np.float32)\n        \n        # Convert to PyTorch tensors (the format our model needs)\n        volume_tensor = torch.FloatTensor(volume).unsqueeze(0)  # Add channel dimension\n        labels_tensor = torch.FloatTensor(labels)\n        \n        return volume_tensor, labels_tensor\n\n# Split data into training and validation sets (80% for learning, 20% for testing)\ntrain_ids, val_ids = train_test_split(\n    processed_ids, \n    test_size=0.2, \n    random_state=42,\n    stratify=train_df['Aneurysm Present'].values  # Keep same proportion of positive cases\n)\n\n# Create datasets\ntrain_dataset = BrainScanDataset(train_ids, train_df, PREPROCESSED_DIR)\nval_dataset = BrainScanDataset(val_ids, train_df, PREPROCESSED_DIR)\n\n# Create data loaders (like a conveyor belt bringing data to our model)\ntrain_loader = DataLoader(train_dataset, batch_size=4, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, num_workers=2)\n\nprint(f\"Training set: {len(train_dataset)} scans\")\nprint(f\"Validation set: {len(val_dataset)} scans\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:35:13.98509Z","iopub.execute_input":"2025-11-08T16:35:13.985977Z","iopub.status.idle":"2025-11-08T16:35:14.001043Z","shell.execute_reply.started":"2025-11-08T16:35:13.985942Z","shell.execute_reply":"2025-11-08T16:35:14.000041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = DenseNet121(\n    spatial_dims=3,     # 3D model\n    in_channels=1,      # grayscale MRI or CT\n    out_channels=14     # number of labels (your case)\n).to(device)\n\ntotal_params = sum(p.numel() for p in model.parameters())\nprint(f\"✅ Model created with {total_params:,} parameters\")\nprint(f\"Using device: {device}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:35:17.174211Z","iopub.execute_input":"2025-11-08T16:35:17.174582Z","iopub.status.idle":"2025-11-08T16:35:17.603298Z","shell.execute_reply.started":"2025-11-08T16:35:17.174552Z","shell.execute_reply":"2025-11-08T16:35:17.602202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"learning_rate = 0.001\nnum_epochs = 36\n\n# General BCEWithLogitsLoss for multilabel classification\ncriterion = nn.BCEWithLogitsLoss()\n\n# Optimizer with weight decay for regularization\noptimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n\n# Scheduler to reduce LR when validation loss stops improving\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode='min', factor=0.5, patience=3, verbose=True\n)\n\n# Early stopping setup\nbest_val_loss = float('inf')\npatience = 5\nno_improve = 0\ntrain_losses = []\nval_losses = []\n\nprint(\"\\n🚀 Starting training...\")\nfor epoch in range(num_epochs):\n    # -------------------------------\n    # 🔹 Training Phase\n    # -------------------------------\n    model.train()\n    running_loss = 0.0\n    \n    train_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs} [Train]\")\n    for volumes, labels in train_bar:\n        volumes = volumes.to(device)\n        labels = labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(volumes)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        train_bar.set_postfix(loss=f\"{running_loss/len(train_bar):.4f}\")\n    \n    avg_train_loss = running_loss / len(train_loader)\n    train_losses.append(avg_train_loss)\n    \n    # -------------------------------\n    # 🔹 Validation Phase\n    # -------------------------------\n    model.eval()\n    val_loss = 0.0\n    \n    with torch.no_grad():\n        for volumes, labels in val_loader:\n            volumes = volumes.to(device)\n            labels = labels.to(device)\n            \n            outputs = model(volumes)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n    \n    avg_val_loss = val_loss / len(val_loader)\n    val_losses.append(avg_val_loss)\n    \n    # -------------------------------\n    # 🔹 Logging & Checkpointing\n    # -------------------------------\n    print(f\"Epoch {epoch+1}/{num_epochs} | \"\n          f\"Train Loss: {avg_train_loss:.4f} | \"\n          f\"Val Loss: {avg_val_loss:.4f}\")\n    \n    # Learning rate adjustment\n    scheduler.step(avg_val_loss)\n    \n    # Early stopping\n    # if avg_val_loss < best_val_loss:\n    #     best_val_loss = avg_val_loss\n    #     torch.save(model.state_dict(), \"/kaggle/working/best_model.pth\")\n    #     print(f\"✅ Best model saved (epoch {epoch+1})\")\n    #     no_improve = 0\n    # else:\n    #     no_improve += 1\n    #     if no_improve >= patience:\n    #         print(f\"⏹️ Early stopping triggered after epoch {epoch+1}\")\n    #         break\n\nprint(f\"\\n🏁 Training complete! Best validation loss: {best_val_loss:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T16:56:37.5814Z","iopub.execute_input":"2025-11-08T16:56:37.581756Z","iopub.status.idle":"2025-11-08T17:30:20.510312Z","shell.execute_reply.started":"2025-11-08T16:56:37.581727Z","shell.execute_reply":"2025-11-08T17:30:20.509431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"torch.save(model.state_dict(), \"/kaggle/working/best_model2.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T17:42:05.490555Z","iopub.execute_input":"2025-11-08T17:42:05.490974Z","iopub.status.idle":"2025-11-08T17:42:05.629443Z","shell.execute_reply.started":"2025-11-08T17:42:05.490942Z","shell.execute_reply":"2025-11-08T17:42:05.628796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\nall_preds = []\nall_labels = []\n\nwith torch.no_grad():\n    for volumes, labels in val_loader:\n        volumes = volumes.to(device)\n        \n        # Get predictions\n        outputs = model(volumes)\n        probs = torch.sigmoid(outputs)  # Convert logits to probabilities\n        \n        all_preds.append(probs.cpu().numpy())\n        all_labels.append(labels.numpy())\n\n# Combine all predictions and labels\npreds = np.vstack(all_preds)\nlabels = np.vstack(all_labels)\n\n# Calculate AUC for each label\nauc_scores = []\nprint(\"\\nPer-class AUC scores:\")\nfor i, col in enumerate(LABEL_COLS):\n    try:\n        auc = roc_auc_score(labels[:, i], preds[:, i])\n        auc_scores.append(auc)\n        print(f\"{col[:30]:<30} AUC: {auc:.4f}\")\n    except:\n        print(f\"{col[:30]:<30} AUC: Not enough samples\")\n\n# Overall mean AUC\nmean_auc = np.mean(auc_scores)\nprint(f\"\\nMean AUC across all labels: {mean_auc:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T17:42:08.461151Z","iopub.execute_input":"2025-11-08T17:42:08.46195Z","iopub.status.idle":"2025-11-08T17:42:12.8375Z","shell.execute_reply.started":"2025-11-08T17:42:08.46192Z","shell.execute_reply":"2025-11-08T17:42:12.836541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot training and validation loss curves\nplt.figure(figsize=(8, 5))\nplt.plot(train_losses, label=\"Train Loss\")\nplt.plot(val_losses, label=\"Validation Loss\")\nplt.title(\"Training vs Validation Loss\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T17:42:34.584717Z","iopub.execute_input":"2025-11-08T17:42:34.585073Z","iopub.status.idle":"2025-11-08T17:42:34.966211Z","shell.execute_reply.started":"2025-11-08T17:42:34.585046Z","shell.execute_reply":"2025-11-08T17:42:34.965276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"/kaggle/input/themodel/best_model.pth\", map_location=device))\nprint(\"✅ Loaded best checkpoint for fine-tuning\")\n\n# Reduce learning rate for fine-tuning\nfor param_group in optimizer.param_groups:\n    param_group['lr'] = 1e-4  # smaller LR for stable fine-tuning\n\nfine_tune_epochs = 15\nfor epoch in range(fine_tune_epochs):\n    model.train()\n    running_loss = 0.0\n    for volumes, labels in tqdm(train_loader, desc=f\"Fine-tuning Epoch {epoch+1}/{fine_tune_epochs}\"):\n        volumes, labels = volumes.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(volumes)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n    print(f\"Epoch {epoch+1} - Fine-tune Loss: {running_loss/len(train_loader):.4f}\")\n\ntorch.save(model.state_dict(), \"/kaggle/working/final_finetuned_model.pth\")\nprint(\"💾 Fine-tuned model saved as final_finetuned_model.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T17:43:24.019555Z","iopub.execute_input":"2025-11-08T17:43:24.019907Z","iopub.status.idle":"2025-11-08T17:55:59.50395Z","shell.execute_reply.started":"2025-11-08T17:43:24.019877Z","shell.execute_reply":"2025-11-08T17:55:59.503222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, roc_auc_score, f1_score\nimport torch\nimport numpy as np\nfrom tqdm import tqdm\n\nmodel.eval()\nall_labels = []\nall_preds = []\n\nwith torch.no_grad():\n    for volumes, labels in tqdm(val_loader, desc=\"Evaluating\"):\n        volumes = volumes.to(device)\n        labels = labels.to(device)\n\n        outputs = model(volumes)\n        preds = torch.sigmoid(outputs)        # convert logits → probabilities (0–1)\n\n        all_labels.append(labels.cpu().numpy())\n        all_preds.append(preds.cpu().numpy())\n\n# Combine all batches\nall_labels = np.vstack(all_labels)\nall_preds = np.vstack(all_preds)\n\n# ------------- METRICS -------------\n# 1️⃣ Threshold predictions at 0.5\nbinary_preds = (all_preds > 0.5).astype(int)\n\n# 2️⃣ Accuracy (average over all labels)\nacc = accuracy_score(all_labels.flatten(), binary_preds.flatten())\n\n# 3️⃣ AUC (per label, then average)\nauc = roc_auc_score(all_labels, all_preds, average=\"macro\")\n\n# 4️⃣ F1-score\nf1 = f1_score(all_labels.flatten(), binary_preds.flatten())\n\nprint(f\"\\n✅ Evaluation Metrics:\")\nprint(f\"Accuracy: {acc:.4f}\")\nprint(f\"Mean AUC: {auc:.4f}\")\nprint(f\"F1 Score: {f1:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T17:58:01.780244Z","iopub.execute_input":"2025-11-08T17:58:01.781132Z","iopub.status.idle":"2025-11-08T17:58:06.064686Z","shell.execute_reply.started":"2025-11-08T17:58:01.781107Z","shell.execute_reply":"2025-11-08T17:58:06.0638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from monai.losses import FocalLoss\n\nnum_epochs = 40\nlearning_rate = 1e-4\npatience = 6\nbest_auc = 0.0\nno_improve = 0\n\n# ==============================\n# ⚙️ Loss, Optimizer, Scheduler\n# ==============================\n# Use Focal Loss to handle class imbalance\ncriterion = FocalLoss(gamma=2.0)\noptimizer = optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=1e-4)\n\n# Scheduler to reduce LR if validation AUC stops improving\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, mode='max', factor=0.5, patience=3, verbose=True\n)\n\n# ==============================\n# 🚀 Training Loop\n# ==============================\nfor epoch in range(num_epochs):\n    model.train()\n    train_loss = 0.0\n    \n    train_bar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{num_epochs} [Train]\")\n    for volumes, labels in train_bar:\n        volumes, labels = volumes.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n        outputs = model(volumes)\n        loss = criterion(outputs, labels)\n        loss.backward()\n\n        # Gradient clipping\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 2.0)\n\n        optimizer.step()\n        train_loss += loss.item()\n\n        train_bar.set_postfix(loss=f\"{train_loss/len(train_bar):.4f}\")\n\n    avg_train_loss = train_loss / len(train_loader)\n\n    # ==============================\n    # 🔍 Validation\n    # ==============================\n    model.eval()\n    val_loss = 0.0\n    all_preds, all_labels = [], []\n\n    with torch.no_grad():\n        for volumes, labels in val_loader:\n            volumes, labels = volumes.to(device), labels.to(device)\n            outputs = model(volumes)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n\n            probs = torch.sigmoid(outputs)\n            all_preds.append(probs.cpu().numpy())\n            all_labels.append(labels.cpu().numpy())\n\n    avg_val_loss = val_loss / len(val_loader)\n    all_preds = np.vstack(all_preds)\n    all_labels = np.vstack(all_labels)\n\n    # Compute metrics\n    binary_preds = (all_preds > 0.5).astype(int)\n    acc = accuracy_score(all_labels.flatten(), binary_preds.flatten())\n    f1 = f1_score(all_labels.flatten(), binary_preds.flatten(), zero_division=0)\n    \n    try:\n        auc = roc_auc_score(all_labels, all_preds, average=\"macro\")\n    except ValueError:\n        auc = 0.0\n\n    print(f\"Epoch {epoch+1}/{num_epochs} | \"\n          f\"Train Loss: {avg_train_loss:.4f} | Val Loss: {avg_val_loss:.4f} | \"\n          f\"AUC: {auc:.4f} | F1: {f1:.4f} | Acc: {acc:.4f}\")\n\n    # Scheduler and early stopping\n    scheduler.step(auc)\n\n    if auc > best_auc:\n        best_auc = auc\n        torch.save(model.state_dict(), \"/kaggle/working/best_model_auc.pth\")\n        print(f\"✅ New best AUC: {auc:.4f} — model saved\")\n        no_improve = 0\n    else:\n        no_improve += 1\n        if no_improve >= patience:\n            print(f\"⏹️ Early stopping — no AUC improvement for {patience} epochs\")\n            break\n\nprint(f\"\\n🏁 Training complete! Best Validation AUC: {best_auc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-08T18:00:28.302475Z","iopub.execute_input":"2025-11-08T18:00:28.302787Z","iopub.status.idle":"2025-11-08T18:14:12.799146Z","shell.execute_reply.started":"2025-11-08T18:00:28.302767Z","shell.execute_reply":"2025-11-08T18:14:12.798233Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Another way","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport functools\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom typing import List, Tuple, Optional\nimport gc\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\n\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold\nfrom sklearn.metrics import roc_auc_score\nimport pydicom\n\nwarnings.filterwarnings('ignore')\n\ndef set_seed(seed=42):\n    \"\"\"Set all random seeds for reproducibility\"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nset_seed(42)\n\n# Device configuration\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\nif torch.cuda.is_available():\n    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n    print(f\"Memory: {torch.cuda.get_device_properties(0).total_memory / 1024**3:.1f} GB\")\n    print(f\"CUDA version: {torch.version.cuda}\")\n    torch.cuda.empty_cache()\nelse:\n    raise RuntimeError(\"CUDA is not available! This code requires GPU.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T17:15:21.523418Z","iopub.execute_input":"2025-11-09T17:15:21.523641Z","iopub.status.idle":"2025-11-09T17:16:07.820168Z","shell.execute_reply.started":"2025-11-09T17:15:21.523615Z","shell.execute_reply":"2025-11-09T17:16:07.819364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv')\nseries_uids_full = df_train['SeriesInstanceUID'].unique()\n\nif DEBUG:\n    print(\"DEBUG MODE: Using only first 10 series_uids\")\n    series_uids = series_uids_full[:10]\nelse:\n    print(\"PRODUCTION MODE: Using all series_uids\")\n    series_uids = series_uids_full\n\n# === Build map: SeriesInstanceUID → list of DICOM paths ===\nseries_dicom_map = {\n    si: glob.glob(os.path.join(rd, 'series', si, '*.dcm'))\n    for si in series_uids\n}\n\n# === Parallel sort each series ===\nwith Pool(cpu_count()) as pool:\n    sorted_results = list(tqdm(pool.imap(sort_series, series_dicom_map.items()),\n                               total=len(series_dicom_map),\n                               desc=\"Sorting DICOM series\"))\n\n# === Generate output rows ===\nrows = []\nfor series_uid, sorted_paths in tqdm(sorted_results, desc=\"Generating CSV rows\"):\n    modality = df_train[df_train['SeriesInstanceUID'] == series_uid]['Modality'].iloc[0]  # Get modality for the current series\n    for idx, path in enumerate(sorted_paths):\n        sop_uid = os.path.splitext(os.path.basename(path))[0]  # use filename as SOPInstanceUID\n        rows.append({\n            'SeriesInstanceUID': series_uid,\n            'SOPInstanceUID': sop_uid,\n            'dicom_filename': path,\n            'relative_index': idx,\n            'Modality': modality\n        })\n\n# === Save as DataFrame ===\ndf_series_index_mapping = pd.DataFrame(rows)\ndf_series_index_mapping = df_series_index_mapping.sort_values(\n    by=['SeriesInstanceUID', 'relative_index']\n)\ndf_series_index_mapping.to_csv('series_index_mapping.csv', index=False)\nprint(\"Saved series_index_mapping.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T17:25:00.89678Z","iopub.execute_input":"2025-11-09T17:25:00.897082Z","iopub.status.idle":"2025-11-09T17:25:00.929103Z","shell.execute_reply.started":"2025-11-09T17:25:00.89706Z","shell.execute_reply":"2025-11-09T17:25:00.928294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Config:\n    # Data paths\n    DATA_DIR = \"/kaggle/input/another-one\"\n    CVT_PNG_DIR = os.path.join(DATA_DIR, \"cvt_png\")\n    SERIES_MAPPING_PATH = os.path.join(DATA_DIR, \"series_index_mapping.csv\")\n    LOCALIZERS_PATH = os.path.join(DATA_DIR, \"train_localizers_with_relative.csv\")\n    TRAIN_CSV_PATH = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\n    \n    # Model parameters for 8-frame processing\n    NUM_FRAMES = 8\n    IMAGE_SIZE = 224\n    NUM_CLASSES = 14\n    BATCH_SIZE = 6  # Reduced for 8-frame processing\n    NUM_EPOCHS = 50\n    LEARNING_RATE = 5e-5\n    \n    # Model configuration\n    MODEL_NAME_BACKBONE = \"tf_efficientnetv2_s.in1k\"\n    USE_METADATA = True\n    USE_WINDOWING = True\n    USE_3CHANNEL_INPUT = True\n    USE_IMPROVED_LOSS = True\n    USE_CLAHE = True\n    USE_STRONG_AUGMENTATION = True\n    \n    # GPU optimization settings\n    NUM_WORKERS = 2\n    PIN_MEMORY = True\n    PREFETCH_FACTOR = 2\n    PERSISTENT_WORKERS = True\n    \n    # Training parameters with robust cross-validation\n    NUM_FOLDS = 5\n    FOLD = 0\n    ACCUMULATION_STEPS = 5  # Adjusted for smaller batch size\n    EARLY_STOPPING_PATIENCE = 3\n    USE_GROUP_CV = True\n    \n    # Data loading optimization\n    CACHE_SIZE = 100\n    \n    # Output\n    OUTPUT_DIR = \"/kaggle/working\"\n    MODEL_NAME = \"eightframe_efficientnetv2s\"\n\nconfig = Config()\n\nprint(\"=== Configuration Summary ===\")\nprint(f\"Model Backbone: {config.MODEL_NAME_BACKBONE}\")\nprint(f\"Number of Frames: {config.NUM_FRAMES}\")\nprint(f\"Batch Size: {config.BATCH_SIZE}\")\nprint(f\"Accumulation Steps: {config.ACCUMULATION_STEPS}\")\nprint(f\"Effective Batch Size: {config.BATCH_SIZE * config.ACCUMULATION_STEPS}\")\nprint(f\"CLAHE Enabled: {config.USE_CLAHE}\")\nprint(f\"Strong Augmentation: {config.USE_STRONG_AUGMENTATION}\")\nprint(f\"Group Cross-Validation: {config.USE_GROUP_CV}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:00:13.067873Z","iopub.execute_input":"2025-11-09T18:00:13.068136Z","iopub.status.idle":"2025-11-09T18:00:13.075262Z","shell.execute_reply.started":"2025-11-09T18:00:13.068116Z","shell.execute_reply":"2025-11-09T18:00:13.074359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Loading data...\")\ntrain_df = pd.read_csv(config.TRAIN_CSV_PATH)\nseries_mapping_df = pd.read_csv(config.SERIES_MAPPING_PATH)\nlocalizers_df = pd.read_csv(config.LOCALIZERS_PATH)\n\nprint(f\"Train data shape: {train_df.shape}\")\nprint(f\"Series mapping shape: {series_mapping_df.shape}\")\nprint(f\"Localizers shape: {localizers_df.shape}\")\n\n# Define target columns\nTARGET_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery', \n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery', \n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present'\n]\n\nprint(f\"Target columns: {len(TARGET_COLS)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:00:16.28447Z","iopub.execute_input":"2025-11-09T18:00:16.285239Z","iopub.status.idle":"2025-11-09T18:00:20.784886Z","shell.execute_reply.started":"2025-11-09T18:00:16.285208Z","shell.execute_reply":"2025-11-09T18:00:20.784061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_windowing_params(modality: str) -> Tuple[float, float]:\n    \"\"\"Get optimal windowing parameters for different modalities\"\"\"\n    windows = {\n        'CT': (40, 80),\n        'CTA': (50, 350), \n        'MRA': (600, 1200),\n        'MRI': (40, 80),\n        'MR': (40, 80)\n    }\n    return windows.get(modality, (40, 80))\n\ndef apply_dicom_windowing(img: np.ndarray, window_center: float, window_width: float) -> np.ndarray:\n    \"\"\"Apply DICOM windowing to normalize image intensities\"\"\"\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    img = (img - img_min) / (img_max - img_min + 1e-7)\n    return (img * 255).astype(np.uint8)\n\ndef apply_clahe_normalization(img: np.ndarray, modality: str) -> np.ndarray:\n    \"\"\"Apply CLAHE with modality-specific optimization\"\"\"\n    if not config.USE_CLAHE:\n        return img\n        \n    if modality in ['CTA', 'MRA']:\n        # Vascular imaging: stronger contrast improvement\n        clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n        img_clahe = cv2.convertScaleAbs(img_clahe, alpha=1.1, beta=5)\n    elif modality in ['MRI', 'MR']:\n        # MRI: gentler improvement with gamma correction\n        clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n        img_clahe = np.power(img_clahe / 255.0, 0.9) * 255\n        img_clahe = img_clahe.astype(np.uint8)\n    else:\n        # CT: standard CLAHE\n        clahe = cv2.createCLAHE(clipLimit=2.5, tileGridSize=(8, 8))\n        img_clahe = clahe.apply(img.astype(np.uint8))\n    \n    return img_clahe\n\ndef robust_normalization(volume: np.ndarray) -> np.ndarray:\n    \"\"\"Apply robust normalization using percentiles\"\"\"\n    p1, p99 = np.percentile(volume.flatten(), [1, 99])\n    volume_norm = np.clip(volume, p1, p99)\n    \n    if p99 > p1:\n        volume_norm = (volume_norm - p1) / (p99 - p1 + 1e-7)\n    else:\n        volume_norm = np.zeros_like(volume_norm)\n        \n    return (volume_norm * 255).astype(np.uint8)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:00:26.220104Z","iopub.execute_input":"2025-11-09T18:00:26.220758Z","iopub.status.idle":"2025-11-09T18:00:26.22886Z","shell.execute_reply.started":"2025-11-09T18:00:26.220733Z","shell.execute_reply":"2025-11-09T18:00:26.22821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_3channel_input_8frame(volume: np.ndarray) -> np.ndarray:\n    \"\"\"Create 3-channel input from 8-frame volume optimized for aneurysm detection\"\"\"\n    if len(volume) == 0:\n        return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n    \n    # Middle slice (most important for anatomical reference)\n    middle_slice = volume[len(volume) // 2]\n    \n    # Maximum Intensity Projection (MIP) - optimized for vascular structures\n    mip = np.max(volume, axis=0)\n    \n    # Standard deviation projection for texture analysis\n    std_proj = np.std(volume, axis=0).astype(np.float32)\n    \n    # Normalize standard deviation projection with robust method\n    if std_proj.max() > std_proj.min():\n        p1, p99 = np.percentile(std_proj, [5, 95])\n        std_proj = np.clip(std_proj, p1, p99)\n        std_proj = ((std_proj - p1) / (p99 - p1 + 1e-7) * 255).astype(np.uint8)\n    else:\n        std_proj = np.zeros_like(std_proj, dtype=np.uint8)\n    \n    return np.stack([middle_slice, mip, std_proj], axis=-1)\n\ndef smart_8_frame_sampling(volume_paths: List[str], series_uid: str = None) -> List[str]:\n    \"\"\"Intelligent 8-frame sampling strategy using every other frame\"\"\"\n    n = len(volume_paths)\n    \n    if n <= 8:\n        # If we have 8 or fewer frames, use all available\n        result = volume_paths[:]\n        # Pad with repetitions if needed\n        while len(result) < 8:\n            result.extend(volume_paths[:8-len(result)])\n        return result[:8]\n    \n    # Skip every other frame starting from a strategic position\n    # Start from 10% into the volume to avoid empty slices at the beginning\n    start_idx = max(0, int(n * 0.1))\n    \n    # Calculate step size to get 8 frames with good coverage\n    available_frames = n - start_idx\n    step = max(1, available_frames // 8)\n    \n    indices = []\n    current_idx = start_idx\n    while len(indices) < 8 and current_idx < n:\n        indices.append(current_idx)\n        current_idx += step\n    \n    # If we need more frames, fill from the remaining\n    while len(indices) < 8:\n        remaining = [i for i in range(n) if i not in indices]\n        if remaining:\n            indices.append(remaining[len(indices) % len(remaining)])\n        else:\n            indices.append(indices[-1])  # Duplicate last frame\n    \n    return [volume_paths[i] for i in indices[:8]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:00:31.15951Z","iopub.execute_input":"2025-11-09T18:00:31.15974Z","iopub.status.idle":"2025-11-09T18:00:31.168008Z","shell.execute_reply.started":"2025-11-09T18:00:31.159725Z","shell.execute_reply":"2025-11-09T18:00:31.167419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_dicom_patient_info(series_uid: str) -> Tuple[str, str]:\n    \"\"\"Extract StudyInstanceUID and PatientID from DICOM metadata\"\"\"\n    try:\n        dicom_dir = f\"/kaggle/input/rsna-intracranial-aneurysm-detection/series/{series_uid}\"\n        if os.path.exists(dicom_dir):\n            dcm_files = [f for f in os.listdir(dicom_dir) if f.endswith('.dcm')]\n            if dcm_files:\n                ds = pydicom.dcmread(\n                    os.path.join(dicom_dir, dcm_files[0]), \n                    stop_before_pixels=True, \n                    force=True\n                )\n                study_uid = getattr(ds, 'StudyInstanceUID', None)\n                patient_id = getattr(ds, 'PatientID', None)\n                return study_uid or f\"fallback_{series_uid[:32]}\", patient_id\n    except Exception:\n        pass\n    \n    # Fallback: use longer prefix from series UID\n    return f\"fallback_{series_uid[:32]}\", f\"fallback_{series_uid[:32]}\"\n\n@functools.lru_cache(maxsize=5000)\ndef get_patient_group_cached(series_uid: str) -> str:\n    \"\"\"Get patient group with caching for performance\"\"\"\n    study_uid, patient_id = extract_dicom_patient_info(series_uid)\n    # Use StudyInstanceUID as primary identifier\n    return study_uid if study_uid and not study_uid.startswith('fallback_') else patient_id\n\ndef create_frame_paths_8frame():\n    \"\"\"Create mapping from series to frame paths optimized for 8-frame processing\"\"\"\n    frame_paths = {}\n    \n    print(\"Creating 8-frame optimized paths from series_index_mapping.csv...\")\n    \n    for series_uid in tqdm(train_df['SeriesInstanceUID'].unique(), desc=\"Processing series\"):\n        # Get series data from mapping\n        series_data = series_mapping_df[series_mapping_df['SeriesInstanceUID'] == series_uid]\n        \n        if len(series_data) == 0:\n            frame_paths[series_uid] = []\n            continue\n            \n        # Get row from train_df to check which diseases are present\n        train_row = train_df[train_df['SeriesInstanceUID'] == series_uid].iloc[0]\n        \n        # Find any disease location that has this series\n        found_paths = []\n        \n        # Check each target column (except Aneurysm Present)\n        for target_col in TARGET_COLS[:-1]:\n            if train_row[target_col] == 1:\n                location_clean = target_col.replace('/', '_')\n                series_dir = os.path.join(config.CVT_PNG_DIR, location_clean, series_uid)\n                \n                if os.path.exists(series_dir):\n                    png_files = sorted(glob.glob(os.path.join(series_dir, \"*.png\")))\n                    if png_files:\n                        found_paths = png_files\n                        break\n        \n        # If no paths found in disease folders, use DICOM structure\n        if not found_paths:\n            dicom_dir = f\"/kaggle/input/rsna-intracranial-aneurysm-detection/series/{series_uid}\"\n            if os.path.exists(dicom_dir):\n                num_frames = len(series_data)\n                found_paths = [f\"dummy_path_{i:04d}.png\" for i in range(num_frames)]\n        \n        # Apply smart 8-frame sampling\n        if found_paths:\n            found_paths = smart_8_frame_sampling(found_paths, series_uid)\n        \n        frame_paths[series_uid] = found_paths\n    \n    return frame_paths\n\n# Create optimized paths\nframe_paths_dict = create_frame_paths_8frame()\nprint(f\"Created 8-frame optimized paths for {len(frame_paths_dict)} series\")\n\n# Filter data\nvalid_series = [uid for uid, paths in frame_paths_dict.items() if len(paths) > 0]\ntrain_df_filtered = train_df[train_df['SeriesInstanceUID'].isin(valid_series)].copy()\nprint(f\"Filtered train data shape: {train_df_filtered.shape}\")\n\n# Check distribution\naneurysm_dist_filtered = train_df_filtered['Aneurysm Present'].value_counts()\nprint(f\"Aneurysm Present distribution: {aneurysm_dist_filtered.to_dict()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:00:34.112603Z","iopub.execute_input":"2025-11-09T18:00:34.113113Z","iopub.status.idle":"2025-11-09T18:05:52.950491Z","shell.execute_reply.started":"2025-11-09T18:00:34.113088Z","shell.execute_reply":"2025-11-09T18:05:52.94971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_robust_cv_split(train_df, n_splits=5):\n    \"\"\"Create robust cross-validation split with true patient separation from DICOM\"\"\"\n    \n    print(\"Creating patient-separated cross-validation split...\")\n    print(\"Extracting true patient IDs from DICOM metadata...\")\n    print(\"This will take a few minutes but ensures proper patient separation.\")\n    \n    # Extract true patient groups from DICOM metadata\n    patient_groups = []\n    for series_uid in tqdm(train_df['SeriesInstanceUID'], desc=\"Reading DICOM patient info\"):\n        patient_group = get_patient_group_cached(series_uid)\n        patient_groups.append(patient_group)\n    \n    # Add patient groups to dataframe\n    train_df = train_df.copy()\n    train_df['patient_id'] = patient_groups\n    \n    n_groups = train_df['patient_id'].nunique()\n    print(f\"True patient groups found: {n_groups}\")\n    \n    # Check if we have enough patient groups\n    if n_groups < n_splits:\n        print(f\"Not enough patient groups ({n_groups}) for {n_splits}-fold CV.\")\n        print(\"Falling back to StratifiedKFold...\")\n        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n        return list(skf.split(train_df, train_df['Aneurysm Present']))\n    \n    # Create stratification key combining modality and aneurysm presence\n    train_df['stratify_key'] = (\n        train_df['Modality'].astype(str) + '_' + \n        train_df['Aneurysm Present'].astype(str)\n    )\n    \n    print(f\"Stratification keys: {train_df['stratify_key'].unique()}\")\n    \n    # Use GroupKFold to ensure patient-level separation\n    group_kfold = GroupKFold(n_splits=n_splits)\n    \n    splits = []\n    for fold_idx, (train_idx, val_idx) in enumerate(group_kfold.split(\n        train_df, \n        groups=train_df['patient_id']\n    )):\n        # Validate patient separation\n        train_fold = train_df.iloc[train_idx]\n        val_fold = train_df.iloc[val_idx]\n        \n        # Check for patient overlap (should be 0)\n        train_patients = set(train_fold['patient_id'])\n        val_patients = set(val_fold['patient_id'])\n        overlap = train_patients.intersection(val_patients)\n        \n        train_dist = train_fold['Aneurysm Present'].value_counts(normalize=True)\n        val_dist = val_fold['Aneurysm Present'].value_counts(normalize=True)\n        \n        print(f\"Fold {fold_idx}:\")\n        print(f\"  Train: {len(train_fold)} samples ({len(train_patients)} patients)\")\n        print(f\"  Val: {len(val_fold)} samples ({len(val_patients)} patients)\")\n        print(f\"  Patient overlap: {len(overlap)} (should be 0!)\")\n        print(f\"  Aneurysm Present - Train: {train_dist.get(1, 0):.3f}, Val: {val_dist.get(1, 0):.3f}\")\n        \n        if len(overlap) > 0:\n            print(f\"  WARNING: Found {len(overlap)} overlapping patients!\")\n        \n        splits.append((train_idx, val_idx))\n    \n    return splits\n\n# Create robust train/validation split\ncv_splits = create_robust_cv_split(train_df_filtered, config.NUM_FOLDS)\ntrain_indices, val_indices = cv_splits[config.FOLD]\n\ntrain_fold_df = train_df_filtered.iloc[train_indices]\nval_fold_df = train_df_filtered.iloc[val_indices]\n\nprint(f\"\\nRobust CV Fold {config.FOLD} Summary:\")\nprint(f\"Train fold size: {len(train_fold_df)}\")\nprint(f\"Validation fold size: {len(val_fold_df)}\")\n\n# Check distributions\nprint(f\"Train Aneurysm Present: {train_fold_df['Aneurysm Present'].value_counts().to_dict()}\")\nprint(f\"Val Aneurysm Present: {val_fold_df['Aneurysm Present'].value_counts().to_dict()}\")\n\n# Check modality distribution\nprint(f\"Train Modality distribution: {train_fold_df['Modality'].value_counts().to_dict()}\")\nprint(f\"Val Modality distribution: {val_fold_df['Modality'].value_counts().to_dict()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:07:43.857014Z","iopub.execute_input":"2025-11-09T18:07:43.85775Z","iopub.status.idle":"2025-11-09T18:13:43.469482Z","shell.execute_reply.started":"2025-11-09T18:07:43.857724Z","shell.execute_reply":"2025-11-09T18:13:43.468687Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if config.USE_STRONG_AUGMENTATION:\n    print(\"Using strong augmentation for better generalization...\")\n    train_transform = A.Compose([\n        # Geometric transformations (safe for medical images)\n        A.Rotate(limit=15, p=0.7),\n        A.HorizontalFlip(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=10, p=0.6),\n        \n        # Advanced geometric transformations for robustness\n        A.ElasticTransform(alpha=50, sigma=5, p=0.3),\n        A.GridDistortion(num_steps=3, distort_limit=0.1, p=0.3),\n        \n        # Image quality variations (simulate different scanners/protocols)\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.6),\n        A.CLAHE(clip_limit=2.0, tile_grid_size=(8,8), p=0.5),\n        A.RandomGamma(gamma_limit=(80, 120), p=0.4),\n        \n        # Noise simulation (scanner differences)\n        A.GaussNoise(var_limit=(10, 80), p=0.4),\n        A.ISONoise(color_shift=(0.01, 0.05), intensity=(0.1, 0.5), p=0.3),\n        A.Blur(blur_limit=3, p=0.2),\n        \n        # Medical image specific augmentations\n        A.CoarseDropout(max_holes=8, max_height=32, max_width=32, p=0.3),\n        \n        # Normalization\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\nelse:\n    print(\"Using standard augmentation...\")\n    train_transform = A.Compose([\n        A.Rotate(limit=10, p=0.5),\n        A.HorizontalFlip(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.3),\n        A.GaussNoise(var_limit=(10, 50), p=0.2),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n\nval_transform = A.Compose([\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:20:07.159442Z","iopub.execute_input":"2025-11-09T18:20:07.159736Z","iopub.status.idle":"2025-11-09T18:20:07.178775Z","shell.execute_reply.started":"2025-11-09T18:20:07.159717Z","shell.execute_reply":"2025-11-09T18:20:07.178132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EightFrameDataset(Dataset):\n    \"\"\"Dataset optimized for 8-frame processing with CLAHE\"\"\"\n    def __init__(self, df, frame_paths_dict, series_mapping_df, num_frames=8, \n                 transform=None, is_training=True):\n        self.df = df.reset_index(drop=True)\n        self.frame_paths_dict = frame_paths_dict\n        self.series_mapping_df = series_mapping_df\n        self.num_frames = num_frames\n        self.transform = transform\n        self.is_training = is_training\n        \n        # Simple LRU cache for recently accessed data\n        self._cache = {}\n        self._cache_keys = []\n        self._max_cache_size = config.CACHE_SIZE\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        # Check cache first\n        if idx in self._cache:\n            return self._cache[idx]\n        \n        row = self.df.iloc[idx]\n        series_uid = row['SeriesInstanceUID']\n        \n        # Get labels\n        labels = torch.tensor(row[TARGET_COLS].values.astype(np.float32))\n        \n        # Extract metadata\n        metadata = self._extract_metadata(row)\n        \n        # Load 8-frame 3-channel image\n        image = self._load_8frame_3channel_image(series_uid, row)\n        \n        result = (image, labels, metadata)\n        \n        # Update cache\n        self._update_cache(idx, result)\n        \n        return result\n    \n    def _update_cache(self, idx, data):\n        \"\"\"Update LRU cache\"\"\"\n        if len(self._cache) >= self._max_cache_size:\n            # Remove oldest entry\n            oldest_idx = self._cache_keys.pop(0)\n            del self._cache[oldest_idx]\n        \n        self._cache[idx] = data\n        self._cache_keys.append(idx)\n    \n    def _extract_metadata(self, row) -> torch.Tensor:\n        \"\"\"Extract and normalize metadata\"\"\"\n        if not config.USE_METADATA:\n            return torch.tensor([0.0, 0.0], dtype=torch.float32)\n        \n        # Age processing\n        age = row.get('PatientAge', 50)\n        if pd.isna(age):\n            age = 50\n        elif isinstance(age, str):\n            age = int(''.join(filter(str.isdigit, age[:3])) or '50')\n        age = min(float(age), 100.0) / 100.0\n        \n        # Sex processing\n        sex = row.get('PatientSex', 'M')\n        sex = 1.0 if sex == 'M' else 0.0\n        \n        return torch.tensor([age, sex], dtype=torch.float32)\n    \n    def _load_8frame_3channel_image(self, series_uid: str, row) -> torch.Tensor:\n        \"\"\"Load 8-frame 3-channel image with processing\"\"\"\n        paths = self.frame_paths_dict.get(series_uid, [])\n        \n        try:\n            if len(paths) == 0 or paths[0].startswith('dummy_path'):\n                volume = self._load_volume_from_dicom_8frame(series_uid, row)\n            else:\n                volume = self._load_volume_from_png_8frame(paths)\n            \n            # Apply robust normalization\n            volume = robust_normalization(volume)\n            \n            # Create 3-channel input optimized for 8 frames\n            image = create_3channel_input_8frame(volume)\n            \n            # Apply transforms\n            if self.transform:\n                transformed = self.transform(image=image)\n                image = transformed['image']\n            \n            return image\n            \n        except Exception as e:\n            print(f\"Error loading {series_uid}: {e}\")\n            # Return dummy image\n            dummy_image = np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE, 3), dtype=np.uint8)\n            if self.transform:\n                transformed = self.transform(image=dummy_image)\n                return transformed['image']\n            return torch.zeros(3, config.IMAGE_SIZE, config.IMAGE_SIZE)\n    \n    def _load_volume_from_png_8frame(self, paths: List[str]) -> np.ndarray:\n        \"\"\"Load PNG volume optimized for 8 frames\"\"\"\n        volume = []\n        \n        # Ensure we have exactly 8 paths\n        if len(paths) != 8:\n            paths = smart_8_frame_sampling(paths)\n        \n        for path in paths:\n            try:\n                img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)\n                if img is not None:\n                    img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), \n                                   interpolation=cv2.INTER_AREA)\n                    volume.append(img)\n            except:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        return np.array(volume) if volume else np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n    \n    def _load_volume_from_dicom_8frame(self, series_uid: str, row) -> np.ndarray:\n        \"\"\"Load DICOM volume optimized for 8 frames with CLAHE\"\"\"\n        series_data = self.series_mapping_df[\n            self.series_mapping_df['SeriesInstanceUID'] == series_uid\n        ].sort_values('relative_index')\n        \n        if len(series_data) == 0:\n            return np.zeros((8, config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8)\n        \n        volume = []\n        modality = row.get('Modality', 'CT')\n        \n        # Sample exactly 8 slices using every-other-frame strategy\n        if len(series_data) <= 8:\n            sampled_data = series_data\n        else:\n            # Apply smart 8-frame sampling logic to indices\n            all_indices = list(range(len(series_data)))\n            sampled_indices = smart_8_frame_sampling([str(i) for i in all_indices])\n            sampled_indices = [int(i) for i in sampled_indices]\n            sampled_data = series_data.iloc[sampled_indices]\n        \n        for _, dicom_row in sampled_data.iterrows():\n            try:\n                ds = pydicom.dcmread(dicom_row['dicom_filename'])\n                img = ds.pixel_array.astype(np.float32)\n                \n                # Handle multi-frame/color images\n                if img.ndim == 3:\n                    if img.shape[-1] == 3:\n                        img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2GRAY).astype(np.float32)\n                    else:\n                        img = img[:, :, 0]\n                \n                # Apply rescale if available\n                if hasattr(ds, 'RescaleSlope') and hasattr(ds, 'RescaleIntercept'):\n                    img = img * ds.RescaleSlope + ds.RescaleIntercept\n                \n                # Apply windowing\n                if config.USE_WINDOWING:\n                    window_center, window_width = get_windowing_params(modality)\n                    img = apply_dicom_windowing(img, window_center, window_width)\n                else:\n                    img_min, img_max = img.min(), img.max()\n                    if img_max > img_min:\n                        img = ((img - img_min) / (img_max - img_min) * 255).astype(np.uint8)\n                    else:\n                        img = np.zeros_like(img, dtype=np.uint8)\n                \n                # Apply CLAHE improvement\n                img = apply_clahe_normalization(img, modality)\n                \n                # High quality resize\n                img = cv2.resize(img, (config.IMAGE_SIZE, config.IMAGE_SIZE), \n                               interpolation=cv2.INTER_AREA)\n                volume.append(img)\n                \n            except Exception as e:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n                continue\n        \n        # Ensure we have exactly 8 frames\n        while len(volume) < 8:\n            if volume:\n                volume.append(volume[-1])  # Duplicate last frame\n            else:\n                volume.append(np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.uint8))\n        \n        return np.array(volume[:8])  # Take exactly 8 frames","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:20:29.535753Z","iopub.execute_input":"2025-11-09T18:20:29.53635Z","iopub.status.idle":"2025-11-09T18:20:29.555177Z","shell.execute_reply.started":"2025-11-09T18:20:29.536327Z","shell.execute_reply":"2025-11-09T18:20:29.554273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Creating 8-frame datasets with CLAHE...\")\ntrain_dataset = EightFrameDataset(\n    train_fold_df, \n    frame_paths_dict, \n    series_mapping_df,\n    num_frames=config.NUM_FRAMES,\n    transform=train_transform,\n    is_training=True\n)\n\nval_dataset = EightFrameDataset(\n    val_fold_df,\n    frame_paths_dict,\n    series_mapping_df,\n    num_frames=config.NUM_FRAMES, \n    transform=val_transform,\n    is_training=False\n)\n\n# Create optimized data loaders\nprint(\"Creating optimized data loaders for 8-frame processing...\")\ntrain_loader = DataLoader(\n    train_dataset,\n    batch_size=config.BATCH_SIZE,\n    shuffle=True,\n    num_workers=config.NUM_WORKERS,\n    pin_memory=config.PIN_MEMORY,\n    drop_last=True,\n    prefetch_factor=config.PREFETCH_FACTOR,\n    persistent_workers=config.PERSISTENT_WORKERS\n)\n\nval_loader = DataLoader(\n    val_dataset,\n    batch_size=config.BATCH_SIZE,\n    shuffle=False,\n    num_workers=config.NUM_WORKERS,\n    pin_memory=config.PIN_MEMORY,\n    prefetch_factor=config.PREFETCH_FACTOR,\n    persistent_workers=config.PERSISTENT_WORKERS\n)\n\nprint(f\"Train batches: {len(train_loader)}\")\nprint(f\"Validation batches: {len(val_loader)}\")\n\n# Test 8-frame data loading speed\nprint(\"Testing 8-frame data loading speed...\")\nimport time\n\nstart_time = time.time()\nfor i, batch in enumerate(train_loader):\n    if i >= 5:  # Test first 5 batches\n        break\n    images, labels, metadata = batch\n    print(f\"Batch {i+1}: Images shape: {images.shape}, Device: {images.device}\")\n\nelapsed = time.time() - start_time\nprint(f\"Loaded 5 batches in {elapsed:.2f} seconds ({elapsed/5:.2f}s per batch)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:20:48.822736Z","iopub.execute_input":"2025-11-09T18:20:48.823416Z","iopub.status.idle":"2025-11-09T18:20:53.866822Z","shell.execute_reply.started":"2025-11-09T18:20:48.823394Z","shell.execute_reply":"2025-11-09T18:20:53.865499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ImprovedMultiFrameModel(nn.Module):\n    \"\"\"Model with EfficientNetV2-S and metadata integration for 8-frame processing\"\"\"\n    def __init__(self, num_frames=8, num_classes=14, pretrained=True):\n        super(ImprovedMultiFrameModel, self).__init__()\n        self.num_frames = num_frames\n        self.num_classes = num_classes\n        self.use_3channel = config.USE_3CHANNEL_INPUT\n        self.use_metadata = config.USE_METADATA\n        \n        # Backbone: EfficientNetV2-S\n        print(f\"Loading backbone: {config.MODEL_NAME_BACKBONE}\")\n        self.backbone = timm.create_model(\n            config.MODEL_NAME_BACKBONE,\n            pretrained=pretrained,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        self.feature_dim = self.backbone.num_features\n        print(f\"Backbone {config.MODEL_NAME_BACKBONE}: {self.feature_dim} features\")\n        \n        # Metadata processing\n        if self.use_metadata:\n            self.meta_fc = nn.Sequential(\n                nn.Linear(2, 16),\n                nn.ReLU(),\n                nn.Dropout(0.2),\n                nn.Linear(16, 32),\n                nn.ReLU()\n            )\n            classifier_input_dim = self.feature_dim + 32\n        else:\n            classifier_input_dim = self.feature_dim\n        \n        # Classifier\n        self.classifier = nn.Sequential(\n            nn.Linear(classifier_input_dim, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, num_classes)\n        )\n        \n    def forward(self, x, meta=None):\n        # 3-channel input processing (optimized for 8-frame data)\n        features = self.backbone(x)  # (batch_size, feature_dim)\n        \n        # Metadata integration\n        if self.use_metadata and meta is not None:\n            meta_features = self.meta_fc(meta)\n            features = torch.cat([features, meta_features], dim=1)\n        \n        # Classification\n        output = self.classifier(features)\n        return output\n\n# Initialize 8-frame model\nprint(\"Initializing 8-frame model...\")\nmodel = ImprovedMultiFrameModel(\n    num_frames=config.NUM_FRAMES,\n    num_classes=config.NUM_CLASSES,\n    pretrained=True\n)\n\nmodel = model.to(device)\n\n# Count parameters\ntotal_params = sum(p.numel() for p in model.parameters())\ntrainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nprint(f\"Total parameters: {total_params:,}\")\nprint(f\"Trainable parameters: {trainable_params:,}\")\nprint(f\"Model device: {next(model.parameters()).device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:22:27.460326Z","iopub.execute_input":"2025-11-09T18:22:27.460571Z","iopub.status.idle":"2025-11-09T18:22:29.591416Z","shell.execute_reply.started":"2025-11-09T18:22:27.460555Z","shell.execute_reply":"2025-11-09T18:22:29.590526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    \"\"\"Focal Loss for addressing class imbalance\"\"\"\n    def __init__(self, alpha=1, gamma=2):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        \n    def forward(self, inputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none')\n        pt = torch.exp(-bce_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * bce_loss\n        return focal_loss.mean()\n\nclass WeightedMultiLabelLoss(nn.Module):\n    \"\"\"Weighted multi-label loss\"\"\"\n    def __init__(self, aneurysm_weight=3.0):\n        super(WeightedMultiLabelLoss, self).__init__()\n        self.weights = torch.ones(config.NUM_CLASSES, device=device)\n        self.weights[-1] = aneurysm_weight\n        \n    def forward(self, outputs, targets):\n        bce_loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction='none')\n        weighted_loss = bce_loss * self.weights\n        return weighted_loss.mean()\n\nclass ImprovedLoss(nn.Module):\n    \"\"\"Advanced combined loss function\"\"\"\n    def __init__(self, aneurysm_weight=3.0, focal_weight=0.3):\n        super(ImprovedLoss, self).__init__()\n        self.aneurysm_weight = aneurysm_weight\n        self.focal_weight = focal_weight\n        \n        self.weights = torch.ones(config.NUM_CLASSES, device=device)\n        self.weights[-1] = aneurysm_weight\n        \n        self.focal_loss = FocalLoss(alpha=1, gamma=2)\n        \n    def forward(self, outputs, targets):\n        # Weighted BCE\n        bce_loss = F.binary_cross_entropy_with_logits(outputs, targets, reduction='none')\n        weighted_bce = (bce_loss * self.weights).mean()\n        \n        # Focal Loss\n        focal_loss = self.focal_loss(outputs, targets)\n        \n        # Combination\n        return (1 - self.focal_weight) * weighted_bce + self.focal_weight * focal_loss\n\ndef get_loss_function():\n    \"\"\"Get loss function based on configuration\"\"\"\n    if config.USE_IMPROVED_LOSS:\n        return ImprovedLoss(aneurysm_weight=3.0, focal_weight=0.3)\n    else:\n        return WeightedMultiLabelLoss(aneurysm_weight=3.0)\n\ndef calculate_competition_metric(y_true, y_pred):\n    \"\"\"Calculate competition metric: weighted multilabel AUC ROC\"\"\"\n    individual_aucs = []\n    \n    # Calculate AUC for first 13 classes\n    for i in range(13):\n        try:\n            if len(np.unique(y_true[:, i])) > 1:\n                auc = roc_auc_score(y_true[:, i], y_pred[:, i])\n            else:\n                auc = 0.5\n            individual_aucs.append(auc)\n        except:\n            individual_aucs.append(0.5)\n    \n    # Calculate AUC for Aneurysm Present\n    try:\n        if len(np.unique(y_true[:, 13])) > 1:\n            aneurysm_present_auc = roc_auc_score(y_true[:, 13], y_pred[:, 13])\n        else:\n            aneurysm_present_auc = 0.5\n    except:\n        aneurysm_present_auc = 0.5\n    \n    # Final score\n    avg_individual = np.mean(individual_aucs)\n    final_score = (aneurysm_present_auc + avg_individual) / 2\n    \n    return final_score, aneurysm_present_auc, avg_individual, individual_aucs\n\n# Training setup\ncriterion = get_loss_function()\noptimizer = AdamW(model.parameters(), lr=config.LEARNING_RATE, weight_decay=1e-4)\nscheduler = CosineAnnealingLR(optimizer, T_max=config.NUM_EPOCHS, eta_min=1e-6)\n\n# Mixed precision training\nscaler = torch.cuda.amp.GradScaler()\n\nprint(\"Training setup complete\")\nprint(f\"Using loss function: {type(criterion).__name__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:22:37.195581Z","iopub.execute_input":"2025-11-09T18:22:37.195959Z","iopub.status.idle":"2025-11-09T18:22:37.211401Z","shell.execute_reply.started":"2025-11-09T18:22:37.195939Z","shell.execute_reply":"2025-11-09T18:22:37.21064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_epoch_optimized(model, train_loader, criterion, optimizer, scaler, device, accumulation_steps):\n    \"\"\"Optimized training function for 8-frame processing\"\"\"\n    model.train()\n    running_loss = 0.0\n    \n    optimizer.zero_grad()\n    \n    for batch_idx, (images, targets, metadata) in enumerate(tqdm(train_loader, desc=\"Training 8-Frame\")):\n        # Move data to GPU efficiently\n        images = images.to(device, non_blocking=True)\n        targets = targets.to(device, non_blocking=True)\n        metadata = metadata.to(device, non_blocking=True)\n        \n        with torch.cuda.amp.autocast():\n            outputs = model(images, metadata)\n            loss = criterion(outputs, targets)\n            loss = loss / accumulation_steps\n        \n        scaler.scale(loss).backward()\n        \n        if (batch_idx + 1) % accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n        \n        running_loss += loss.item() * accumulation_steps\n    \n    # Handle remaining gradients\n    if len(train_loader) % accumulation_steps != 0:\n        scaler.step(optimizer)\n        scaler.update()\n        optimizer.zero_grad()\n    \n    return running_loss / len(train_loader)\n\ndef validate_epoch_optimized(model, val_loader, criterion, device):\n    \"\"\"Optimized validation function for 8-frame processing\"\"\"\n    model.eval()\n    running_loss = 0.0\n    all_outputs = []\n    all_targets = []\n    \n    with torch.no_grad():\n        for images, targets, metadata in tqdm(val_loader, desc=\"Validating 8-Frame\"):\n            # Move data to GPU efficiently\n            images = images.to(device, non_blocking=True)\n            targets = targets.to(device, non_blocking=True)\n            metadata = metadata.to(device, non_blocking=True)\n                \n            with torch.cuda.amp.autocast():\n                logits = model(images, metadata)\n                loss = criterion(logits, targets)\n            \n            outputs = torch.sigmoid(logits)\n            \n            running_loss += loss.item()\n            all_outputs.append(outputs.cpu().numpy())\n            all_targets.append(targets.cpu().numpy())\n    \n    all_outputs = np.concatenate(all_outputs)\n    all_targets = np.concatenate(all_targets)\n    \n    final_score, aneurysm_auc, avg_individual, individual_aucs = calculate_competition_metric(\n        all_targets, all_outputs\n    )\n    \n    return running_loss / len(val_loader), final_score, aneurysm_auc, avg_individual\n\ndef check_gpu_utilization():\n    \"\"\"Check current GPU memory usage\"\"\"\n    if torch.cuda.is_available():\n        allocated = torch.cuda.memory_allocated() / 1024**3\n        reserved = torch.cuda.memory_reserved() / 1024**3\n        max_memory = torch.cuda.get_device_properties(0).total_memory / 1024**3\n        print(f\"GPU Memory - Allocated: {allocated:.2f}GB, Reserved: {reserved:.2f}GB, Total: {max_memory:.2f}GB\")\n        utilization = (allocated/max_memory)*100\n        print(f\"GPU Utilization: {utilization:.1f}%\")\n        return utilization\n    return 0\n\nprint(\"Initial GPU status for 8-frame processing:\")\ncheck_gpu_utilization()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:22:55.057196Z","iopub.execute_input":"2025-11-09T18:22:55.057505Z","iopub.status.idle":"2025-11-09T18:22:55.073872Z","shell.execute_reply.started":"2025-11-09T18:22:55.057484Z","shell.execute_reply":"2025-11-09T18:22:55.07292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training loop with 8-frame optimization and true patient separation\nbest_score = 0.0\nbest_epoch = 0\npatience_counter = 0\ntrain_losses = []\nval_losses = []\nval_scores = []\n\nprint(\"Starting 8-frame training with patient-separated CV...\")\nprint(f\"Batch size: {config.BATCH_SIZE}, Workers: {config.NUM_WORKERS}\")\nprint(f\"Frames per sample: {config.NUM_FRAMES}\")\nprint(f\"CLAHE enabled: {config.USE_CLAHE}\")\nprint(f\"Strong augmentation: {config.USE_STRONG_AUGMENTATION}\")\nprint(f\"True patient separation: {config.USE_GROUP_CV}\")\n\nfor epoch in range(config.NUM_EPOCHS):\n    print(f\"\\nEpoch {epoch+1}/{config.NUM_EPOCHS}\")\n    print(\"-\" * 50)\n    \n    # Training with 8-frame processing\n    train_loss = train_epoch_optimized(\n        model, train_loader, criterion, optimizer, scaler, device, config.ACCUMULATION_STEPS\n    )\n    \n    # Validation with 8-frame processing\n    val_loss, val_score, aneurysm_auc, avg_individual = validate_epoch_optimized(\n        model, val_loader, criterion, device\n    )\n    \n    # Learning rate scheduling\n    scheduler.step()\n    \n    # Log metrics\n    train_losses.append(train_loss)\n    val_losses.append(val_loss)\n    val_scores.append(val_score)\n    \n    print(f\"Train Loss: {train_loss:.6f}\")\n    print(f\"Val Loss: {val_loss:.6f}\")\n    print(f\"Val Score: {val_score:.6f}\")\n    print(f\"Aneurysm AUC: {aneurysm_auc:.6f}\")\n    print(f\"Avg Individual AUC: {avg_individual:.6f}\")\n    print(f\"Learning Rate: {optimizer.param_groups[0]['lr']:.8f}\")\n    \n    # GPU utilization\n    gpu_util = check_gpu_utilization()\n    \n    # Early stopping and model saving\n    if val_score > best_score:\n        best_score = val_score\n        best_epoch = epoch + 1\n        patience_counter = 0\n        \n        # Save model\n        model_path = os.path.join(config.OUTPUT_DIR, f\"{config.MODEL_NAME}_best.pth\")\n        torch.save({\n            'epoch': epoch + 1,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'scheduler_state_dict': scheduler.state_dict(),\n            'best_score': best_score,\n            'val_loss': val_loss,\n            'aneurysm_auc': aneurysm_auc,\n            'avg_individual_auc': avg_individual,\n            'config': config,\n            'model_config': {\n                'backbone': config.MODEL_NAME_BACKBONE,\n                'num_frames': config.NUM_FRAMES,\n                'use_3channel': config.USE_3CHANNEL_INPUT,\n                'use_metadata': config.USE_METADATA,\n                'use_windowing': config.USE_WINDOWING,\n                'use_improved_loss': config.USE_IMPROVED_LOSS,\n                'use_clahe': config.USE_CLAHE,\n                'use_strong_augmentation': config.USE_STRONG_AUGMENTATION,\n                'use_group_cv': config.USE_GROUP_CV\n            }\n        }, model_path)\n        \n        print(f\"New best model saved! Score: {best_score:.6f}\")\n    else:\n        patience_counter += 1\n        print(f\"No improvement. Patience: {patience_counter}/{config.EARLY_STOPPING_PATIENCE}\")\n        \n        if patience_counter >= config.EARLY_STOPPING_PATIENCE:\n            print(f\"Early stopping triggered at epoch {epoch + 1}\")\n            break\n    \n    # Memory cleanup\n    torch.cuda.empty_cache()\n\nprint(\"\\n\" + \"=\"*70)\nprint(\"8-FRAME TRAINING WITH PATIENT SEPARATION COMPLETED\")\nprint(\"=\"*70)\nprint(f\"Best Score: {best_score:.6f} at Epoch {best_epoch}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T18:23:42.131123Z","iopub.execute_input":"2025-11-09T18:23:42.131727Z","iopub.status.idle":"2025-11-09T20:56:20.125038Z","shell.execute_reply.started":"2025-11-09T18:23:42.131706Z","shell.execute_reply":"2025-11-09T20:56:20.123926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}