{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"}],"dockerImageVersionId":31155,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ndf = pd.read_csv(\"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\")\ndf.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:06:39.480949Z","iopub.execute_input":"2025-10-07T06:06:39.481174Z","iopub.status.idle":"2025-10-07T06:06:41.662731Z","shell.execute_reply.started":"2025-10-07T06:06:39.481151Z","shell.execute_reply":"2025-10-07T06:06:41.661947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.Modality.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:07:18.310237Z","iopub.execute_input":"2025-10-07T06:07:18.310518Z","iopub.status.idle":"2025-10-07T06:07:18.323809Z","shell.execute_reply.started":"2025-10-07T06:07:18.310496Z","shell.execute_reply":"2025-10-07T06:07:18.323162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[\"Aneurysm Present\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:07:57.602919Z","iopub.execute_input":"2025-10-07T06:07:57.603179Z","iopub.status.idle":"2025-10-07T06:07:57.612173Z","shell.execute_reply.started":"2025-10-07T06:07:57.60316Z","shell.execute_reply":"2025-10-07T06:07:57.611467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df2 = pd.read_csv(\"/kaggle/input/rsna-intracranial-aneurysm-detection/train_localizers.csv\")\ndf2.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:12:21.096057Z","iopub.execute_input":"2025-10-07T06:12:21.096702Z","iopub.status.idle":"2025-10-07T06:12:21.130889Z","shell.execute_reply.started":"2025-10-07T06:12:21.096674Z","shell.execute_reply":"2025-10-07T06:12:21.130176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.groupby('Modality')['Aneurysm Present'].value_counts().unstack(fill_value=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:16:33.414859Z","iopub.execute_input":"2025-10-07T06:16:33.415364Z","iopub.status.idle":"2025-10-07T06:16:33.478073Z","shell.execute_reply.started":"2025-10-07T06:16:33.415339Z","shell.execute_reply":"2025-10-07T06:16:33.477359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['SeriesInstanceUID'].nunique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:17:28.425775Z","iopub.execute_input":"2025-10-07T06:17:28.426069Z","iopub.status.idle":"2025-10-07T06:17:28.432325Z","shell.execute_reply.started":"2025-10-07T06:17:28.426049Z","shell.execute_reply":"2025-10-07T06:17:28.431692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport random\nimport json\nimport ast\nimport copy\nimport time\nfrom typing import Optional, Tuple, Dict, Any, List\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import models, transforms\n\nfrom sklearn.metrics import roc_auc_score, f1_score, precision_score, recall_score, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# --------------------------\n# CONFIG - edit paths here\n# --------------------------\nCONFIG = {\n    \"data_root\": \"/kaggle/input/rsna-intracranial-aneurysm-detection\",  # root folder containing 'series', 'train.csv', 'train_localizers.csv', 'segmentations' if present\n    \"series_folder\": \"series\",\n    \"train_csv\": \"train.csv\",\n    \"localizers_csv\": \"train_localizers.csv\",\n    \"cache_dir\": \"./cache_images\",\n    \"manifest_path\": \"./selected_series_manifest.csv\",\n    \"modalities_to_process\": None,  # None => process all modalities found; or e.g. ['MRA','CTA']\n    \"per_modality_total\": 300,\n    \"per_modality_pos\": 150,\n    \"per_modality_neg\": 150,\n    \"random_seed\": 42,\n    \"target_image_size\": 224,\n    \"batch_size\": 16,\n    \"num_workers\": 4,\n    \"device\": \"cuda\" if torch.cuda.is_available() else \"cpu\",\n    \"model_name\": \"resnet50\",  # options: resnet18, resnet34, resnet50\n    \"head_lr\": 1e-4,\n    \"ft_lr\": 1e-6,\n    \"weight_decay\": 1e-4,\n    \"epochs_head\": 5,\n    \"epochs_finetune\": 15,\n    \"patience\": 5,  # early stopping patience\n    \"save_dir\": \"./checkpoints\",\n    \"augment_fill_shortfall\": True,  # automatically plan augmentation to reach required counts\n    \"use_mip_for_modality\": [\"MRA\", \"CTA\"],  # default use MIP for angiographic modalities\n    \"windowing\": {  # default windowing strategy: use DICOM WindowCenter/Width when available; otherwise percentile\n        \"use_dicom_window\": True,\n        \"percentile_clip\": (1, 99)\n    },\n    \"cache_npy\": True,  # store processed 2D images as .npy in cache_dir\n    \"verbose\": True\n}\n\nos.makedirs(CONFIG[\"cache_dir\"], exist_ok=True)\nos.makedirs(CONFIG[\"save_dir\"], exist_ok=True)\n\n# --------------------------\n# Utilities & reproducibility\n# --------------------------\ndef set_seed(seed: int):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    # torch.backends.cudnn.deterministic = True  # deterministic, but can slow things\n    # torch.backends.cudnn.benchmark = False\n\nset_seed(CONFIG[\"random_seed\"])\n\ndef log(msg: str):\n    if CONFIG[\"verbose\"]:\n        print(msg)\n\n# --------------------------\n# 1) Balanced selection manifest\n# --------------------------\ndef create_balanced_manifest(train_csv_path: str,\n                             manifest_path: str,\n                             modalities_to_process: Optional[List[str]] = None,\n                             pos_target: int = 150,\n                             neg_target: int = 150,\n                             augment_fill_shortfall: bool = True) -> pd.DataFrame:\n    \"\"\"\n    Create a CSV manifest with balanced series per modality.\n\n    Output columns:\n      SeriesInstanceUID, PatientID (if present), Modality, Aneurysm Present (0/1),\n      augment_copies (int): how many augmented copies to generate (for training only)\n    \"\"\"\n    df = pd.read_csv(train_csv_path)\n    # Normalize modality strings\n    df['Modality'] = df['Modality'].astype(str).str.upper().str.strip()\n    if modalities_to_process:\n        modalities = modalities_to_process\n    else:\n        modalities = sorted(df['Modality'].unique().tolist())\n    manifest_rows = []\n    for modality in modalities:\n        group = df[df['Modality'] == modality]\n        pos = group[group['Aneurysm Present'] == 1].copy()\n        neg = group[group['Aneurysm Present'] == 0].copy()\n        # sample without replacement\n        chosen_pos = pos.sample(min(len(pos), pos_target), random_state=CONFIG[\"random_seed\"])\n        chosen_neg = neg.sample(min(len(neg), neg_target), random_state=CONFIG[\"random_seed\"])\n        pos_aug_needed = pos_target - len(chosen_pos)\n        neg_aug_needed = neg_target - len(chosen_neg)\n        # For augmentation: select seed rows to augment (repeat with augmentation)\n        def plan_aug(seed_df, aug_needed):\n            plan = {}\n            if aug_needed <= 0:\n                return plan\n            if len(seed_df) == 0:\n                return plan\n            # repeat random choices from seed_df\n            seed_ids = seed_df['SeriesInstanceUID'].tolist()\n            # choose with replacement\n            chosen = random.choices(seed_ids, k=aug_needed)\n            for sid in chosen:\n                plan[sid] = plan.get(sid, 0) + 1\n            return plan\n        pos_plan = plan_aug(chosen_pos if len(chosen_pos)>0 else pos, pos_aug_needed) if augment_fill_shortfall else {}\n        neg_plan = plan_aug(chosen_neg if len(chosen_neg)>0 else neg, neg_aug_needed) if augment_fill_shortfall else {}\n        # build manifest rows\n        for _, r in chosen_pos.iterrows():\n            augment_copies = pos_plan.get(r['SeriesInstanceUID'], 0)\n            manifest_rows.append({\n                \"SeriesInstanceUID\": r['SeriesInstanceUID'],\n                \"PatientID\": r.get('PatientID', None),\n                \"Modality\": modality,\n                \"Aneurysm Present\": int(r['Aneurysm Present']),\n                \"augment_copies\": augment_copies\n            })\n        for sid, cnt in pos_plan.items():\n            # duplicates from selected seeds: include rows for augmented copies (they'll be flagged by augment_index)\n            manifests_seed = df[df['SeriesInstanceUID'] == sid].iloc[0]\n            manifest_rows.append({\n                \"SeriesInstanceUID\": manifests_seed['SeriesInstanceUID'],\n                \"PatientID\": manifests_seed.get('PatientID', None),\n                \"Modality\": modality,\n                \"Aneurysm Present\": int(manifests_seed['Aneurysm Present']),\n                \"augment_copies\": 1  # each entry will represent one augmented image\n            })\n        for _, r in chosen_neg.iterrows():\n            augment_copies = neg_plan.get(r['SeriesInstanceUID'], 0)\n            manifest_rows.append({\n                \"SeriesInstanceUID\": r['SeriesInstanceUID'],\n                \"PatientID\": r.get('PatientID', None),\n                \"Modality\": modality,\n                \"Aneurysm Present\": int(r['Aneurysm Present']),\n                \"augment_copies\": augment_copies\n            })\n        for sid, cnt in neg_plan.items():\n            manifests_seed = df[df['SeriesInstanceUID'] == sid].iloc[0]\n            manifest_rows.append({\n                \"SeriesInstanceUID\": manifests_seed['SeriesInstanceUID'],\n                \"PatientID\": manifests_seed.get('PatientID', None),\n                \"Modality\": modality,\n                \"Aneurysm Present\": int(manifests_seed['Aneurysm Present']),\n                \"augment_copies\": 1\n            })\n        log(f\"Modality {modality}: chosen_pos={len(chosen_pos)} chosen_neg={len(chosen_neg)} pos_aug_needed={pos_aug_needed} neg_aug_needed={neg_aug_needed}\")\n    manifest_df = pd.DataFrame(manifest_rows).reset_index(drop=True)\n    manifest_df.to_csv(manifest_path, index=False)\n    log(f\"Saved manifest to {manifest_path}; total rows = {len(manifest_df)}\")\n    return manifest_df\n\n# --------------------------\n# 2) DICOM loading and volume reconstruction\n# --------------------------\ndef read_dicom_headers_quick(paths: List[str]) -> List[pydicom.dataset.FileDataset]:\n    out = []\n    for p in paths:\n        try:\n            ds = pydicom.dcmread(p, stop_before_pixels=True, force=True)\n            ds.__filepath = p\n            out.append(ds)\n        except Exception as e:\n            log(f\"Warning: failed reading header {p}: {e}\")\n    return out\n\ndef sort_dicom_slices(headers: List[pydicom.dataset.FileDataset]) -> List[pydicom.dataset.FileDataset]:\n    # prefer ImagePositionPatient z coordinate\n    def z_coord(hdr):\n        ipp = getattr(hdr, \"ImagePositionPatient\", None)\n        if ipp is not None and len(ipp) >= 3:\n            return float(ipp[2])\n        inst = getattr(hdr, \"InstanceNumber\", None)\n        if inst is not None:\n            return float(inst)\n        # fallback to file name sort\n        return float(0)\n    headers_sorted = sorted(headers, key=z_coord)\n    return headers_sorted\n\ndef load_series_volume(series_path: str) -> Tuple[np.ndarray, List[pydicom.dataset.FileDataset]]:\n    \"\"\"\n    Load all .dcm files in series_path and return a 3D numpy array (Z, H, W) and sorted headers.\n    Applies Rescale Slope/Intercept to pixel_array if present.\n    \"\"\"\n    dicom_files = [os.path.join(series_path, f) for f in os.listdir(series_path) if f.lower().endswith('.dcm')]\n    if len(dicom_files) == 0:\n        raise FileNotFoundError(f\"No DICOM files found in {series_path}\")\n    headers = read_dicom_headers_quick(dicom_files)\n    headers_sorted = sort_dicom_slices(headers)\n    slices = []\n    for hdr in headers_sorted:\n        p = getattr(hdr, \"__filepath\", None)\n        try:\n            ds_full = pydicom.dcmread(p, force=True)\n            arr = ds_full.pixel_array.astype(np.float32)\n            slope = float(getattr(ds_full, \"RescaleSlope\", 1.0))\n            intercept = float(getattr(ds_full, \"RescaleIntercept\", 0.0))\n            arr = arr * slope + intercept\n            slices.append(arr)\n        except Exception as e:\n            log(f\"Warning: failed reading pixels from {p}: {e}\")\n    if len(slices) == 0:\n        raise RuntimeError(f\"No pixel data extracted for series {series_path}\")\n    volume = np.stack(slices, axis=0)  # shape (Z,H,W)\n    return volume, headers_sorted\n\n# --------------------------\n# 3) Localizer handling\n# --------------------------\ndef load_localizers(localizers_csv_path: str) -> pd.DataFrame:\n    if not os.path.exists(localizers_csv_path):\n        log(\"No localizers CSV found; continuing without localizers.\")\n        return pd.DataFrame()\n    locs = pd.read_csv(localizers_csv_path)\n    # Ensure coordinates parsed\n    if 'coordinates' in locs.columns:\n        def parse_coords(c):\n            if pd.isna(c):\n                return {}\n            try:\n                if isinstance(c, str):\n                    return ast.literal_eval(c) if (c.strip().startswith(\"{\") or c.strip().startswith(\"[\")) else json.loads(c)\n                elif isinstance(c, dict):\n                    return c\n                else:\n                    return {}\n            except Exception:\n                try:\n                    return json.loads(c)\n                except Exception:\n                    return {}\n        locs['coords_parsed'] = locs['coordinates'].apply(parse_coords) if 'coordinates' in locs.columns else [{}]*len(locs)\n    else:\n        locs['coords_parsed'] = [{}]*len(locs)\n    return locs\n\ndef find_localizer_for_series(locs_df: pd.DataFrame, series_uid: str) -> Optional[Dict[str,Any]]:\n    if locs_df is None or locs_df.empty:\n        return None\n    df = locs_df[locs_df['SeriesInstanceUID'] == series_uid]\n    if df.shape[0] == 0:\n        return None\n    # return first match as dict (could be multiple)\n    row = df.iloc[0]\n    return {\n        \"SOPInstanceUID\": row.get('SOPInstanceUID', None),\n        \"coords\": row.get('coords_parsed', {})\n    }\n\n# --------------------------\n# 4) 3D -> 2D conversion (MIP, slice, cropping)\n# --------------------------\ndef volume_to_2d_image(volume: np.ndarray,\n                       headers_sorted: List[pydicom.dataset.FileDataset],\n                       method: str = \"mip\",\n                       localizer_sop: Optional[str] = None,\n                       local_coords: Optional[Dict[str, float]] = None,\n                       crop_size: int = 128) -> Tuple[np.ndarray, Dict[str,Any]]:\n    \"\"\"\n    Converts a 3D Z,H,W volume to a single 2D image.\n    method: 'mip' or 'slice'\n    If method == 'slice' and localizer_sop is provided, selects that slice.\n    Returns (image_2d, info)\n    \"\"\"\n    info = {\"method\": method, \"used_slice_index\": None, \"used_crop\": None}\n    if method == \"mip\":\n        img = np.max(volume, axis=0)\n        info[\"used_slice_index\"] = \"mip\"\n    elif method == \"slice\":\n        # Attempt to find index by SOPInstanceUID\n        idx = None\n        if localizer_sop:\n            for i, hdr in enumerate(headers_sorted):\n                if getattr(hdr, \"SOPInstanceUID\", None) == localizer_sop:\n                    idx = i\n                    break\n        if idx is None:\n            # fallback to middle slice\n            idx = volume.shape[0] // 2\n        img = volume[idx]\n        info[\"used_slice_index\"] = int(idx)\n    else:\n        # default to mip\n        img = np.max(volume, axis=0)\n        info[\"used_slice_index\"] = \"mip_default\"\n    # crop around local_coords if given\n    H, W = img.shape\n    if local_coords and \"x\" in local_coords and \"y\" in local_coords:\n        x = int(round(local_coords[\"x\"]))\n        y = int(round(local_coords[\"y\"]))\n        half = crop_size // 2\n        x1 = max(0, x - half)\n        y1 = max(0, y - half)\n        x2 = min(W, x + half)\n        y2 = min(H, y + half)\n        crop = img[y1:y2, x1:x2]\n        if crop.size > 0:\n            info[\"used_crop\"] = (x1, y1, x2, y2)\n            # if crop not full size, pad\n            pad_h = crop_size - crop.shape[0]\n            pad_w = crop_size - crop.shape[1]\n            if pad_h > 0 or pad_w > 0:\n                crop = np.pad(crop, ((0, pad_h), (0, pad_w)), mode='constant', constant_values=np.min(img))\n            img = crop\n    return img.astype(np.float32), info\n\n# --------------------------\n# 5) Windowing / normalization / resize\n# --------------------------\ndef apply_windowing(img: np.ndarray,\n                    headers_sorted: Optional[List[pydicom.dataset.FileDataset]] = None,\n                    use_dicom_window: bool = True,\n                    percentile_clip: Tuple[int,int] = (1,99)) -> np.ndarray:\n    \"\"\"\n    img: 2D float32 numpy array\n    If DICOM tags available (WindowCenter/Width) in headers_sorted[0], apply that.\n    Otherwise use percentile clipping.\n    Returns image in range [0,1].\n    \"\"\"\n    if use_dicom_window and headers_sorted:\n        # try to find window center / width from any header that has them\n        center, width = None, None\n        for hdr in headers_sorted:\n            wc = getattr(hdr, \"WindowCenter\", None)\n            ww = getattr(hdr, \"WindowWidth\", None)\n            if wc is not None and ww is not None:\n                try:\n                    center = float(wc[0] if isinstance(wc, (list, tuple)) else wc)\n                    width = float(ww[0] if isinstance(ww, (list, tuple)) else ww)\n                    break\n                except Exception:\n                    continue\n        if center is not None and width is not None and width > 0:\n            low = center - width / 2.0\n            high = center + width / 2.0\n            imgw = np.clip(img, low, high)\n            imgw = (imgw - low) / (high - low)\n            imgw = np.nan_to_num(imgw, nan=0.0, posinf=1.0, neginf=0.0)\n            return imgw.astype(np.float32)\n    # fallback percentile clipping\n    lowp, highp = np.percentile(img, percentile_clip[0]), np.percentile(img, percentile_clip[1])\n    imgc = np.clip(img, lowp, highp)\n    if highp - lowp > 0:\n        imgc = (imgc - lowp) / (highp - lowp)\n    else:\n        imgc = imgc - lowp\n    imgc = np.nan_to_num(imgc, nan=0.0, posinf=1.0, neginf=0.0)\n    return imgc.astype(np.float32)\n\ndef resize_and_to_uint8(img: np.ndarray, target_size: int = 224) -> np.ndarray:\n    # input img in [0,1], float32\n    h, w = img.shape\n    if h == target_size and w == target_size:\n        out = (img * 255.0).astype(np.uint8)\n        return out\n    out = cv2.resize(img, (target_size, target_size), interpolation=cv2.INTER_LINEAR)\n    out = np.clip(out * 255.0, 0, 255).astype(np.uint8)\n    return out\n\n# --------------------------\n# 6) Dataset & transforms\n# --------------------------\nIMAGENET_MEAN = [0.485, 0.456, 0.406]\nIMAGENET_STD  = [0.229, 0.224, 0.225]\n\ndef get_train_transforms(target_size: int):\n    return A.Compose([\n        A.RandomRotate90(p=0.2),  # 0/90/180/270 random sometimes - helpful\n        A.Rotate(limit=10, p=0.5),\n        A.Flip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.02, scale_limit=0.05, rotate_limit=0, p=0.2),\n        A.Resize(target_size, target_size),\n        A.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n        ToTensorV2()\n    ])\n\ndef get_val_transforms(target_size: int):\n    return A.Compose([\n        A.Resize(target_size, target_size),\n        A.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD),\n        ToTensorV2()\n    ])\n\nclass AneurysmSeriesDataset(Dataset):\n    def __init__(self, manifest_df: pd.DataFrame, data_root: str, series_folder: str,\n                 localizers_df: pd.DataFrame = None, transforms=None, cache_dir: str = None,\n                 use_mip_for_modality: List[str] = None, target_size: int = 224):\n        \"\"\"\n        manifest_df: DataFrame where each row is a series to include. augment_copies in manifest represent duplication for underrepresented classes.\n        NOTE: If a SeriesInstanceUID appears multiple times in manifest (due to augmentation plan), we treat each row as a separate sample.\n        \"\"\"\n        self.manifest = manifest_df.reset_index(drop=True).copy()\n        self.data_root = data_root\n        self.series_folder = series_folder\n        self.localizers_df = localizers_df\n        self.transforms = transforms\n        self.cache_dir = cache_dir\n        self.use_mip_for_modality = set(use_mip_for_modality or [])\n        self.target_size = target_size\n        # expand manifest rows where augment_copies > 0 into repeated rows flagged as augmented\n        expanded_rows = []\n        for _, row in self.manifest.iterrows():\n            base = row.to_dict()\n            aug = int(base.get('augment_copies', 0))\n            # first add original\n            expanded_rows.append({**base, \"is_augmented\": False})\n            # additional augmented entries\n            for i in range(aug):\n                expanded_rows.append({**base, \"is_augmented\": True, \"augment_index\": i})\n        self.entries = pd.DataFrame(expanded_rows).reset_index(drop=True)\n        log(f\"Dataset entries: {len(self.entries)} (including augmented copies).\")\n    def __len__(self):\n        return len(self.entries)\n    def __getitem__(self, idx):\n        row = self.entries.iloc[idx]\n        series_uid = row['SeriesInstanceUID']\n        modality = str(row['Modality']).upper()\n        label = int(row['Aneurysm Present'])\n        is_augmented = bool(row.get('is_augmented', False))\n        # cache filename key\n        cache_name = f\"{series_uid}\"\n        if is_augmented:\n            cache_name = f\"{series_uid}_aug{row.get('augment_index',0)}\"\n        cache_path = os.path.join(self.cache_dir, cache_name + \".npy\") if self.cache_dir else None\n        if cache_path and os.path.exists(cache_path):\n            # load cached processed image (uint8 with shape H,W or H,W,3)\n            img_uint8 = np.load(cache_path)\n        else:\n            # load series, convert to 2D, window, resize -> uint8\n            series_path = os.path.join(self.data_root, self.series_folder, series_uid)\n            try:\n                volume, headers_sorted = load_series_volume(series_path)\n            except Exception as e:\n                raise RuntimeError(f\"Failed to load series {series_uid} at {series_path}: {e}\")\n            # localizer\n            localizer = find_localizer_for_series(self.localizers_df, series_uid) if (self.localizers_df is not None and not self.localizers_df.empty) else None\n            local_coords = localizer.get('coords') if localizer else {}\n            local_sop = localizer.get('SOPInstanceUID') if localizer else None\n            # choose method\n            method = \"mip\" if modality in self.use_mip_for_modality else \"slice\"\n            img2d, info = volume_to_2d_image(volume, headers_sorted, method=method, localizer_sop=local_sop, local_coords=local_coords, crop_size=min(128, self.target_size))\n            # window & normalize\n            imgw = apply_windowing(img2d, headers_sorted=headers_sorted if CONFIG[\"windowing\"][\"use_dicom_window\"] else None,\n                                   use_dicom_window=CONFIG[\"windowing\"][\"use_dicom_window\"],\n                                   percentile_clip=CONFIG[\"windowing\"][\"percentile_clip\"])\n            img_uint8 = resize_and_to_uint8(imgw, target_size=self.target_size)\n            # save cache\n            if cache_path:\n                try:\n                    np.save(cache_path, img_uint8)\n                except Exception as e:\n                    log(f\"Warning: failed to save cache {cache_path}: {e}\")\n        # At this point img_uint8 is H,W uint8. Convert to 3-channel\n        if img_uint8.ndim == 2:\n            img_rgb = np.stack([img_uint8, img_uint8, img_uint8], axis=2)\n        elif img_uint8.ndim == 3 and img_uint8.shape[2] == 3:\n            img_rgb = img_uint8\n        else:\n            # handle odd channel shapes\n            img_rgb = np.stack([img_uint8[...,0], img_uint8[...,0], img_uint8[...,0]], axis=2)\n        # If row flagged as augmented: we will apply augmentations (transforms passed in) which include randomness\n        if self.transforms:\n            # Albumentations expects image in HWC format uint8\n            augmented = self.transforms(image=img_rgb)\n            img_tensor = augmented['image']  # this is a torch.Tensor [C,H,W], normalized\n        else:\n            # fallback transform: convert to tensor and normalize\n            t = transforms.Compose([\n                transforms.ToTensor(),\n                transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)\n            ])\n            img_tensor = t(img_rgb)\n        return img_tensor, label, series_uid\n\n# --------------------------\n# 7) Model building & training helpers\n# --------------------------\ndef build_model(model_name: str = \"resnet50\", pretrained: bool = True, num_classes: int = 2, in_channels: int = 3):\n    if model_name.lower() == \"resnet50\":\n        model = models.resnet50(pretrained=pretrained)\n    elif model_name.lower() == \"resnet34\":\n        model = models.resnet34(pretrained=pretrained)\n    elif model_name.lower() == \"resnet18\":\n        model = models.resnet18(pretrained=pretrained)\n    else:\n        raise ValueError(\"Unsupported model_name\")\n    # adapt first conv if in_channels != 3\n    if in_channels != 3:\n        old_conv = model.conv1\n        new_conv = nn.Conv2d(in_channels, old_conv.out_channels, kernel_size=old_conv.kernel_size,\n                             stride=old_conv.stride, padding=old_conv.padding, bias=old_conv.bias is not None)\n        # initialize new_conv weights by averaging existing channels\n        with torch.no_grad():\n            if in_channels == 1:\n                new_conv.weight[:] = old_conv.weight.mean(dim=1, keepdim=True)\n        model.conv1 = new_conv\n    # replace fc\n    in_feats = model.fc.in_features\n    model.fc = nn.Linear(in_feats, num_classes)\n    return model\n\ndef compute_metrics(y_true: List[int], y_pred_probs: List[float], threshold: float = 0.5) -> Dict[str, float]:\n    probs = np.array(y_pred_probs)\n    y_true = np.array(y_true)\n    y_pred = (probs >= threshold).astype(int)\n    metrics = {}\n    try:\n        metrics['auc'] = float(roc_auc_score(y_true, probs))\n    except Exception:\n        metrics['auc'] = float('nan')\n    metrics['accuracy'] = float(accuracy_score(y_true, y_pred))\n    metrics['precision'] = float(precision_score(y_true, y_pred, zero_division=0))\n    metrics['recall'] = float(recall_score(y_true, y_pred, zero_division=0))\n    metrics['f1'] = float(f1_score(y_true, y_pred, zero_division=0))\n    return metrics\n\n# --------------------------\n# 8) Training loop (two-phase)\n# --------------------------\ndef train_model(manifest_df: pd.DataFrame, localizers_df: pd.DataFrame, data_root: str, series_folder: str):\n    # Prepare train/val split (stratified by modality & label)\n    # We will create splits within each modality to ensure 80/20 stratified\n    train_idx = []\n    val_idx = []\n    for modality, group in manifest_df.groupby('Modality'):\n        grp = group.reset_index(drop=True)\n        # stratify by 'Aneurysm Present'\n        if len(grp) < 2:\n            # small groups: put all to train\n            train_idx += grp.index.tolist()\n            continue\n        # use train_test_split but keep indices consistent with manifest_df index — we'll use local group indexing and map back\n        gtrain, gval = train_test_split(grp, test_size=0.2, stratify=grp['Aneurysm Present'], random_state=CONFIG[\"random_seed\"])\n        # get original indices\n        train_idx += gtrain.index.tolist()\n        val_idx += gval.index.tolist()\n    train_df = manifest_df.iloc[train_idx].reset_index(drop=True)\n    val_df = manifest_df.iloc[val_idx].reset_index(drop=True)\n    log(f\"Train size: {len(train_df)}  Val size: {len(val_df)}\")\n    # Datasets\n    train_ds = AneurysmSeriesDataset(train_df, data_root=data_root, series_folder=series_folder,\n                                     localizers_df=localizers_df, transforms=get_train_transforms(CONFIG[\"target_image_size\"]),\n                                     cache_dir=CONFIG[\"cache_dir\"], use_mip_for_modality=CONFIG[\"use_mip_for_modality\"],\n                                     target_size=CONFIG[\"target_image_size\"])\n    val_ds = AneurysmSeriesDataset(val_df, data_root=data_root, series_folder=series_folder,\n                                     localizers_df=localizers_df, transforms=get_val_transforms(CONFIG[\"target_image_size\"]),\n                                     cache_dir=CONFIG[\"cache_dir\"], use_mip_for_modality=CONFIG[\"use_mip_for_modality\"],\n                                     target_size=CONFIG[\"target_image_size\"])\n    # DataLoaders\n    train_loader = DataLoader(train_ds, batch_size=CONFIG[\"batch_size\"], shuffle=True, num_workers=CONFIG[\"num_workers\"], pin_memory=True)\n    val_loader   = DataLoader(val_ds, batch_size=CONFIG[\"batch_size\"], shuffle=False, num_workers=CONFIG[\"num_workers\"], pin_memory=True)\n    # Model\n    model = build_model(CONFIG[\"model_name\"], pretrained=True, num_classes=2, in_channels=3)\n    model = model.to(CONFIG[\"device\"])\n    # Loss\n    criterion = nn.CrossEntropyLoss()\n    # Phase 1: train head only\n    for p in model.parameters():\n        p.requires_grad = False\n    for p in model.fc.parameters():\n        p.requires_grad = True\n    optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=CONFIG[\"head_lr\"], weight_decay=CONFIG[\"weight_decay\"])\n    scaler = torch.cuda.amp.GradScaler(enabled=(CONFIG[\"device\"].startswith(\"cuda\")))\n    best_val_auc = -1.0\n    best_epoch = -1\n    history_rows = []\n    # helper functions\n    def run_one_epoch(epoch_idx, loader, training=True):\n        if training:\n            model.train()\n        else:\n            model.eval()\n        running_loss = 0.0\n        y_true_all = []\n        y_prob_all = []\n        pbar = tqdm(loader, desc=(\"Train\" if training else \"Val\") + f\" Epoch {epoch_idx}\", leave=False)\n        for batch in pbar:\n            imgs, labels, sids = batch\n            imgs = imgs.to(CONFIG[\"device\"], non_blocking=True)\n            labels = labels.to(CONFIG[\"device\"], non_blocking=True)\n            with torch.set_grad_enabled(training):\n                with torch.cuda.amp.autocast(enabled=(CONFIG[\"device\"].startswith(\"cuda\"))):\n                    outputs = model(imgs)\n                    loss = criterion(outputs, labels)\n                if training:\n                    optimizer.zero_grad()\n                    scaler.scale(loss).backward()\n                    scaler.step(optimizer)\n                    scaler.update()\n            running_loss += float(loss.item()) * imgs.shape[0]\n            probs = F.softmax(outputs.detach(), dim=1)[:,1].cpu().numpy()\n            y_prob_all.extend(probs.tolist())\n            y_true_all.extend(labels.cpu().numpy().tolist())\n            pbar.set_postfix(loss=running_loss / ((pbar.n + 1) * loader.batch_size))\n        avg_loss = running_loss / len(loader.dataset)\n        metrics = compute_metrics(y_true_all, y_prob_all)\n        metrics['loss'] = avg_loss\n        return metrics\n    # training head\n    log(\"Starting Phase 1: training classifier head only.\")\n    patience_cnt = 0\n    for epoch in range(CONFIG[\"epochs_head\"]):\n        t0 = time.time()\n        train_metrics = run_one_epoch(epoch, train_loader, training=True)\n        val_metrics = run_one_epoch(epoch, val_loader, training=False)\n        elapsed = time.time() - t0\n        log(f\"[Head] Epoch {epoch+1}/{CONFIG['epochs_head']}  train_loss={train_metrics['loss']:.4f} val_loss={val_metrics['loss']:.4f} val_auc={val_metrics['auc']:.4f} time={elapsed:.1f}s\")\n        history_rows.append({\"phase\":\"head\",\"epoch\":epoch+1,\"train_loss\":train_metrics['loss'],\"val_loss\":val_metrics['loss'],\n                             \"val_auc\":val_metrics['auc'],\"val_f1\":val_metrics['f1']})\n        # checkpoint best\n        if val_metrics['auc'] > best_val_auc:\n            best_val_auc = val_metrics['auc']\n            best_epoch = epoch\n            torch.save(model.state_dict(), os.path.join(CONFIG[\"save_dir\"], \"best_head.pth\"))\n            log(f\"Saved best_head.pth with val_auc={best_val_auc:.4f}\")\n            patience_cnt = 0\n        else:\n            patience_cnt += 1\n            if patience_cnt >= CONFIG[\"patience\"]:\n                log(\"Early stopping head training due to no improvement.\")\n                break\n    # Phase 2: fine-tune entire model\n    log(\"Starting Phase 2: fine-tuning entire model.\")\n    for p in model.parameters():\n        p.requires_grad = True\n    optimizer = torch.optim.AdamW(model.parameters(), lr=CONFIG[\"ft_lr\"], weight_decay=CONFIG[\"weight_decay\"])\n    patience_cnt = 0\n    for epoch in range(CONFIG[\"epochs_finetune\"]):\n        t0 = time.time()\n        train_metrics = run_one_epoch(epoch, train_loader, training=True)\n        val_metrics = run_one_epoch(epoch, val_loader, training=False)\n        elapsed = time.time() - t0\n        log(f\"[FT ] Epoch {epoch+1}/{CONFIG['epochs_finetune']}  train_loss={train_metrics['loss']:.4f} val_loss={val_metrics['loss']:.4f} val_auc={val_metrics['auc']:.4f} time={elapsed:.1f}s\")\n        history_rows.append({\"phase\":\"finetune\",\"epoch\":epoch+1,\"train_loss\":train_metrics['loss'],\"val_loss\":val_metrics['loss'],\n                             \"val_auc\":val_metrics['auc'],\"val_f1\":val_metrics['f1']})\n        # checkpoint best\n        if val_metrics['auc'] > best_val_auc:\n            best_val_auc = val_metrics['auc']\n            best_epoch = epoch\n            torch.save(model.state_dict(), os.path.join(CONFIG[\"save_dir\"], \"best_finetune.pth\"))\n            log(f\"Saved best_finetune.pth with val_auc={best_val_auc:.4f}\")\n            patience_cnt = 0\n        else:\n            patience_cnt += 1\n            if patience_cnt >= CONFIG[\"patience\"]:\n                log(\"Early stopping finetune due to no improvement.\")\n                break\n    # save final model\n    torch.save(model.state_dict(), os.path.join(CONFIG[\"save_dir\"], \"final_model.pth\"))\n    # save history\n    hist_df = pd.DataFrame(history_rows)\n    hist_df.to_csv(os.path.join(CONFIG[\"save_dir\"], \"training_history.csv\"), index=False)\n    log(f\"Training complete. Best val_auc={best_val_auc:.4f} at epoch {best_epoch}. Artifacts in {CONFIG['save_dir']}\")\n\n# --------------------------\n# 9) Main execution\n# --------------------------\ndef main():\n    data_root = CONFIG[\"data_root\"]\n    train_csv_path = os.path.join(data_root, CONFIG[\"train_csv\"])\n    localizers_csv_path = os.path.join(data_root, CONFIG[\"localizers_csv\"])\n    manifest_path = CONFIG[\"manifest_path\"]\n    # 1) Create balanced manifest if not exists\n    if not os.path.exists(manifest_path):\n        manifest_df = create_balanced_manifest(train_csv_path, manifest_path,\n                                               modalities_to_process=CONFIG[\"modalities_to_process\"],\n                                               pos_target=CONFIG[\"per_modality_pos\"],\n                                               neg_target=CONFIG[\"per_modality_neg\"],\n                                               augment_fill_shortfall=CONFIG[\"augment_fill_shortfall\"])\n    else:\n        manifest_df = pd.read_csv(manifest_path)\n        log(f\"Loaded existing manifest from {manifest_path}\")\n    # 2) Load localizers (if available)\n    localizers_df = load_localizers(localizers_csv_path) if os.path.exists(localizers_csv_path) else pd.DataFrame()\n    # 3) Train model\n    train_model(manifest_df, localizers_df, data_root, CONFIG[\"series_folder\"])\n\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T06:36:11.043782Z","iopub.execute_input":"2025-10-07T06:36:11.044087Z","iopub.status.idle":"2025-10-07T06:36:26.844751Z","shell.execute_reply.started":"2025-10-07T06:36:11.044066Z","shell.execute_reply":"2025-10-07T06:36:26.843712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport pydicom\nimport cv2\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nimport torchvision.models as models\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport warnings\nimport json\nfrom pathlib import Path\nimport random\nfrom collections import defaultdict\n\nwarnings.filterwarnings('ignore')\n\nclass Config:\n    \"\"\"Configuration class for the medical imaging pipeline\"\"\"\n    \n    # Data paths\n    DATA_ROOT = r\"/kaggle/input/rsna-intracranial-aneurysm-detection\"  # Update this path\n    SERIES_PATH = os.path.join(DATA_ROOT, \"series\")\n    TRAIN_CSV = os.path.join(DATA_ROOT, \"train.csv\")\n    TRAIN_LOCALIZERS_CSV = os.path.join(DATA_ROOT, \"train_localizers.csv\")\n    \n    # Sampling parameters\n    SAMPLES_PER_CLASS = 150\n    TOTAL_SAMPLES_PER_MODALITY = 300\n    \n    # Image parameters\n    IMAGE_SIZE = 224\n    WINDOWING_CENTER = 40  # Brain window center\n    WINDOWING_WIDTH = 80   # Brain window width\n    \n    # Training parameters\n    BATCH_SIZE = 32\n    LEARNING_RATE_PHASE1 = 1e-4  # For classifier head only\n    LEARNING_RATE_PHASE2 = 1e-6  # For full fine-tuning\n    EPOCHS_PHASE1 = 5\n    EPOCHS_PHASE2 = 15\n    VALIDATION_SPLIT = 0.2\n    DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    \n    # Model parameters\n    MODEL_NAME = 'resnet50'\n    NUM_CLASSES = 2\n\nclass DICOMLoader:\n    \"\"\"Handles DICOM file loading and 3D to 2D conversion\"\"\"\n    \n    @staticmethod\n    def load_dicom_series(series_path):\n        \"\"\"Load all DICOM files from a series directory and return as 3D volume\"\"\"\n        dcm_files = []\n        dicom_dir = Path(series_path)\n        \n        if not dicom_dir.exists():\n            print(f\"Warning: Directory does not exist: {series_path}\")\n            return None\n            \n        for dcm_file in dicom_dir.glob(\"*.dcm\"):\n            try:\n                ds = pydicom.dcmread(str(dcm_file), force=True)\n                if hasattr(ds, 'pixel_array') and ds.pixel_array is not None:\n                    # Handle different pixel data types\n                    pixel_array = ds.pixel_array.astype(np.float32)\n                    if pixel_array.size > 0:  # Check if array is not empty\n                        dcm_files.append((ds, pixel_array))\n            except Exception as e:\n                print(f\"Warning: Error loading {dcm_file}: {e}\")\n                continue\n        \n        if not dcm_files:\n            print(f\"Warning: No valid DICOM files found in {series_path}\")\n            return None\n        \n        # Sort by instance number if available\n        try:\n            dcm_files.sort(key=lambda x: int(x[0].InstanceNumber) if hasattr(x[0], 'InstanceNumber') else 0)\n        except:\n            pass\n        \n        # Stack into 3D volume\n        try:\n            pixel_arrays = [dcm_data[1] for dcm_data in dcm_files]\n            \n            # Ensure all arrays have the same shape\n            shapes = [arr.shape for arr in pixel_arrays]\n            if len(set(shapes)) > 1:\n                # Resize to the most common shape\n                from collections import Counter\n                most_common_shape = Counter(shapes).most_common(1)[0][0]\n                pixel_arrays = [cv2.resize(arr, (most_common_shape[1], most_common_shape[0])) \n                              if arr.shape != most_common_shape else arr \n                              for arr in pixel_arrays]\n            \n            volume = np.stack(pixel_arrays, axis=0)\n            return volume, dcm_files[0][0]  # Return volume and first DICOM for metadata\n        except Exception as e:\n            print(f\"Warning: Error stacking DICOM files from {series_path}: {e}\")\n            # Return the first valid image as a single slice\n            if dcm_files:\n                return dcm_files[0][1][np.newaxis, ...], dcm_files[0][0]\n            return None\n    \n    @staticmethod\n    def convert_3d_to_2d(volume, method='middle_slice'):\n        \"\"\"Convert 3D volume to 2D image using specified method\"\"\"\n        if volume is None or volume.size == 0:\n            return None\n        \n        # Ensure volume is at least 2D\n        if len(volume.shape) == 1:\n            return None\n        elif len(volume.shape) == 2:\n            return volume\n        elif len(volume.shape) >= 3:\n            if method == 'middle_slice':\n                return volume[volume.shape[0] // 2]\n            elif method == 'max_intensity_projection':\n                return np.max(volume, axis=0)\n            elif method == 'mean_projection':\n                return np.mean(volume, axis=0)\n            else:\n                return volume[volume.shape[0] // 2]\n        else:\n            return None\n\nclass ImagePreprocessor:\n    \"\"\"Handles image preprocessing including windowing, normalization, and resizing\"\"\"\n    \n    @staticmethod\n    def apply_windowing(image, center=40, width=80):\n        \"\"\"Apply windowing to enhance contrast\"\"\"\n        if image is None:\n            return None\n        \n        # Ensure image is a numpy array\n        if not isinstance(image, np.ndarray):\n            return None\n        \n        # Handle empty or invalid arrays\n        if image.size == 0 or not np.isfinite(image).any():\n            return None\n        \n        min_val = center - width // 2\n        max_val = center + width // 2\n        \n        try:\n            windowed = np.clip(image, min_val, max_val)\n            windowed = (windowed - min_val) / (max_val - min_val)\n            return windowed\n        except Exception as e:\n            print(f\"Warning: Windowing failed: {e}\")\n            return None\n    \n    @staticmethod\n    def normalize_image(image):\n        \"\"\"Normalize image to [0, 1] range\"\"\"\n        if image is None:\n            return None\n        \n        try:\n            # Handle constant images\n            if image.max() == image.min():\n                return np.zeros_like(image)\n            \n            normalized = (image - image.min()) / (image.max() - image.min())\n            return normalized\n        except Exception as e:\n            print(f\"Warning: Normalization failed: {e}\")\n            return None\n    \n    @staticmethod\n    def resize_image(image, target_size=224):\n        \"\"\"Resize image to target size\"\"\"\n        if image is None:\n            return None\n        \n        try:\n            # Ensure image is 2D\n            if len(image.shape) != 2:\n                return None\n            \n            resized = cv2.resize(image, (target_size, target_size))\n            return resized\n        except Exception as e:\n            print(f\"Warning: Resizing failed: {e}\")\n            return None\n    \n    @staticmethod\n    def preprocess_dicom_image(image, config):\n        \"\"\"Complete preprocessing pipeline for DICOM image\"\"\"\n        if image is None:\n            # Return a blank image if input is None\n            return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.float32)\n        \n        # Apply windowing\n        windowed = ImagePreprocessor.apply_windowing(\n            image, config.WINDOWING_CENTER, config.WINDOWING_WIDTH\n        )\n        \n        if windowed is None:\n            # Fallback: try simple normalization\n            windowed = ImagePreprocessor.normalize_image(image)\n        \n        if windowed is None:\n            # Final fallback: return blank image\n            return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.float32)\n        \n        # Normalize\n        normalized = ImagePreprocessor.normalize_image(windowed)\n        if normalized is None:\n            return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.float32)\n        \n        # Convert to uint8 for OpenCV operations\n        try:\n            normalized_uint8 = (normalized * 255).astype(np.uint8)\n        except:\n            return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.float32)\n        \n        # Resize\n        resized = ImagePreprocessor.resize_image(normalized_uint8, config.IMAGE_SIZE)\n        if resized is None:\n            return np.zeros((config.IMAGE_SIZE, config.IMAGE_SIZE), dtype=np.float32)\n        \n        # Convert back to float and normalize to [0, 1]\n        final_image = resized.astype(np.float32) / 255.0\n        \n        return final_image\n\nclass DataSampler:\n    \"\"\"Handles balanced data sampling and augmentation\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n    \n    def load_metadata(self):\n        \"\"\"Load train.csv and train_localizers.csv\"\"\"\n        train_df = pd.read_csv(self.config.TRAIN_CSV)\n        try:\n            localizers_df = pd.read_csv(self.config.TRAIN_LOCALIZERS_CSV)\n        except:\n            localizers_df = None\n            print(\"Warning: Could not load train_localizers.csv\")\n        return train_df, localizers_df\n    \n    def sample_balanced_data(self, train_df):\n        \"\"\"Create balanced dataset for each modality\"\"\"\n        balanced_data = {}\n        \n        modalities = train_df['Modality'].unique()\n        print(f\"Found modalities: {modalities}\")\n        \n        for modality in modalities:\n            modality_data = train_df[train_df['Modality'] == modality]\n            \n            positive_samples = modality_data[modality_data['Aneurysm Present'] == 1]\n            negative_samples = modality_data[modality_data['Aneurysm Present'] == 0]\n            \n            print(f\"\\n{modality} - Positive: {len(positive_samples)}, Negative: {len(negative_samples)}\")\n            \n            # Sample or augment to reach target count\n            selected_positive = self._sample_or_augment(positive_samples, self.config.SAMPLES_PER_CLASS)\n            selected_negative = self._sample_or_augment(negative_samples, self.config.SAMPLES_PER_CLASS)\n            \n            balanced_data[modality] = {\n                'positive': selected_positive,\n                'negative': selected_negative\n            }\n            \n            print(f\"Selected - Positive: {len(selected_positive)}, Negative: {len(selected_negative)}\")\n        \n        return balanced_data\n    \n    def _sample_or_augment(self, samples, target_count):\n        \"\"\"Sample or augment data to reach target count\"\"\"\n        if len(samples) >= target_count:\n            return samples.sample(n=target_count, random_state=42)\n        else:\n            # Need augmentation\n            augmentation_needed = target_count - len(samples)\n            print(f\"Augmentation needed: {augmentation_needed} samples\")\n            \n            # Simple augmentation by repeating samples with slight modifications\n            augmented_samples = []\n            original_samples = samples.copy()\n            \n            for i in range(augmentation_needed):\n                sample_to_duplicate = original_samples.iloc[i % len(original_samples)].copy()\n                # Mark as augmented for tracking\n                sample_to_duplicate['is_augmented'] = True\n                augmented_samples.append(sample_to_duplicate)\n            \n            # Combine original and augmented\n            all_samples = pd.concat([original_samples, pd.DataFrame(augmented_samples)], ignore_index=True)\n            return all_samples\n\nclass MedicalImageDataset(Dataset):\n    \"\"\"PyTorch dataset for medical images\"\"\"\n    \n    def __init__(self, series_data, config, transform=None, is_training=True):\n        self.series_data = series_data\n        self.config = config\n        self.transform = transform\n        self.is_training = is_training\n        \n        # Prepare data list\n        self.data_list = []\n        for _, row in series_data.iterrows():\n            self.data_list.append({\n                'series_uid': row['SeriesInstanceUID'],\n                'label': row['Aneurysm Present'],\n                'modality': row['Modality'],\n                'is_augmented': row.get('is_augmented', False)\n            })\n        \n        print(f\"Dataset created with {len(self.data_list)} samples\")\n    \n    def __len__(self):\n        return len(self.data_list)\n    \n    def __getitem__(self, idx):\n        item = self.data_list[idx]\n        series_uid = item['series_uid']\n        label = item['label']\n        is_augmented = item['is_augmented']\n        \n        # Load DICOM series\n        series_path = os.path.join(self.config.SERIES_PATH, series_uid)\n        volume_data = DICOMLoader.load_dicom_series(series_path)\n        \n        if volume_data is None:\n            # Return a blank image if loading fails\n            print(f\"Warning: Failed to load series {series_uid}, using blank image\")\n            image = np.zeros((self.config.IMAGE_SIZE, self.config.IMAGE_SIZE), dtype=np.float32)\n        else:\n            volume, dicom_metadata = volume_data\n            # Convert 3D to 2D\n            image_2d = DICOMLoader.convert_3d_to_2d(volume, method='middle_slice')\n            # Preprocess\n            image = ImagePreprocessor.preprocess_dicom_image(image_2d, self.config)\n        \n        # Apply additional augmentation for augmented samples\n        if is_augmented and self.is_training:\n            image = self._apply_augmentation(image)\n        \n        # Convert to 3-channel for ResNet\n        image = np.stack([image, image, image], axis=0)  # Shape: (3, H, W)\n        \n        # Apply transforms\n        if self.transform:\n            image = torch.from_numpy(image).float()\n            image = self.transform(image)\n        else:\n            image = torch.from_numpy(image).float()\n        \n        label = torch.tensor(label, dtype=torch.long)\n        \n        return image, label\n    \n    def _apply_augmentation(self, image):\n        \"\"\"Apply simple augmentation for training\"\"\"\n        if image is None or image.size == 0:\n            return image\n        \n        try:\n            # Random rotation\n            if random.random() > 0.5:\n                angle = random.uniform(-10, 10)\n                h, w = image.shape\n                M = cv2.getRotationMatrix2D((w/2, h/2), angle, 1)\n                image = cv2.warpAffine(image, M, (w, h))\n            \n            # Random flip\n            if random.random() > 0.5:\n                image = cv2.flip(image, 1)  # Horizontal flip\n            \n            # Brightness adjustment\n            if random.random() > 0.5:\n                brightness_factor = random.uniform(0.8, 1.2)\n                image = np.clip(image * brightness_factor, 0, 1)\n        except Exception as e:\n            print(f\"Warning: Augmentation failed: {e}\")\n        \n        return image\n\nclass ResNetModel(nn.Module):\n    \"\"\"ResNet model for binary classification\"\"\"\n    \n    def __init__(self, model_name='resnet50', num_classes=2, pretrained=True):\n        super(ResNetModel, self).__init__()\n        \n        if model_name == 'resnet50':\n            self.backbone = models.resnet50(pretrained=pretrained)\n        elif model_name == 'resnet34':\n            self.backbone = models.resnet34(pretrained=pretrained)\n        else:\n            raise ValueError(f\"Unsupported model: {model_name}\")\n        \n        # Replace final fully connected layer\n        in_features = self.backbone.fc.in_features\n        self.backbone.fc = nn.Linear(in_features, num_classes)\n    \n    def forward(self, x):\n        return self.backbone(x)\n    \n    def freeze_backbone(self):\n        \"\"\"Freeze all layers except the final classifier\"\"\"\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n        \n        # Unfreeze the final layer\n        for param in self.backbone.fc.parameters():\n            param.requires_grad = True\n    \n    def unfreeze_all(self):\n        \"\"\"Unfreeze all layers for fine-tuning\"\"\"\n        for param in self.backbone.parameters():\n            param.requires_grad = True\n\nclass Trainer:\n    \"\"\"Training manager for the ResNet model\"\"\"\n    \n    def __init__(self, model, config):\n        self.model = model\n        self.config = config\n        self.device = config.DEVICE\n        self.model.to(self.device)\n        \n        self.criterion = nn.CrossEntropyLoss()\n        self.train_losses = []\n        self.val_losses = []\n        self.train_accuracies = []\n        self.val_accuracies = []\n    \n    def train_phase1(self, train_loader, val_loader):\n        \"\"\"Phase 1: Train only the classifier head\"\"\"\n        print(\"\\n=== Phase 1: Training classifier head ===\")\n        \n        # Freeze backbone\n        self.model.freeze_backbone()\n        \n        optimizer = optim.Adam(\n            filter(lambda p: p.requires_grad, self.model.parameters()),\n            lr=self.config.LEARNING_RATE_PHASE1\n        )\n        \n        self._train_epochs(train_loader, val_loader, optimizer, self.config.EPOCHS_PHASE1, \"Phase1\")\n    \n    def train_phase2(self, train_loader, val_loader):\n        \"\"\"Phase 2: Fine-tune entire model\"\"\"\n        print(\"\\n=== Phase 2: Fine-tuning entire model ===\")\n        \n        # Unfreeze all layers\n        self.model.unfreeze_all()\n        \n        optimizer = optim.Adam(\n            self.model.parameters(),\n            lr=self.config.LEARNING_RATE_PHASE2,\n            weight_decay=1e-4\n        )\n        \n        self._train_epochs(train_loader, val_loader, optimizer, self.config.EPOCHS_PHASE2, \"Phase2\")\n    \n    def _train_epochs(self, train_loader, val_loader, optimizer, epochs, phase_name):\n        \"\"\"Train for specified number of epochs\"\"\"\n        best_val_acc = 0.0\n        \n        for epoch in range(epochs):\n            # Training\n            train_loss, train_acc = self._train_epoch(train_loader, optimizer)\n            \n            # Validation\n            val_loss, val_acc = self._validate_epoch(val_loader)\n            \n            # Store metrics\n            self.train_losses.append(train_loss)\n            self.val_losses.append(val_loss)\n            self.train_accuracies.append(train_acc)\n            self.val_accuracies.append(val_acc)\n            \n            print(f\"{phase_name} Epoch {epoch+1}/{epochs}\")\n            print(f\"Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}\")\n            print(f\"Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.4f}\")\n            print(\"-\" * 50)\n            \n            # Save best model\n            if val_acc > best_val_acc:\n                best_val_acc = val_acc\n                torch.save(self.model.state_dict(), f'best_model_{phase_name.lower()}.pth')\n    \n    def _train_epoch(self, train_loader, optimizer):\n        \"\"\"Train for one epoch\"\"\"\n        self.model.train()\n        total_loss = 0.0\n        correct = 0\n        total = 0\n        \n        for batch_idx, (data, target) in enumerate(tqdm(train_loader, desc=\"Training\")):\n            try:\n                data, target = data.to(self.device), target.to(self.device)\n                \n                optimizer.zero_grad()\n                output = self.model(data)\n                loss = self.criterion(output, target)\n                loss.backward()\n                optimizer.step()\n                \n                total_loss += loss.item()\n                pred = output.argmax(dim=1)\n                correct += pred.eq(target).sum().item()\n                total += target.size(0)\n            except Exception as e:\n                print(f\"Warning: Error in training batch {batch_idx}: {e}\")\n                continue\n        \n        avg_loss = total_loss / len(train_loader) if len(train_loader) > 0 else 0\n        accuracy = correct / total if total > 0 else 0\n        \n        return avg_loss, accuracy\n    \n    def _validate_epoch(self, val_loader):\n        \"\"\"Validate for one epoch\"\"\"\n        self.model.eval()\n        total_loss = 0.0\n        correct = 0\n        total = 0\n        \n        with torch.no_grad():\n            for data, target in tqdm(val_loader, desc=\"Validation\"):\n                try:\n                    data, target = data.to(self.device), target.to(self.device)\n                    output = self.model(data)\n                    loss = self.criterion(output, target)\n                    \n                    total_loss += loss.item()\n                    pred = output.argmax(dim=1)\n                    correct += pred.eq(target).sum().item()\n                    total += target.size(0)\n                except Exception as e:\n                    print(f\"Warning: Error in validation batch: {e}\")\n                    continue\n        \n        avg_loss = total_loss / len(val_loader) if len(val_loader) > 0 else 0\n        accuracy = correct / total if total > 0 else 0\n        \n        return avg_loss, accuracy\n    \n    def evaluate(self, test_loader):\n        \"\"\"Evaluate model on test set\"\"\"\n        self.model.eval()\n        all_preds = []\n        all_targets = []\n        \n        with torch.no_grad():\n            for data, target in tqdm(test_loader, desc=\"Evaluating\"):\n                try:\n                    data, target = data.to(self.device), target.to(self.device)\n                    output = self.model(data)\n                    pred = output.argmax(dim=1)\n                    \n                    all_preds.extend(pred.cpu().numpy())\n                    all_targets.extend(target.cpu().numpy())\n                except Exception as e:\n                    print(f\"Warning: Error in evaluation batch: {e}\")\n                    continue\n        \n        if len(all_targets) == 0:\n            return 0.0, \"No valid predictions\", np.array([[0, 0], [0, 0]])\n        \n        accuracy = accuracy_score(all_targets, all_preds)\n        report = classification_report(all_targets, all_preds)\n        cm = confusion_matrix(all_targets, all_preds)\n        \n        return accuracy, report, cm\n    \n    def plot_training_history(self):\n        \"\"\"Plot training history\"\"\"\n        if not self.train_losses:\n            print(\"No training history to plot\")\n            return\n            \n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 5))\n        \n        # Plot losses\n        ax1.plot(self.train_losses, label='Train Loss')\n        ax1.plot(self.val_losses, label='Validation Loss')\n        ax1.set_title('Training and Validation Loss')\n        ax1.set_xlabel('Epoch')\n        ax1.set_ylabel('Loss')\n        ax1.legend()\n        ax1.grid(True)\n        \n        # Plot accuracies\n        ax2.plot(self.train_accuracies, label='Train Accuracy')\n        ax2.plot(self.val_accuracies, label='Validation Accuracy')\n        ax2.set_title('Training and Validation Accuracy')\n        ax2.set_xlabel('Epoch')\n        ax2.set_ylabel('Accuracy')\n        ax2.legend()\n        ax2.grid(True)\n        \n        plt.tight_layout()\n        plt.savefig('training_history.png', dpi=300, bbox_inches='tight')\n        plt.show()\n\nclass MedicalImagingPipeline:\n    \"\"\"Main pipeline class that orchestrates the entire process\"\"\"\n    \n    def __init__(self, config):\n        self.config = config\n        \n    def run_complete_pipeline(self):\n        \"\"\"Run the complete pipeline from data loading to model evaluation\"\"\"\n        print(\"Starting Medical Imaging Pipeline for Aneurysm Detection\")\n        print(f\"Device: {self.config.DEVICE}\")\n        print(\"=\" * 60)\n        \n        # Step 1: Load metadata and sample balanced data\n        print(\"\\n1. Loading metadata and creating balanced dataset...\")\n        sampler = DataSampler(self.config)\n        train_df, localizers_df = sampler.load_metadata()\n        balanced_data = sampler.sample_balanced_data(train_df)\n        \n        # Step 2: Prepare datasets for each modality\n        print(\"\\n2. Preparing datasets...\")\n        all_datasets = {}\n        \n        for modality, data in balanced_data.items():\n            print(f\"\\nProcessing {modality}...\")\n            \n            # Combine positive and negative samples\n            combined_data = pd.concat([data['positive'], data['negative']], ignore_index=True)\n            \n            # Split into train and validation\n            train_data, val_data = train_test_split(\n                combined_data, \n                test_size=self.config.VALIDATION_SPLIT, \n                stratify=combined_data['Aneurysm Present'],\n                random_state=42\n            )\n            \n            print(f\"Train samples: {len(train_data)}, Validation samples: {len(val_data)}\")\n            \n            # Define transforms\n            train_transform = transforms.Compose([\n                transforms.RandomRotation(10),\n                transforms.RandomHorizontalFlip(),\n                transforms.ColorJitter(brightness=0.1, contrast=0.1),\n                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n            ])\n            \n            val_transform = transforms.Compose([\n                transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n            ])\n            \n            # Create datasets\n            train_dataset = MedicalImageDataset(train_data, self.config, train_transform, is_training=True)\n            val_dataset = MedicalImageDataset(val_data, self.config, val_transform, is_training=False)\n            \n            # Create data loaders with error handling\n            train_loader = DataLoader(\n                train_dataset, \n                batch_size=self.config.BATCH_SIZE, \n                shuffle=True, \n                num_workers=0,  # Set to 0 to avoid multiprocessing issues\n                pin_memory=False\n            )\n            val_loader = DataLoader(\n                val_dataset, \n                batch_size=self.config.BATCH_SIZE, \n                shuffle=False, \n                num_workers=0,\n                pin_memory=False\n            )\n            \n            all_datasets[modality] = {\n                'train_loader': train_loader,\n                'val_loader': val_loader,\n                'train_size': len(train_dataset),\n                'val_size': len(val_dataset)\n            }\n        \n        # Step 3: Train models for each modality\n        print(\"\\n3. Training models...\")\n        trained_models = {}\n        \n        for modality, datasets in all_datasets.items():\n            print(f\"\\n{'='*20} Training {modality} Model {'='*20}\")\n            \n            # Initialize model\n            model = ResNetModel(\n                model_name=self.config.MODEL_NAME,\n                num_classes=self.config.NUM_CLASSES,\n                pretrained=True\n            )\n            \n            # Initialize trainer\n            trainer = Trainer(model, self.config)\n            \n            # Two-phase training\n            trainer.train_phase1(datasets['train_loader'], datasets['val_loader'])\n            trainer.train_phase2(datasets['train_loader'], datasets['val_loader'])\n            \n            # Evaluate\n            print(f\"\\nEvaluating {modality} model...\")\n            accuracy, report, cm = trainer.evaluate(datasets['val_loader'])\n            \n            print(f\"Final Validation Accuracy: {accuracy:.4f}\")\n            print(\"\\nClassification Report:\")\n            print(report)\n            \n            # Plot confusion matrix\n            try:\n                plt.figure(figsize=(8, 6))\n                sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n                plt.title(f'Confusion Matrix - {modality}')\n                plt.ylabel('True Label')\n                plt.xlabel('Predicted Label')\n                plt.savefig(f'confusion_matrix_{modality}.png', dpi=300, bbox_inches='tight')\n                plt.show()\n            except Exception as e:\n                print(f\"Warning: Could not plot confusion matrix: {e}\")\n            \n            # Plot training history\n            trainer.plot_training_history()\n            \n            trained_models[modality] = {\n                'model': model,\n                'trainer': trainer,\n                'accuracy': accuracy,\n                'report': report,\n                'confusion_matrix': cm\n            }\n        \n        # Step 4: Summary\n        print(\"\\n\" + \"=\"*60)\n        print(\"TRAINING SUMMARY\")\n        print(\"=\"*60)\n        \n        for modality, results in trained_models.items():\n            print(f\"{modality}: Validation Accuracy = {results['accuracy']:.4f}\")\n        \n        return trained_models\n\ndef main():\n    \"\"\"Main execution function\"\"\"\n    # Initialize configuration\n    config = Config()\n    \n    # Verify data paths exist\n    if not os.path.exists(config.DATA_ROOT):\n        print(f\"Error: Data root path does not exist: {config.DATA_ROOT}\")\n        print(\"Please update the DATA_ROOT path in the Config class\")\n        return\n    \n    if not os.path.exists(config.TRAIN_CSV):\n        print(f\"Error: train.csv not found at: {config.TRAIN_CSV}\")\n        return\n    \n    # Create and run pipeline\n    pipeline = MedicalImagingPipeline(config)\n    trained_models = pipeline.run_complete_pipeline()\n    \n    print(\"\\nPipeline completed successfully!\")\n    print(f\"Trained models available for {len(trained_models)} modalities\")\n\nif __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-07T07:36:11.015024Z","iopub.execute_input":"2025-10-07T07:36:11.015809Z","iopub.status.idle":"2025-10-07T12:51:43.235867Z","shell.execute_reply.started":"2025-10-07T07:36:11.015779Z","shell.execute_reply":"2025-10-07T12:51:43.230319Z"}},"outputs":[],"execution_count":null}]}