{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":99552,"databundleVersionId":13441085}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q monai timm pydicom nibabel torchio","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-07T18:11:37.347815Z","iopub.execute_input":"2025-09-07T18:11:37.348723Z","iopub.status.idle":"2025-09-07T18:11:40.86934Z","shell.execute_reply.started":"2025-09-07T18:11:37.348697Z","shell.execute_reply":"2025-09-07T18:11:40.868308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import os\n# import multiprocessing as mp\n# import numpy as np\n# import pandas as pd\n# import torch\n# import torch.nn as nn\n# from torch.utils.data import Dataset, DataLoader\n# import monai\n# from monai.data import CacheDataset, DataLoader as MonaiDataLoader\n# from monai.transforms import (\n#     EnsureChannelFirstd, Resized, NormalizeIntensityd, RandFlipd,\n#     RandRotated, Compose, ToTensord\n# )\n# import pydicom\n# from pathlib import Path\n# from tqdm import tqdm\n# from functools import partial\n# from sklearn.metrics import roc_auc_score\n# import SimpleITK as sitk\n# import h5py\n# import torchio as tio\n# from collections import OrderedDict\n# from typing import Tuple, List\n# from scipy import ndimage\n# import ast\n\n\n# # Config\n# DATA_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series/\"\n# TRAIN_CSV = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\n# LOCALIZERS_CSV = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train_localizers.csv\"\n# SEGMENTATIONS_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection/segmentations/\"\n# OUTPUT_DIR = \"/kaggle/working/preprocessed/\"\n# BATCH_SIZE = 8\n# NUM_EPOCHS = 5\n# LR = 1e-4\n# DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# TARGET_SIZE = (128, 128, 64)  # Downsample resolution\n# TARGET_SPACING_MM = 1.0  # Isotropic spacing\n# CTA_WINDOW = (-1000, 1000)  # HU window for CTA\n# MRI_Z_CLIP = 3.0  # Z-score clip for MRI\n# LRU_CAPACITY = 10  # Memory cache capacity\n# ID_COL = \"SeriesInstanceUID\"\n# LABEL_COLS = [\n#     \"Left Infraclinoid Internal Carotid Artery\",\n#     \"Right Infraclinoid Internal Carotid Artery\",\n#     \"Left Supraclinoid Internal Carotid Artery\",\n#     \"Right Supraclinoid Internal Carotid Artery\",\n#     \"Left Middle Cerebral Artery\",\n#     \"Right Middle Cerebral Artery\",\n#     \"Anterior Communicating Artery\",\n#     \"Left Anterior Cerebral Artery\",\n#     \"Right Anterior Cerebral Artery\",\n#     \"Left Posterior Communicating Artery\",\n#     \"Right Posterior Communicating Artery\",\n#     \"Basilar Tip\",\n#     \"Other Posterior Circulation\",\n#     \"Aneurysm Present\"\n# ]\n\n# # Utility Functions\n# def _safe_zoom(volume: np.ndarray, zoom_factors: Tuple[float, ...], order: int = 1) -> np.ndarray:\n#     \"\"\"Robust wrapper around ndimage.zoom to avoid rank mismatch and invalid factors.\"\"\"\n#     volume = np.nan_to_num(volume, copy=False)\n#     zf = tuple(float(max(1e-6, f)) for f in zoom_factors)  # avoid zeros/negatives\n#     if len(zf) != volume.ndim:\n#         if len(zf) > volume.ndim:\n#             zf = zf[:volume.ndim]\n#         else:\n#             zf = (1.0,) * (volume.ndim - len(zf)) + zf\n#     return ndimage.zoom(volume, zf, order=order)\n\n# def _resize_slice(arr: np.ndarray, out_h: int, out_w: int) -> np.ndarray:\n#     \"\"\"Resize a 2D slice to (out_h, out_w) using safe zoom.\"\"\"\n#     h, w = arr.shape\n#     if h == out_h and w == out_w:\n#         return arr.astype(np.float32, copy=False)\n#     zy = out_h / max(h, 1)\n#     zx = out_w / max(w, 1)\n#     return _safe_zoom(arr, (zy, zx), order=1).astype(np.float32, copy=False)\n\n# # DICOM Processor Class\n# class DICOMProcessor:\n#     def __init__(\n#         self,\n#         target_size: Tuple[int, int, int] = TARGET_SIZE,\n#         target_spacing_mm: float = TARGET_SPACING_MM,\n#         cta_window: Tuple[float, float] = CTA_WINDOW,\n#         mri_z_clip: float = MRI_Z_CLIP,\n#         lru_capacity: int = LRU_CAPACITY,\n#     ):\n#         self.target_size = target_size\n#         self.target_spacing_mm = target_spacing_mm\n#         self.cta_window = cta_window\n#         self.mri_z_clip = mri_z_clip\n#         self.memory_cache = OrderedDict()\n#         self.lru_capacity = lru_capacity\n\n#     def _cache_put(self, key: str, vol: np.ndarray):\n#         self.memory_cache[key] = vol\n#         self.memory_cache.move_to_end(key)\n#         if len(self.memory_cache) > self.lru_capacity:\n#             self.memory_cache.popitem(last=False)\n\n#     def _cache_get(self, key: str):\n#         if key in self.memory_cache:\n#             vol = self.memory_cache[key]\n#             self.memory_cache.move_to_end(key)\n#             return vol\n#         return None\n\n#     def _sort_slices(self, ds_list: List[pydicom.dataset.FileDataset]) -> List[pydicom.dataset.FileDataset]:\n#         try:\n#             orient = np.array(ds_list[0].ImageOrientationPatient, dtype=np.float32)\n#             row, col = orient[:3], orient[3:]\n#             normal = np.cross(row, col)\n#             return sorted(ds_list, key=lambda ds: float(np.dot(np.array(ds.ImagePositionPatient, dtype=np.float32), normal)))\n#         except Exception:\n#             return sorted(ds_list, key=lambda ds: getattr(ds, \"InstanceNumber\", 0))\n\n#     def _get_spacing(self, ds_sorted: List[pydicom.dataset.FileDataset], has_multiframe: bool = False) -> Tuple[float, float, float]:\n#         try:\n#             dy, dx = map(float, ds_sorted[0].PixelSpacing)\n#         except Exception:\n#             dy, dx = 1.0, 1.0\n#         if has_multiframe:\n#             dz = float(getattr(ds_sorted[0], \"SpacingBetweenSlices\", getattr(ds_sorted[0], \"SliceThickness\", 1.0)))\n#         else:\n#             zs = [np.linalg.norm(np.array(ds_sorted[i].ImagePositionPatient, dtype=np.float32) - np.array(ds_sorted[i-1].ImagePositionPatient, dtype=np.float32)) for i in range(1, len(ds_sorted))]\n#             dz = np.median([z for z in zs if z > 0]) if zs else float(getattr(ds_sorted[0], \"SliceThickness\", 1.0))\n#         return (dz if dz > 0 else 1.0, dy if dy > 0 else 1.0, dx if dx > 0 else 1.0)\n\n#     def _choose_base_shape(self, ds_list: List[pydicom.dataset.FileDataset]) -> Tuple[int, int]:\n#         shapes = [(int(ds.Rows), int(ds.Columns)) for ds in ds_list if hasattr(ds, \"Rows\") and hasattr(ds, \"Columns\")]\n#         if not shapes:\n#             shapes = [(ds.pixel_array.shape[-2], ds.pixel_array.shape[-1]) for ds in ds_list if hasattr(ds, \"pixel_array\")]\n#         vals, counts = np.unique(shapes, return_counts=True, axis=0)\n#         return tuple(int(x) for x in vals[counts.argmax()]) if vals.size else (512, 512)\n\n#     def _normalize_by_modality(self, volume: np.ndarray, modality_tag: str) -> np.ndarray:\n#         volume = np.nan_to_num(volume, copy=False)\n#         if modality_tag == \"CT\":\n#             c, w = self.cta_window\n#             lo, hi = c - w / 2.0, c + w / 2.0\n#             return np.clip((volume - lo) / (hi - lo + 1e-6), 0, 1).astype(np.float32)\n#         else:\n#             mean = float(volume.mean())\n#             std = float(volume.std() + 1e-6)\n#             v = np.clip((volume - mean) / std, -self.mri_z_clip, self.mri_z_clip)\n#             return ((v + self.mri_z_clip) / (2.0 * self.mri_z_clip)).astype(np.float32)\n\n#     def load_dicom_series(self, series_path: str) -> np.ndarray:\n#         series_id = os.path.basename(series_path)\n#         m = self._cache_get(series_id)\n#         if m is not None and m.shape == self.target_size:\n#             return m\n\n#         try:\n#             dicoms = []\n#             for root, _, files in os.walk(series_path):\n#                 for f in files:\n#                     if f.endswith(\".dcm\"):\n#                         try:\n#                             ds = pydicom.dcmread(os.path.join(root, f), force=True)\n#                             if hasattr(ds, \"PixelData\"):\n#                                 dicoms.append(ds)\n#                         except Exception as e:\n#                             print(f\"[DICOM read] {series_id}: {e}\")\n#                             continue\n#             if not dicoms:\n#                 raise ValueError(f\"No valid DICOM files in {series_path}\")\n\n#             dicoms = self._sort_slices(dicoms)\n#             has_multiframe = any(getattr(ds, \"NumberOfFrames\", 1) > 1 for ds in dicoms)\n#             spacing = self._get_spacing(dicoms, has_multiframe)\n#             base_h, base_w = self._choose_base_shape(dicoms)\n#             modality_tag = getattr(dicoms[0], \"Modality\", \"\").upper()\n\n#             vol_slices = []\n#             for ds in dicoms:\n#                 arr = ds.pixel_array\n#                 if arr.ndim >= 3:\n#                     h, w = arr.shape[-2], arr.shape[-1]\n#                     n = int(np.prod(arr.shape[:-2]))\n#                     arr = arr.reshape(n, h, w)\n#                 else:\n#                     arr = arr[np.newaxis, ...]\n#                 for sl in arr:\n#                     sl = sl.astype(np.float32)\n#                     if getattr(ds, \"PhotometricInterpretation\", \"MONOCHROME2\") == \"MONOCHROME1\":\n#                         sl = sl.max() - sl\n#                     slope = float(getattr(ds, \"RescaleSlope\", 1.0))\n#                     intercept = float(getattr(ds, \"RescaleIntercept\", 0.0))\n#                     sl = sl * slope + intercept\n#                     sl = _resize_slice(sl, base_h, base_w)\n#                     vol_slices.append(sl)\n\n#             if not vol_slices:\n#                 raise ValueError(f\"No valid slices in {series_id}\")\n#             volume = np.stack(vol_slices, axis=0)\n\n#             # Resample to target spacing\n#             dz, dy, dx = spacing\n#             z, y, x = volume.shape\n#             newD = max(1, int(round(z * dz / self.target_spacing_mm)))\n#             newH = max(1, int(round(y * dy / self.target_spacing_mm)))\n#             newW = max(1, int(round(x * dx / self.target_spacing_mm)))\n#             volume = _safe_zoom(volume, (newD / z, newH / y, newW / x), order=1)\n\n#             # Resize to target grid\n#             tz, ty, tx = self.target_size\n#             z, y, x = volume.shape\n#             volume = _safe_zoom(volume, (tz / z, ty / y, tx / x), order=1).astype(np.float32)\n\n#             volume = self._normalize_by_modality(volume, modality_tag)\n#             self._cache_put(series_id, volume)\n#             return volume\n\n#         except Exception as e:\n#             print(f\"[Processor] Error processing {series_id}: {e}\")\n#             vol = np.zeros(self.target_size, dtype=np.float32)\n#             self._cache_put(series_id, vol)\n#             return vol\n\n\n# def process_row(args):\n#     row_dict, data_dir, label_cols, processor = args\n#     series_id = row_dict[\"SeriesInstanceUID\"]\n#     series_path = os.path.join(data_dir, series_id)\n\n#     # Labels\n#     labels = [row_dict.get(col, 0) for col in label_cols]\n\n#     # Load DICOM\n#     volume = processor.load_dicom_series(series_path)\n#     return series_path, volume, np.array(labels, dtype=np.float32)\n\n# # def preprocess_dataset():\n# #     os.makedirs(OUTPUT_DIR, exist_ok=True)\n# #     train_df = pd.read_csv(TRAIN_CSV)\n\n# #     # Keep only aneurysm cases\n# #     train_df = train_df[train_df[\"Aneurysm Present\"] == 1].copy()\n# #     print(f\"Preprocessing {len(train_df)} series with aneurysm\")\n\n# #     # --- Load and parse localizers ---\n# #     localizers_df = pd.read_csv(LOCALIZERS_CSV)\n\n# #     def parse_coordinates(coord_str):\n# #         try:\n# #             d = ast.literal_eval(coord_str)\n# #             return pd.Series({\n# #                 \"center_x\": d.get(\"x\", np.nan),\n# #                 \"center_y\": d.get(\"y\", np.nan),\n# #                 \"center_z\": d.get(\"z\", np.nan) if \"z\" in d else np.nan\n# #             })\n# #         except Exception:\n# #             return pd.Series({\"center_x\": np.nan, \"center_y\": np.nan, \"center_z\": np.nan})\n\n# #     coord_df = localizers_df[\"coordinates\"].apply(parse_coordinates)\n# #     localizers_df = pd.concat([localizers_df.drop(columns=[\"coordinates\"]), coord_df], axis=1)\n\n# #     # Average coordinates per series\n# #     localizers_df = localizers_df.groupby(\"SeriesInstanceUID\")[[\"center_x\", \"center_y\", \"center_z\"]].mean().reset_index()\n\n# #     # Merge with training data\n# #     train_df = train_df.merge(localizers_df, on=\"SeriesInstanceUID\", how=\"left\")\n# #     train_df = train_df[:100]\n\n# #     # Fill NaNs with global mean\n# #     for col in [\"center_x\", \"center_y\", \"center_z\"]:\n# #         train_df[col] = train_df[col].fillna(train_df[col].mean())\n\n# #     # --- Prepare multiprocessing args ---\n# #     processor = DICOMProcessor(\n# #         target_size=TARGET_SIZE,\n# #         target_spacing_mm=TARGET_SPACING_MM,\n# #         cta_window=CTA_WINDOW,\n# #         mri_z_clip=MRI_Z_CLIP\n# #     )\n\n# #     args = [(row._asdict(), DATA_DIR, LABEL_COLS, processor) \n# #             for row in train_df.itertuples(index=False)]\n\n# #     series_data = []\n# #     with mp.Pool(processes=min(mp.cpu_count(), 4)) as pool:\n# #         results = list(tqdm(pool.imap(process_row, args),\n# #                             total=len(train_df), desc=\"Preprocessing\"))\n\n# #     for series_path, volume, labels in results:\n# #         if volume is not None and not np.all(volume == 0):\n# #             series_id = os.path.basename(series_path)\n# #             np.save(os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"), volume)\n# #             series_data.append({\n# #                 \"image\": os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"),\n# #                 \"labels\": labels\n# #             })\n\n# #     pd.DataFrame(series_data).to_csv(os.path.join(OUTPUT_DIR, \"series_metadata.csv\"), index=False)\n# #     print(\"Preprocessing done!\")\n\n# # # Dataset\n# # class AneurysmDataset(Dataset):\n# #     def __init__(self, data_file: str, transform=None):\n# #         self.data = pd.read_csv(data_file)\n# #         self.transform = transform\n\n# #     def __len__(self):\n# #         return len(self.data)\n\n# #     def __getitem__(self, idx):\n# #         row = self.data.iloc[idx]\n# #         image = np.load(row[\"image\"]).astype(np.float32)\n# #         labels = np.array(eval(row[\"labels\"]) if isinstance(row[\"labels\"], str) else row[\"labels\"], dtype=np.float32)\n        \n# #         # Light augmentation with torchio\n# #         if self.transform:\n# #             subject = tio.Subject(image=tio.ScalarImage(tensor=image[np.newaxis, ...]))\n# #             transformed = self.transform(subject)\n# #             image = transformed[\"image\"].numpy().squeeze(0)\n\n# #         image_tensor = torch.from_numpy(image).unsqueeze(0) if image.ndim == 3 else torch.from_numpy(image)  # Ensure (1, D, H, W)\n# #         labels_tensor = torch.from_numpy(labels)\n# #         return image_tensor, labels_tensor\n\n# # train_transforms = tio.Compose([\n# #     tio.RandomFlip(axes=(0,), p=0.5),\n# #     tio.RandomAffine(degrees=10, p=0.5),\n# #     tio.ToCanonical()  # Ensure canonical orientation, implicit tensor conversion\n# # ])\n# # val_transforms = tio.Compose([\n# #     tio.ToCanonical()  # Ensure canonical orientation\n# # ])\n\n\n# # Preprocessing Function\n# def preprocess_dataset():\n#     os.makedirs(OUTPUT_DIR, exist_ok=True)\n#     train_df = pd.read_csv(TRAIN_CSV)\n\n#     # Keep only aneurysm cases\n#     train_df = train_df[train_df[\"Aneurysm Present\"] == 1].copy()\n#     print(f\"Preprocessing {len(train_df)} series with aneurysm\")\n\n#     # --- Load and parse localizers ---\n#     localizers_df = pd.read_csv(LOCALIZERS_CSV)\n\n#     def parse_coordinates(coord_str):\n#         try:\n#             d = ast.literal_eval(coord_str)\n#             return pd.Series({\n#                 \"center_x\": d.get(\"x\", np.nan),\n#                 \"center_y\": d.get(\"y\", np.nan),\n#                 \"center_z\": d.get(\"z\", np.nan) if \"z\" in d else np.nan\n#             })\n#         except Exception:\n#             return pd.Series({\"center_x\": np.nan, \"center_y\": np.nan, \"center_z\": np.nan})\n\n#     coord_df = localizers_df[\"coordinates\"].apply(parse_coordinates)\n#     localizers_df = pd.concat([localizers_df.drop(columns=[\"coordinates\"]), coord_df], axis=1)\n\n#     # Average coordinates per series\n#     localizers_df = localizers_df.groupby(\"SeriesInstanceUID\")[[\"center_x\", \"center_y\", \"center_z\"]].mean().reset_index()\n\n#     # Merge with training data\n#     train_df = train_df.merge(localizers_df, on=\"SeriesInstanceUID\", how=\"left\")\n#     train_df = train_df[:50]\n\n#     # Fill NaNs with global mean\n#     for col in [\"center_x\", \"center_y\", \"center_z\"]:\n#         train_df[col] = train_df[col].fillna(train_df[col].mean())\n\n#     # --- Prepare multiprocessing args ---\n#     processor = DICOMProcessor(\n#         target_size=TARGET_SIZE,\n#         target_spacing_mm=TARGET_SPACING_MM,\n#         cta_window=CTA_WINDOW,\n#         mri_z_clip=MRI_Z_CLIP\n#     )\n\n#     args = [(row._asdict(), DATA_DIR, LABEL_COLS, processor) \n#             for row in train_df.itertuples(index=False)]\n\n#     series_data = []\n#     with mp.Pool(processes=min(mp.cpu_count(), 4)) as pool:\n#         results = list(tqdm(pool.imap(process_row, args),\n#                             total=len(train_df), desc=\"Preprocessing\"))\n\n#     for series_path, volume, labels in results:\n#         if volume is not None and not np.all(volume == 0):\n#             series_id = os.path.basename(series_path)\n#             np.save(os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"), volume)\n#             # Save labels as a JSON-compatible string\n#             series_data.append({\n#                 \"image\": os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"),\n#                 \"labels\": labels.tolist()  # Convert to list for JSON compatibility\n#             })\n\n#     pd.DataFrame(series_data).to_csv(os.path.join(OUTPUT_DIR, \"series_metadata.csv\"), index=False)\n#     print(\"Preprocessing done!\")\n\n# # Dataset\n# class AneurysmDataset(Dataset):\n#     def __init__(self, data_file: str, transform=None):\n#         self.data = pd.read_csv(data_file)\n#         self.transform = transform\n\n#     def __len__(self):\n#         return len(self.data)\n\n#     def __getitem__(self, idx):\n#         row = self.data.iloc[idx]\n#         image = np.load(row[\"image\"]).astype(np.float32)\n#         # Parse labels string using ast.literal_eval\n#         labels = np.array(ast.literal_eval(row[\"labels\"]), dtype=np.float32) if isinstance(row[\"labels\"], str) else np.array(row[\"labels\"], dtype=np.float32)\n        \n#         # Light augmentation with torchio\n#         if self.transform:\n#             subject = tio.Subject(image=tio.ScalarImage(tensor=image[np.newaxis, ...]))\n#             transformed = self.transform(subject)\n#             image = transformed[\"image\"].numpy().squeeze(0)\n\n#         image_tensor = torch.from_numpy(image).unsqueeze(0) if image.ndim == 3 else torch.from_numpy(image)  # Ensure (1, D, H, W)\n#         labels_tensor = torch.from_numpy(labels)\n#         return image_tensor, labels_tensor\n# # Model\n# class EfficientNet3D(nn.Module):\n#     def __init__(self, num_classes=14):\n#         super().__init__()\n#         self.backbone = monai.networks.nets.EfficientNetBN(\"efficientnet-b0\", spatial_dims=3, in_channels=1)\n#         self.pool = nn.AdaptiveAvgPool3d(1)\n#         self.fc = nn.Linear(self.backbone._fc.in_features, num_classes)\n#         self.backbone._fc = nn.Identity()\n\n#     def forward(self, x):\n#         x = self.backbone(x)\n#         x = self.pool(x).view(x.size(0), -1)\n#         x = self.fc(x)\n#         return torch.sigmoid(x)\n\n        \n# # Training Loop\n# def train_one_epoch(model, loader, optimizer, scaler, criterion):\n#     model.train()\n#     losses = []\n#     for batch in tqdm(loader, desc=\"Training\"):\n#         images, labels = batch[0].to(DEVICE), batch[1].to(DEVICE)\n#         optimizer.zero_grad()\n#         with torch.cuda.amp.autocast():\n#             outputs = model(images)\n#             loss = criterion(outputs, labels)\n#         scaler.scale(loss).backward()\n#         scaler.step(optimizer)\n#         scaler.update()\n#         losses.append(loss.item())\n#     return np.mean(losses)\n\n# def validate(model, loader, criterion):\n#     model.eval()\n#     preds, truths = [], []\n#     losses = []\n#     with torch.no_grad():\n#         for batch in tqdm(loader, desc=\"Validating\"):\n#             images, labels = batch[0].to(DEVICE), batch[1].to(DEVICE)\n#             outputs = model(images)\n#             loss = criterion(outputs, labels)\n#             losses.append(loss.item())\n#             preds.append(outputs.cpu().numpy())\n#             truths.append(labels.cpu().numpy())\n#     preds = np.concatenate(preds)\n#     truths = np.concatenate(truths)\n#     aucs = [roc_auc_score(truths[:, i], preds[:, i]) for i in range(len(LABEL_COLS))]\n#     final_score = (aucs[-1] + np.mean(aucs[:-1])) / 2\n#     return np.mean(losses), final_score, aucs\n\n# # Main\n# def main():\n#     # Preprocess\n#     preprocess_dataset()\n\n#     # Data\n#     metadata_file = os.path.join(OUTPUT_DIR, \"series_metadata.csv\")\n#     train_idx = int(0.8 * len(pd.read_csv(metadata_file)))\n#     train_data = pd.read_csv(metadata_file)[:train_idx]\n#     val_data = pd.read_csv(metadata_file)[train_idx:]\n#     train_dataset = CacheDataset(AneurysmDataset(metadata_file, train_transforms), cache_rate=0.1, num_workers=4)\n#     val_dataset = CacheDataset(AneurysmDataset(metadata_file, val_transforms), cache_rate=0.1, num_workers=4)\n#     train_loader = MonaiDataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, prefetch_factor=2)\n#     val_loader = MonaiDataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, prefetch_factor=2)\n\n#     # Model\n#     model = EfficientNet3D(num_classes=len(LABEL_COLS)).to(DEVICE)\n#     criterion = nn.BCELoss()\n#     optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n#     scaler = torch.cuda.amp.GradScaler()\n\n#     # Train\n#     best_score = 0\n#     for epoch in range(NUM_EPOCHS):\n#         train_loss = train_one_epoch(model, train_loader, optimizer, scaler, criterion)\n#         val_loss, val_score, aucs = validate(model, val_loader, criterion)\n#         print(f\"Epoch {epoch+1}/{NUM_EPOCHS}, Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}, Val AUC: {val_score:.4f}\")\n#         if val_score > best_score:\n#             best_score = val_score\n#             torch.save(model.state_dict(), \"best_model.pth\")\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-07T18:11:40.871119Z","iopub.execute_input":"2025-09-07T18:11:40.871448Z","iopub.status.idle":"2025-09-07T18:11:40.884825Z","shell.execute_reply.started":"2025-09-07T18:11:40.871402Z","shell.execute_reply":"2025-09-07T18:11:40.884057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport multiprocessing as mp\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport monai\nfrom monai.data import CacheDataset, DataLoader as MonaiDataLoader\nfrom monai.transforms import (\n    EnsureChannelFirstd, Resized, NormalizeIntensityd, RandFlipd,\n    RandRotated, Compose, ToTensord\n)\nimport pydicom\nfrom pathlib import Path\nfrom tqdm import tqdm\nfrom functools import partial\nfrom sklearn.metrics import roc_auc_score\nimport SimpleITK as sitk\nimport h5py\nimport torchio as tio\nfrom collections import OrderedDict\nfrom typing import Tuple, List\nfrom scipy import ndimage\nimport ast\nfrom sklearn.model_selection import train_test_split\n\n# Config\nDATA_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection/series/\"\nTRAIN_CSV = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train.csv\"\nLOCALIZERS_CSV = \"/kaggle/input/rsna-intracranial-aneurysm-detection/train_localizers.csv\"\nSEGMENTATIONS_DIR = \"/kaggle/input/rsna-intracranial-aneurysm-detection/segmentations/\"\nOUTPUT_DIR = \"/kaggle/working/preprocessed/\"\nBATCH_SIZE = 8\nNUM_EPOCHS = 30\nLR = 1e-4\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nTARGET_SIZE = (128, 128, 64)  # Downsample resolution\nTARGET_SPACING_MM = 1.0  # Isotropic spacing\nCTA_WINDOW = (-1000, 1000)  # HU window for CTA\nMRI_Z_CLIP = 3.0  # Z-score clip for MRI\nLRU_CAPACITY = 10  # Memory cache capacity\nID_COL = \"SeriesInstanceUID\"\nLABEL_COLS = [\n    \"Left Infraclinoid Internal Carotid Artery\",\n    \"Right Infraclinoid Internal Carotid Artery\",\n    \"Left Supraclinoid Internal Carotid Artery\",\n    \"Right Supraclinoid Internal Carotid Artery\",\n    \"Left Middle Cerebral Artery\",\n    \"Right Middle Cerebral Artery\",\n    \"Anterior Communicating Artery\",\n    \"Left Anterior Cerebral Artery\",\n    \"Right Anterior Cerebral Artery\",\n    \"Left Posterior Communicating Artery\",\n    \"Right Posterior Communicating Artery\",\n    \"Basilar Tip\",\n    \"Other Posterior Circulation\",\n    \"Aneurysm Present\"\n]\n\n# Utility Functions\ndef _safe_zoom(volume: np.ndarray, zoom_factors: Tuple[float, ...], order: int = 1) -> np.ndarray:\n    \"\"\"Robust wrapper around ndimage.zoom to avoid rank mismatch and invalid factors.\"\"\"\n    volume = np.nan_to_num(volume, copy=False)\n    zf = tuple(float(max(1e-6, f)) for f in zoom_factors)  # avoid zeros/negatives\n    if len(zf) != volume.ndim:\n        if len(zf) > volume.ndim:\n            zf = zf[:volume.ndim]\n        else:\n            zf = (1.0,) * (volume.ndim - len(zf)) + zf\n    return ndimage.zoom(volume, zf, order=order)\n\ndef _resize_slice(arr: np.ndarray, out_h: int, out_w: int) -> np.ndarray:\n    \"\"\"Resize a 2D slice to (out_h, out_w) using safe zoom.\"\"\"\n    h, w = arr.shape\n    if h == out_h and w == out_w:\n        return arr.astype(np.float32, copy=False)\n    zy = out_h / max(h, 1)\n    zx = out_w / max(w, 1)\n    return _safe_zoom(arr, (zy, zx), order=1).astype(np.float32, copy=False)\n\n# DICOM Processor Class\nclass DICOMProcessor:\n    def __init__(\n        self,\n        target_size: Tuple[int, int, int] = TARGET_SIZE,\n        target_spacing_mm: float = TARGET_SPACING_MM,\n        cta_window: Tuple[float, float] = CTA_WINDOW,\n        mri_z_clip: float = MRI_Z_CLIP,\n        lru_capacity: int = LRU_CAPACITY,\n    ):\n        self.target_size = target_size\n        self.target_spacing_mm = target_spacing_mm\n        self.cta_window = cta_window\n        self.mri_z_clip = mri_z_clip\n        self.memory_cache = OrderedDict()\n        self.lru_capacity = lru_capacity\n\n    def _cache_put(self, key: str, vol: np.ndarray):\n        self.memory_cache[key] = vol\n        self.memory_cache.move_to_end(key)\n        if len(self.memory_cache) > self.lru_capacity:\n            self.memory_cache.popitem(last=False)\n\n    def _cache_get(self, key: str):\n        if key in self.memory_cache:\n            vol = self.memory_cache[key]\n            self.memory_cache.move_to_end(key)\n            return vol\n        return None\n\n    def _sort_slices(self, ds_list: List[pydicom.dataset.FileDataset]) -> List[pydicom.dataset.FileDataset]:\n        try:\n            orient = np.array(ds_list[0].ImageOrientationPatient, dtype=np.float32)\n            row, col = orient[:3], orient[3:]\n            normal = np.cross(row, col)\n            return sorted(ds_list, key=lambda ds: float(np.dot(np.array(ds.ImagePositionPatient, dtype=np.float32), normal)))\n        except Exception:\n            return sorted(ds_list, key=lambda ds: getattr(ds, \"InstanceNumber\", 0))\n\n    def _get_spacing(self, ds_sorted: List[pydicom.dataset.FileDataset], has_multiframe: bool = False) -> Tuple[float, float, float]:\n        try:\n            dy, dx = map(float, ds_sorted[0].PixelSpacing)\n        except Exception:\n            dy, dx = 1.0, 1.0\n        if has_multiframe:\n            dz = float(getattr(ds_sorted[0], \"SpacingBetweenSlices\", getattr(ds_sorted[0], \"SliceThickness\", 1.0)))\n        else:\n            zs = [np.linalg.norm(np.array(ds_sorted[i].ImagePositionPatient, dtype=np.float32) - np.array(ds_sorted[i-1].ImagePositionPatient, dtype=np.float32)) for i in range(1, len(ds_sorted))]\n            dz = np.median([z for z in zs if z > 0]) if zs else float(getattr(ds_sorted[0], \"SliceThickness\", 1.0))\n        return (dz if dz > 0 else 1.0, dy if dy > 0 else 1.0, dx if dx > 0 else 1.0)\n\n    def _choose_base_shape(self, ds_list: List[pydicom.dataset.FileDataset]) -> Tuple[int, int]:\n        shapes = [(int(ds.Rows), int(ds.Columns)) for ds in ds_list if hasattr(ds, \"Rows\") and hasattr(ds, \"Columns\")]\n        if not shapes:\n            shapes = [(ds.pixel_array.shape[-2], ds.pixel_array.shape[-1]) for ds in ds_list if hasattr(ds, \"pixel_array\")]\n        vals, counts = np.unique(shapes, return_counts=True, axis=0)\n        return tuple(int(x) for x in vals[counts.argmax()]) if vals.size else (512, 512)\n\n    def _normalize_by_modality(self, volume: np.ndarray, modality_tag: str) -> np.ndarray:\n        volume = np.nan_to_num(volume, copy=False)\n        if modality_tag == \"CT\":\n            c, w = self.cta_window\n            lo, hi = c - w / 2.0, c + w / 2.0\n            return np.clip((volume - lo) / (hi - lo + 1e-6), 0, 1).astype(np.float32)\n        else:\n            mean = float(volume.mean())\n            std = float(volume.std() + 1e-6)\n            v = np.clip((volume - mean) / std, -self.mri_z_clip, self.mri_z_clip)\n            return ((v + self.mri_z_clip) / (2.0 * self.mri_z_clip)).astype(np.float32)\n\n    def load_dicom_series(self, series_path: str) -> np.ndarray:\n        series_id = os.path.basename(series_path)\n        m = self._cache_get(series_id)\n        if m is not None and m.shape == self.target_size:\n            return m\n\n        try:\n            dicoms = []\n            for root, _, files in os.walk(series_path):\n                for f in files:\n                    if f.endswith(\".dcm\"):\n                        try:\n                            ds = pydicom.dcmread(os.path.join(root, f), force=True)\n                            if hasattr(ds, \"PixelData\"):\n                                dicoms.append(ds)\n                        except Exception as e:\n                            print(f\"[DICOM read] {series_id}: {e}\")\n                            continue\n            if not dicoms:\n                raise ValueError(f\"No valid DICOM files in {series_path}\")\n\n            dicoms = self._sort_slices(dicoms)\n            has_multiframe = any(getattr(ds, \"NumberOfFrames\", 1) > 1 for ds in dicoms)\n            spacing = self._get_spacing(dicoms, has_multiframe)\n            base_h, base_w = self._choose_base_shape(dicoms)\n            modality_tag = getattr(dicoms[0], \"Modality\", \"\").upper()\n\n            vol_slices = []\n            for ds in dicoms:\n                arr = ds.pixel_array\n                if arr.ndim >= 3:\n                    h, w = arr.shape[-2], arr.shape[-1]\n                    n = int(np.prod(arr.shape[:-2]))\n                    arr = arr.reshape(n, h, w)\n                else:\n                    arr = arr[np.newaxis, ...]\n                for sl in arr:\n                    sl = sl.astype(np.float32)\n                    if getattr(ds, \"PhotometricInterpretation\", \"MONOCHROME2\") == \"MONOCHROME1\":\n                        sl = sl.max() - sl\n                    slope = float(getattr(ds, \"RescaleSlope\", 1.0))\n                    intercept = float(getattr(ds, \"RescaleIntercept\", 0.0))\n                    sl = sl * slope + intercept\n                    sl = _resize_slice(sl, base_h, base_w)\n                    vol_slices.append(sl)\n\n            if not vol_slices:\n                raise ValueError(f\"No valid slices in {series_id}\")\n            volume = np.stack(vol_slices, axis=0)\n\n            # Resample to target spacing\n            dz, dy, dx = spacing\n            z, y, x = volume.shape\n            newD = max(1, int(round(z * dz / self.target_spacing_mm)))\n            newH = max(1, int(round(y * dy / self.target_spacing_mm)))\n            newW = max(1, int(round(x * dx / self.target_spacing_mm)))\n            volume = _safe_zoom(volume, (newD / z, newH / y, newW / x), order=1)\n\n            # Resize to target grid\n            tz, ty, tx = self.target_size\n            z, y, x = volume.shape\n            volume = _safe_zoom(volume, (tz / z, ty / y, tx / x), order=1).astype(np.float32)\n\n            volume = self._normalize_by_modality(volume, modality_tag)\n            self._cache_put(series_id, volume)\n            return volume\n\n        except Exception as e:\n            print(f\"[Processor] Error processing {series_id}: {e}\")\n            vol = np.zeros(self.target_size, dtype=np.float32)\n            self._cache_put(series_id, vol)\n            return vol\n\n\ndef process_row(args):\n    row_dict, data_dir, label_cols, processor = args\n    series_id = row_dict[\"SeriesInstanceUID\"]\n    series_path = os.path.join(data_dir, series_id)\n\n    # Labels\n    labels = [row_dict.get(col, 0) for col in label_cols]\n\n    # Load DICOM\n    volume = processor.load_dicom_series(series_path)\n    return series_path, volume, np.array(labels, dtype=np.float32)\n\ndef preprocess_dataset():\n    os.makedirs(OUTPUT_DIR, exist_ok=True)\n    train_df = pd.read_csv(TRAIN_CSV)\n    # train_df = train_df.sample(n=100, random_state=42)\n\n    print(f\"Preprocessing {len(train_df)} series (with both positive & negative cases)\")\n\n    # --- Load and parse localizers ---\n    localizers_df = pd.read_csv(LOCALIZERS_CSV)\n\n    def parse_coordinates(coord_str):\n        try:\n            d = ast.literal_eval(coord_str)\n            return pd.Series({\n                \"center_x\": d.get(\"x\", np.nan),\n                \"center_y\": d.get(\"y\", np.nan),\n                \"center_z\": d.get(\"z\", np.nan) if \"z\" in d else np.nan\n            })\n        except Exception:\n            return pd.Series({\"center_x\": np.nan, \"center_y\": np.nan, \"center_z\": np.nan})\n\n    coord_df = localizers_df[\"coordinates\"].apply(parse_coordinates)\n    localizers_df = pd.concat([localizers_df.drop(columns=[\"coordinates\"]), coord_df], axis=1)\n\n    # Average coordinates per series\n    localizers_df = localizers_df.groupby(\"SeriesInstanceUID\")[[\"center_x\", \"center_y\", \"center_z\"]].mean().reset_index()\n\n    # Merge with training data (keep all rows)\n    train_df = train_df.merge(localizers_df, on=\"SeriesInstanceUID\", how=\"left\")\n\n    # Fill missing coordinates with global mean\n    for col in [\"center_x\", \"center_y\", \"center_z\"]:\n        train_df[col] = train_df[col].fillna(0)\n\n    print(\"Label distribution (Aneurysm Present):\")\n    print(train_df[\"Aneurysm Present\"].value_counts())\n\n    # --- Prepare multiprocessing args ---\n    processor = DICOMProcessor(\n        target_size=TARGET_SIZE,\n        target_spacing_mm=TARGET_SPACING_MM,\n        cta_window=CTA_WINDOW,\n        mri_z_clip=MRI_Z_CLIP\n    )\n\n    args = [(row.to_dict(), DATA_DIR, LABEL_COLS, processor) \n        for _, row in train_df.iterrows()]\n\n\n    series_data = []\n    with mp.Pool(processes=min(mp.cpu_count(), 4)) as pool:\n        results = list(tqdm(pool.imap(process_row, args),\n                            total=len(train_df), desc=\"Preprocessing\"))\n\n    for series_path, volume, labels in results:\n        if volume is not None and not np.all(volume == 0):\n            series_id = os.path.basename(series_path)\n            np.save(os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"), volume)\n            series_data.append({\n                \"image\": os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"),\n                \"labels\": labels.tolist()\n            })\n\n    pd.DataFrame(series_data).to_csv(os.path.join(OUTPUT_DIR, \"series_metadata.csv\"), index=False)\n    print(\"Preprocessing done!\")\n\n# Dataset\nclass AneurysmDataset(Dataset):\n    def __init__(self, data_file: str, transform=None):\n        self.data = pd.read_csv(data_file)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        image = np.load(row[\"image\"]).astype(np.float32)\n        # Ensure image is 3D (D, H, W) and add channel dimension\n        if image.ndim == 3:\n            image = np.expand_dims(image, axis=0)  # Shape: (1, D, H, W)\n        elif image.ndim == 4 and image.shape[0] == 1:\n            image = image.squeeze(0)  # Ensure no extra batch dimension from transform\n        # Parse labels string using ast.literal_eval\n        labels = np.array(ast.literal_eval(row[\"labels\"]), dtype=np.float32) if isinstance(row[\"labels\"], str) else np.array(row[\"labels\"], dtype=np.float32)\n        \n        # Light augmentation with torchio\n        if self.transform:\n            subject = tio.Subject(image=tio.ScalarImage(tensor=image))\n            transformed = self.transform(subject)\n            image = transformed[\"image\"].numpy()\n\n        # Ensure 5D tensor for model (batch is added by DataLoader)\n        image_tensor = torch.from_numpy(image).float()  # Shape: (1, D, H, W)\n        labels_tensor = torch.from_numpy(labels)\n        return image_tensor, labels_tensor\n\ntrain_transforms = tio.Compose([\n    tio.RandomFlip(axes=(0,), p=0.5),\n    tio.RandomAffine(degrees=10, p=0.5),\n    tio.ToCanonical()  # Ensure canonical orientation\n])\nval_transforms = tio.Compose([\n    tio.ToCanonical()  # Ensure canonical orientation\n])\n\n# Model\nclass EfficientNet3D(nn.Module):\n    def __init__(self, num_classes=14):\n        super().__init__()\n        self.backbone = monai.networks.nets.EfficientNetBN(\"efficientnet-b0\", spatial_dims=3, in_channels=1, pretrained=False)\n        self.pool = nn.AdaptiveAvgPool3d(1)  # Output: (batch_size, channels, 1, 1, 1)\n        self.fc = nn.Linear(self.backbone._fc.in_features, num_classes)\n        self.backbone._fc = nn.Identity()\n\n    def forward(self, x):\n        # Ensure input is 5D (batch_size, channels, depth, height, width)\n        if x.dim() != 5:\n            raise ValueError(f\"Expected 5D input tensor (batch_size, channels, depth, height, width), got {x.shape}\")\n        \n        x = self.backbone(x)\n        # Check and adjust backbone output dimensions\n        if x.dim() < 3:  # If backbone output is 2D, reshape to add dummy spatial dimensions\n            x = x.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1)  # Add 3 dummy dimensions\n        x = self.pool(x)  # Ensure pooling works on at least 5D tensor\n        x = x.view(x.size(0), -1)  # Flatten to (batch_size, features)\n        x = self.fc(x)  # Return raw logits\n        return x  # Removed torch.sigmoid()\n\n# Training Loop\ndef train_one_epoch(model, loader, optimizer, scaler, criterion):\n    model.train()\n    losses = []\n    for batch in tqdm(loader, desc=\"Training\"):\n        images, labels = batch[0].to(DEVICE), batch[1].to(DEVICE)\n        optimizer.zero_grad()\n        with torch.cuda.amp.autocast():\n            # Debug: Print input shape to verify\n            # print(f\"Input shape to model: {images.shape}\")\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n        losses.append(loss.item())\n    return np.mean(losses)\n\ndef validate(model, loader, criterion):\n    model.eval()\n    preds, truths = [], []\n    losses = []\n    with torch.no_grad():\n        for batch in tqdm(loader, desc=\"Validating\"):\n            images, labels = batch[0].to(DEVICE), batch[1].to(DEVICE)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            losses.append(loss.item())\n            preds.append(outputs.cpu().numpy())\n            truths.append(labels.cpu().numpy())\n    preds = np.concatenate(preds)\n    truths = np.concatenate(truths)\n    \n    # Calculate AUCs, handling single-class cases\n    aucs = []\n    for i in range(len(LABEL_COLS)):\n        y_true = truths[:, i]\n        y_pred = preds[:, i]\n        if len(np.unique(y_true)) == 1:  # Check if only one class is present\n            print(f\"Warning: Only one class present in column {LABEL_COLS[i]}. AUC set to 0.5.\")\n            aucs.append(0.5)  # Default to 0.5 (no discrimination)\n        else:\n            aucs.append(roc_auc_score(y_true, y_pred))\n    \n    final_score = (aucs[-1] + np.mean(aucs[:-1])) / 2\n    return np.mean(losses), final_score, aucs\n\n# Main\ndef main():\n    # Preprocess\n    preprocess_dataset()\n\n    # Data\n    metadata_file = os.path.join(OUTPUT_DIR, \"series_metadata.csv\")\n    metadata = pd.read_csv(metadata_file)\n    metadata = metadata.sample(frac=1, random_state=42).reset_index(drop=True)\n    train_idx = int(0.8 * len(metadata))\n    train_data = metadata[:train_idx]\n    val_data = metadata[train_idx:]\n\n\n    train_dataset = CacheDataset(AneurysmDataset(metadata_file, train_transforms), cache_rate=0.1, num_workers=4)\n    val_dataset = CacheDataset(AneurysmDataset(metadata_file, val_transforms), cache_rate=0.1, num_workers=4)\n    train_loader = MonaiDataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4, prefetch_factor=2)\n    val_loader = MonaiDataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, prefetch_factor=2)\n\n    # Model\n    model = EfficientNet3D(num_classes=len(LABEL_COLS)).to(DEVICE)\n    criterion = nn.BCEWithLogitsLoss()  # Changed to BCEWithLogitsLoss\n    optimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n    scaler = torch.cuda.amp.GradScaler()\n\n    # Early stopping parameters\n    patience = 5\n    counter = 0\n    best_score = 0\n    best_epoch = 0\n\n    for epoch in range(NUM_EPOCHS):\n        train_loss = train_one_epoch(model, train_loader, optimizer, scaler, criterion)\n        val_loss, val_score, aucs = validate(model, val_loader, criterion)\n\n        print(f\"Epoch {epoch+1}/{NUM_EPOCHS}, \"\n              f\"Train Loss: {train_loss:.4f}, \"\n              f\"Val Loss: {val_loss:.4f}, \"\n              f\"Val AUC: {val_score:.4f}\")\n\n        # Nếu tốt hơn -> lưu lại\n        if val_score > best_score:\n            best_score = val_score\n            best_epoch = epoch\n            torch.save(model.state_dict(), \"best_model.pth\")\n            counter = 0  # reset bộ đếm\n        else:\n            counter += 1\n            print(f\"No improvement. EarlyStopping counter: {counter}/{patience}\")\n\n        # Nếu chờ đủ patience mà không cải thiện -> dừng\n        if counter >= patience:\n            print(f\"Early stopping at epoch {epoch+1}. Best epoch was {best_epoch+1} with score {best_score:.4f}\")\n            break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-07T18:11:40.885821Z","iopub.execute_input":"2025-09-07T18:11:40.886126Z","iopub.status.idle":"2025-09-07T18:11:40.939671Z","shell.execute_reply.started":"2025-09-07T18:11:40.886083Z","shell.execute_reply":"2025-09-07T18:11:40.939003Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# sid = \"1.2.826.0.1.3680043.8.498.11208788596258922886794998326857227331\"\n# orig = train_df.loc[train_df[\"SeriesInstanceUID\"] == sid, LABEL_COLS].values\n# new = df.loc[df[\"image\"].str.contains(sid), \"labels\"].values\n# print(\"Original:\", orig)\n# print(\"Preprocessed:\", new)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-07T18:11:40.941297Z","iopub.execute_input":"2025-09-07T18:11:40.941495Z","iopub.status.idle":"2025-09-07T18:11:40.958413Z","shell.execute_reply.started":"2025-09-07T18:11:40.941479Z","shell.execute_reply":"2025-09-07T18:11:40.957756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    main()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-07T18:11:40.959201Z","iopub.execute_input":"2025-09-07T18:11:40.959469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_file = os.path.join(OUTPUT_DIR, \"series_metadata.csv\")\nmetadata = pd.read_csv(metadata_file)\nmetadata.head(5)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(metadata['labels'].iloc[0])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(metadata['image'].iloc[0])","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## For segmentation","metadata":{}},{"cell_type":"code","source":"\n# # Preprocessing Function\n# def preprocess_dataset():\n#     os.makedirs(OUTPUT_DIR, exist_ok=True)\n#     train_df = pd.read_csv(TRAIN_CSV)\n\n#     # Keep only aneurysm cases (for now, can expand to include negatives later)\n#     train_df = train_df[train_df[\"Aneurysm Present\"] == 1].copy()\n#     print(f\"Preprocessing {len(train_df)} series with aneurysm\")\n\n#     # --- Load and parse localizers ---\n#     localizers_df = pd.read_csv(LOCALIZERS_CSV)\n\n#     def parse_coordinates(coord_str):\n#         try:\n#             d = ast.literal_eval(coord_str)\n#             return pd.Series({\n#                 \"center_x\": d.get(\"x\", np.nan),\n#                 \"center_y\": d.get(\"y\", np.nan),\n#                 \"center_z\": d.get(\"z\", np.nan) if \"z\" in d else np.nan\n#             })\n#         except Exception:\n#             return pd.Series({\"center_x\": np.nan, \"center_y\": np.nan, \"center_z\": np.nan})\n\n#     coord_df = localizers_df[\"coordinates\"].apply(parse_coordinates)\n#     localizers_df = pd.concat([localizers_df.drop(columns=[\"coordinates\"]), coord_df], axis=1)\n\n#     # Average coordinates per series\n#     localizers_df = localizers_df.groupby(\"SeriesInstanceUID\")[[\"center_x\", \"center_y\", \"center_z\"]].mean().reset_index()\n\n#     # Merge with training data\n#     train_df = train_df.merge(localizers_df, on=\"SeriesInstanceUID\", how=\"left\")\n#     train_df = train_df[:100]\n\n#     # Fill NaNs with global mean\n#     for col in [\"center_x\", \"center_y\", \"center_z\"]:\n#         train_df[col] = train_df[col].fillna(train_df[col].mean())\n\n#     # --- Prepare multiprocessing args ---\n#     processor = DICOMProcessor(\n#         target_size=TARGET_SIZE,\n#         target_spacing_mm=TARGET_SPACING_MM,\n#         cta_window=CTA_WINDOW,\n#         mri_z_clip=MRI_Z_CLIP\n#     )\n\n#     args = [(row._asdict(), DATA_DIR, LABEL_COLS, processor) \n#             for row in train_df.itertuples(index=False)]\n\n#     series_data = []\n#     with mp.Pool(processes=min(mp.cpu_count(), 4)) as pool:\n#         results = list(tqdm(pool.imap(process_row, args),\n#                             total=len(train_df), desc=\"Preprocessing\"))\n\n#     for series_path, volume, labels in results:\n#         if volume is not None and not np.all(volume == 0):\n#             series_id = os.path.basename(series_path)\n#             np.save(os.path.join(OUTPUT_DIR, f\"{series_id}.npy\"), volume)\n            \n#             # Load and process segmentation mask\n#             seg_path = os.path.join(SEGMENTATIONS_DIR, f\"{series_id}.nii.gz\")  # Adjust extension if needed\n#             try:\n#                 seg_mask = sitk.ReadImage(seg_path)\n#                 seg_array = sitk.GetArrayFromImage(seg_mask)\n#                 # Resize segmentation mask to match volume shape\n#                 seg_array = _safe_zoom(seg_array, (TARGET_SIZE[0] / seg_array.shape[0],\n#                                                  TARGET_SIZE[1] / seg_array.shape[1],\n#                                                  TARGET_SIZE[2] / seg_array.shape[2]), order=0)\n#                 seg_array = (seg_array > 0).astype(np.float32)  # Binary mask\n#             except Exception as e:\n#                 print(f\"[Segmentation] Error loading {series_id}: {e}\")\n#                 seg_array = np.zeros(TARGET_SIZE, dtype=np.float32)\n\n#             # Crop volume using segmentation mask (simple bounding box approach)\n#             if np.any(seg_array):\n#                 coords = np.where(seg_array)\n#                 z_min, z_max = max(0, coords[0].min() - 10), min(TARGET_SIZE[0], coords[0].max() + 10)\n#                 y_min, y_max = max(0, coords[1].min() - 10), min(TARGET_SIZE[1], coords[1].max() + 10)\n#                 x_min, x_max = max(0, coords[2].min() - 10), min(TARGET_SIZE[2], coords[2].max() + 10)\n#                 volume_cropped = volume[z_min:z_max, y_min:y_max, x_min:x_max]\n#                 # Resize back to TARGET_SIZE if cropped region is smaller\n#                 if volume_cropped.shape != TARGET_SIZE:\n#                     volume_cropped = _safe_zoom(volume_cropped, (TARGET_SIZE[0] / volume_cropped.shape[0],\n#                                                                TARGET_SIZE[1] / volume_cropped.shape[1],\n#                                                                TARGET_SIZE[2] / volume_cropped.shape[2]))\n#             else:\n#                 volume_cropped = volume\n\n#             np.save(os.path.join(OUTPUT_DIR, f\"{series_id}_cropped.npy\"), volume_cropped)\n#             series_data.append({\n#                 \"image\": os.path.join(OUTPUT_DIR, f\"{series_id}_cropped.npy\"),\n#                 \"labels\": labels.tolist()\n#             })\n\n#     pd.DataFrame(series_data).to_csv(os.path.join(OUTPUT_DIR, \"series_metadata.csv\"), index=False)\n#     print(\"Preprocessing done!\")\n\n# # Dataset\n# class AneurysmDataset(Dataset):\n#     def __init__(self, data_file: str, transform=None):\n#         self.data = pd.read_csv(data_file)\n#         self.transform = transform\n\n#     def __len__(self):\n#         return len(self.data)\n\n#     def __getitem__(self, idx):\n#         row = self.data.iloc[idx]\n#         image = np.load(row[\"image\"]).astype(np.float32)\n#         # Ensure image is 3D (D, H, W) and add channel dimension\n#         if image.ndim == 3:\n#             image = np.expand_dims(image, axis=0)  # Shape: (1, D, H, W)\n#         elif image.ndim == 4 and image.shape[0] == 1:\n#             image = image.squeeze(0)  # Ensure no extra batch dimension from transform\n        \n#         # Parse labels string using ast.literal_eval\n#         labels = np.array(ast.literal_eval(row[\"labels\"]), dtype=np.float32) if isinstance(row[\"labels\"], str) else np.array(row[\"labels\"], dtype=np.float32)\n        \n#         # Light augmentation with torchio\n#         if self.transform:\n#             subject = tio.Subject(image=tio.ScalarImage(tensor=image))\n#             transformed = self.transform(subject)\n#             image = transformed[\"image\"].numpy()\n\n#         # Ensure 5D tensor for model (batch is added by DataLoader)\n#         image_tensor = torch.from_numpy(image).float()  # Shape: (1, D, H, W)\n#         labels_tensor = torch.from_numpy(labels)\n#         return image_tensor, labels_tensor","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}