{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nRSNA Intracranial Aneurysm Detection - Full Production Pipeline\nDetects presence and location of intracranial aneurysms in multimodal imaging data\n\"\"\"\n\nimport os\nimport gc\nimport sys\nimport json\nimport shutil\nimport warnings\nimport psutil\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom typing import List, Optional, Tuple, Dict, Any\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib\nmatplotlib.use('Agg')\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.auto import tqdm\n\nimport pydicom\nimport cv2\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import (\n    roc_auc_score, roc_curve, precision_recall_curve,\n    confusion_matrix, classification_report, f1_score\n)\nfrom sklearn.decomposition import PCA\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers, models, callbacks, optimizers\nfrom tensorflow.keras.applications import EfficientNetB0, ResNet50\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nwarnings.filterwarnings('ignore')\n\n# Configure TensorFlow for memory efficiency\ngpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n            tf.config.experimental.set_virtual_device_configuration(\n                gpu,\n                [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048)]\n            )\n    except RuntimeError as e:\n        print(f\"GPU configuration error: {e}\")\n\n# Global Constants\nID_COL = 'SeriesInstanceUID'\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\nDICOM_TAG_ALLOWLIST = [\n    'BitsAllocated', 'BitsStored', 'Columns', 'FrameOfReferenceUID',\n    'HighBit', 'ImageOrientationPatient', 'ImagePositionPatient',\n    'InstanceNumber', 'Modality', 'PatientID', 'PhotometricInterpretation',\n    'PixelRepresentation', 'PixelSpacing', 'PlanarConfiguration',\n    'RescaleIntercept', 'RescaleSlope', 'RescaleType', 'Rows',\n    'SOPClassUID', 'SOPInstanceUID', 'SamplesPerPixel',\n    'SliceThickness', 'SpacingBetweenSlices', 'StudyInstanceUID',\n    'TransferSyntaxUID',\n]\n\nBASE_PATH = '/kaggle/input/rsna-intracranial-aneurysm-detection'\nWORKING_PATH = '/kaggle/working'\nIMG_SIZE = 224\nBATCH_SIZE = 8\nEPOCHS = 15\nMAX_SERIES_SAMPLE = 200\nMEMORY_LIMIT_GB = 14\n\n\nclass MemoryMonitor:\n    \"\"\"Real-time memory monitoring and safety checks\"\"\"\n    \n    def __init__(self, limit_gb: float = MEMORY_LIMIT_GB):\n        self.limit_bytes = limit_gb * 1024**3\n        self.process = psutil.Process()\n    \n    def get_usage(self) -> Dict[str, float]:\n        mem = self.process.memory_info()\n        return {\n            'used_gb': mem.rss / 1024**3,\n            'percent': (mem.rss / self.limit_bytes) * 100\n        }\n    \n    def is_safe(self, threshold: float = 0.85) -> bool:\n        usage = self.get_usage()\n        return usage['percent'] < (threshold * 100)\n    \n    def log_usage(self, stage: str = \"\"):\n        usage = self.get_usage()\n        print(f\"[{stage}] Memory: {usage['used_gb']:.2f}GB ({usage['percent']:.1f}%)\")\n    \n    def force_cleanup(self):\n        \"\"\"Aggressive memory cleanup\"\"\"\n        gc.collect()\n        if gpus:\n            try:\n                tf.keras.backend.clear_session()\n            except:\n                pass\n        gc.collect()\n\n\ndef weighted_multilabel_auc(\n    y_true: np.ndarray,\n    y_scores: np.ndarray,\n    class_weights: Optional[List[float]] = None,\n) -> float:\n    \"\"\"Compute weighted AUC for multilabel classification\"\"\"\n    y_true = np.asarray(y_true)\n    y_scores = np.asarray(y_scores)\n    n_classes = y_true.shape[1]\n    \n    try:\n        individual_aucs = roc_auc_score(y_true, y_scores, average=None)\n    except ValueError:\n        return 0.0\n    \n    if class_weights is None:\n        weights_array = np.ones(n_classes)\n    else:\n        weights_array = np.asarray(class_weights)\n    \n    weights_array = weights_array / np.sum(weights_array)\n    return float(np.sum(individual_aucs * weights_array))\n\n\ndef calculate_competition_metric(y_true: np.ndarray, y_pred: np.ndarray) -> float:\n    \"\"\"Calculate the competition metric with special weighting\"\"\"\n    weights = [1.0] * 13 + [13.0]  # Last column (Aneurysm Present) has weight 13\n    return weighted_multilabel_auc(y_true, y_pred, class_weights=weights)\n\n\nclass DataLoader:\n    \"\"\"Memory-efficient data loading with chunking\"\"\"\n    \n    def __init__(self, base_path: str, monitor: MemoryMonitor):\n        self.base_path = Path(base_path)\n        self.monitor = monitor\n        self.train_df = None\n        self.localizers_df = None\n        \n    def load_metadata(self) -> Tuple[pd.DataFrame, pd.DataFrame]:\n        \"\"\"Load training metadata with memory efficiency\"\"\"\n        print(\"\\n=== Loading Metadata ===\")\n        self.monitor.log_usage(\"Before metadata load\")\n        \n        train_path = self.base_path / 'train.csv'\n        localizers_path = self.base_path / 'train_localizers.csv'\n        \n        if train_path.exists():\n            self.train_df = pd.read_csv(train_path)\n            print(f\"Loaded {len(self.train_df)} training records\")\n        else:\n            print(\"WARNING: train.csv not found, creating dummy data\")\n            self.train_df = self._create_dummy_data()\n        \n        if localizers_path.exists():\n            self.localizers_df = pd.read_csv(localizers_path)\n            print(f\"Loaded {len(self.localizers_df)} localizer records\")\n        else:\n            print(\"WARNING: train_localizers.csv not found\")\n            self.localizers_df = pd.DataFrame()\n        \n        self.monitor.log_usage(\"After metadata load\")\n        return self.train_df, self.localizers_df\n    \n    def _create_dummy_data(self) -> pd.DataFrame:\n        \"\"\"Create synthetic data for testing\"\"\"\n        n_samples = 500\n        data = {\n            'SeriesInstanceUID': [f'series_{i}' for i in range(n_samples)],\n            'Modality': np.random.choice(['CTA', 'MRA', 'MRI'], n_samples),\n            'PatientAge': np.random.randint(20, 80, n_samples),\n            'PatientSex': np.random.choice(['M', 'F'], n_samples),\n        }\n        \n        for col in LABEL_COLS:\n            if col == 'Aneurysm Present':\n                data[col] = np.random.choice([0, 1], n_samples, p=[0.7, 0.3])\n            else:\n                data[col] = np.random.choice([0, 1], n_samples, p=[0.9, 0.1])\n        \n        return pd.DataFrame(data)\n    \n    def load_dicom_series(self, series_id: str, max_slices: int = 64) -> Optional[np.ndarray]:\n        \"\"\"Load DICOM series with memory constraints\"\"\"\n        series_path = self.base_path / 'series' / series_id\n        \n        if not series_path.exists():\n            return None\n        \n        dcm_files = sorted(list(series_path.glob('*.dcm')))[:max_slices]\n        \n        if not dcm_files:\n            return None\n        \n        images = []\n        for dcm_file in dcm_files:\n            try:\n                ds = pydicom.dcmread(str(dcm_file), force=True)\n                img = ds.pixel_array\n                \n                # Normalize\n                img = img.astype(np.float32)\n                if img.max() > img.min():\n                    img = (img - img.min()) / (img.max() - img.min())\n                \n                # Resize\n                img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n                images.append(img)\n            except Exception as e:\n                continue\n        \n        if not images:\n            return None\n        \n        # Stack and take middle slices\n        volume = np.array(images)\n        return volume\n    \n    def extract_features_from_series(self, series_id: str) -> Dict[str, Any]:\n        \"\"\"Extract features from a DICOM series\"\"\"\n        features = {\n            'series_id': series_id,\n            'num_slices': 0,\n            'mean_intensity': 0.0,\n            'std_intensity': 0.0,\n            'slice_thickness': 0.0,\n            'pixel_spacing_x': 0.0,\n            'pixel_spacing_y': 0.0,\n        }\n        \n        volume = self.load_dicom_series(series_id)\n        \n        if volume is not None and len(volume) > 0:\n            features['num_slices'] = len(volume)\n            features['mean_intensity'] = float(np.mean(volume))\n            features['std_intensity'] = float(np.std(volume))\n        \n        # Try to get DICOM metadata\n        series_path = self.base_path / 'series' / series_id\n        if series_path.exists():\n            dcm_files = list(series_path.glob('*.dcm'))\n            if dcm_files:\n                try:\n                    ds = pydicom.dcmread(str(dcm_files[0]), force=True)\n                    if hasattr(ds, 'SliceThickness'):\n                        features['slice_thickness'] = float(ds.SliceThickness)\n                    if hasattr(ds, 'PixelSpacing'):\n                        features['pixel_spacing_x'] = float(ds.PixelSpacing[0])\n                        features['pixel_spacing_y'] = float(ds.PixelSpacing[1])\n                except:\n                    pass\n        \n        return features\n\n\nclass FeatureEngineering:\n    \"\"\"Comprehensive feature engineering pipeline\"\"\"\n    \n    def __init__(self, monitor: MemoryMonitor):\n        self.monitor = monitor\n        self.scaler = StandardScaler()\n        self.le_modality = LabelEncoder()\n        self.le_sex = LabelEncoder()\n        \n    def engineer_features(self, df: pd.DataFrame, is_train: bool = True) -> pd.DataFrame:\n        \"\"\"Create advanced features from metadata\"\"\"\n        print(\"\\n=== Feature Engineering ===\")\n        self.monitor.log_usage(\"Before feature engineering\")\n        \n        df = df.copy()\n        \n        # Handle missing values\n        if 'PatientAge' in df.columns:\n            df['PatientAge'] = df['PatientAge'].fillna(df['PatientAge'].median())\n        \n        # Encode categorical variables\n        if 'Modality' in df.columns:\n            if is_train:\n                df['Modality_encoded'] = self.le_modality.fit_transform(df['Modality'].fillna('Unknown'))\n            else:\n                df['Modality_encoded'] = self.le_modality.transform(df['Modality'].fillna('Unknown'))\n            \n            # One-hot encode modality\n            modality_dummies = pd.get_dummies(df['Modality'], prefix='modality')\n            df = pd.concat([df, modality_dummies], axis=1)\n        \n        if 'PatientSex' in df.columns:\n            if is_train:\n                df['PatientSex_encoded'] = self.le_sex.fit_transform(df['PatientSex'].fillna('U'))\n            else:\n                df['PatientSex_encoded'] = self.le_sex.transform(df['PatientSex'].fillna('U'))\n        \n        # Age-based features\n        if 'PatientAge' in df.columns:\n            df['age_group'] = pd.cut(df['PatientAge'], bins=[0, 30, 50, 70, 100], labels=[0, 1, 2, 3])\n            df['age_group'] = df['age_group'].astype(int)\n            df['age_squared'] = df['PatientAge'] ** 2\n            df['age_log'] = np.log1p(df['PatientAge'])\n        \n        # Location-based aggregations\n        location_cols = [col for col in LABEL_COLS if col != 'Aneurysm Present']\n        if all(col in df.columns for col in location_cols):\n            df['total_aneurysm_locations'] = df[location_cols].sum(axis=1)\n            df['left_aneurysms'] = df[[col for col in location_cols if 'Left' in col]].sum(axis=1)\n            df['right_aneurysms'] = df[[col for col in location_cols if 'Right' in col]].sum(axis=1)\n            df['anterior_aneurysms'] = df[[col for col in location_cols if 'Anterior' in col]].sum(axis=1)\n            df['posterior_aneurysms'] = df[[col for col in location_cols if 'Posterior' in col]].sum(axis=1)\n        \n        self.monitor.log_usage(\"After feature engineering\")\n        gc.collect()\n        \n        return df\n    \n    def scale_features(self, X_train: np.ndarray, X_val: np.ndarray = None, \n                      X_test: np.ndarray = None) -> Tuple:\n        \"\"\"Scale numerical features\"\"\"\n        X_train_scaled = self.scaler.fit_transform(X_train)\n        \n        results = [X_train_scaled]\n        \n        if X_val is not None:\n            results.append(self.scaler.transform(X_val))\n        \n        if X_test is not None:\n            results.append(self.scaler.transform(X_test))\n        \n        return tuple(results) if len(results) > 1 else results[0]\n\n\nclass ImageFeatureExtractor:\n    \"\"\"Extract features from medical images using CNN\"\"\"\n    \n    def __init__(self, monitor: MemoryMonitor):\n        self.monitor = monitor\n        self.model = None\n        \n    def build_feature_extractor(self):\n        \"\"\"Build a lightweight CNN for feature extraction\"\"\"\n        base_model = EfficientNetB0(\n            include_top=False,\n            weights='imagenet',\n            input_shape=(IMG_SIZE, IMG_SIZE, 3),\n            pooling='avg'\n        )\n        base_model.trainable = False\n        \n        self.model = base_model\n        return self.model\n    \n    def extract_features_batch(self, images: np.ndarray) -> np.ndarray:\n        \"\"\"Extract features from batch of images\"\"\"\n        if self.model is None:\n            self.build_feature_extractor()\n        \n        # Convert grayscale to RGB\n        if len(images.shape) == 3:\n            images = np.stack([images] * 3, axis=-1)\n        elif images.shape[-1] == 1:\n            images = np.repeat(images, 3, axis=-1)\n        \n        features = self.model.predict(images, batch_size=BATCH_SIZE, verbose=0)\n        return features\n\n\nclass AneurysmDetectionModel:\n    \"\"\"Main model for aneurysm detection with ensemble approach\"\"\"\n    \n    def __init__(self, monitor: MemoryMonitor):\n        self.monitor = monitor\n        self.cnn_model = None\n        self.rf_model = None\n        self.gb_model = None\n        self.history = None\n        self.best_weights = None\n        self.best_val_metric = 0.0\n        \n    def build_cnn_model(self, input_shape: Tuple[int, int, int], num_outputs: int = 14):\n        \"\"\"Build CNN model with attention mechanism\"\"\"\n        inputs = layers.Input(shape=input_shape)\n        \n        # Efficient CNN backbone\n        x = layers.Conv2D(32, 3, padding='same', activation='relu')(inputs)\n        x = layers.BatchNormalization()(x)\n        x = layers.MaxPooling2D(2)(x)\n        \n        x = layers.Conv2D(64, 3, padding='same', activation='relu')(x)\n        x = layers.BatchNormalization()(x)\n        x = layers.MaxPooling2D(2)(x)\n        \n        x = layers.Conv2D(128, 3, padding='same', activation='relu')(x)\n        x = layers.BatchNormalization()(x)\n        \n        # Attention mechanism\n        attention = layers.Conv2D(1, 1, activation='sigmoid')(x)\n        x = layers.Multiply()([x, attention])\n        \n        x = layers.GlobalAveragePooling2D()(x)\n        x = layers.Dropout(0.4)(x)\n        x = layers.Dense(128, activation='relu')(x)\n        x = layers.Dropout(0.3)(x)\n        x = layers.Dense(64, activation='relu')(x)\n        \n        outputs = layers.Dense(num_outputs, activation='sigmoid')(x)\n        \n        model = models.Model(inputs=inputs, outputs=outputs)\n        \n        model.compile(\n            optimizer=optimizers.Adam(learning_rate=0.001),\n            loss='binary_crossentropy',\n            metrics=['accuracy', tf.keras.metrics.AUC(name='auc')]\n        )\n        \n        self.cnn_model = model\n        return model\n    \n    def train_cnn(self, X_train: np.ndarray, y_train: np.ndarray,\n                  X_val: np.ndarray, y_val: np.ndarray,\n                  class_weights: Dict = None):\n        \"\"\"Train CNN with early stopping and checkpointing\"\"\"\n        print(\"\\n=== Training CNN Model ===\")\n        self.monitor.log_usage(\"Before CNN training\")\n        \n        if len(X_train.shape) == 3:\n            X_train = np.expand_dims(X_train, -1)\n            X_val = np.expand_dims(X_val, -1)\n        \n        if self.cnn_model is None:\n            self.build_cnn_model(X_train.shape[1:], y_train.shape[1])\n        \n        # Callbacks\n        checkpoint_path = os.path.join(WORKING_PATH, 'best_cnn_model.h5')\n        \n        callbacks_list = [\n            callbacks.EarlyStopping(\n                monitor='val_auc',\n                patience=5,\n                mode='max',\n                restore_best_weights=True,\n                verbose=1\n            ),\n            callbacks.ModelCheckpoint(\n                checkpoint_path,\n                monitor='val_auc',\n                mode='max',\n                save_best_only=True,\n                verbose=1\n            ),\n            callbacks.ReduceLROnPlateau(\n                monitor='val_auc',\n                factor=0.5,\n                patience=3,\n                mode='max',\n                verbose=1\n            )\n        ]\n        \n        # Data augmentation\n        datagen = ImageDataGenerator(\n            rotation_range=10,\n            width_shift_range=0.1,\n            height_shift_range=0.1,\n            zoom_range=0.1,\n            horizontal_flip=True,\n            fill_mode='nearest'\n        )\n        \n        self.history = self.cnn_model.fit(\n            datagen.flow(X_train, y_train, batch_size=BATCH_SIZE),\n            validation_data=(X_val, y_val),\n            epochs=EPOCHS,\n            callbacks=callbacks_list,\n            class_weight=class_weights,\n            verbose=1\n        )\n        \n        self.monitor.log_usage(\"After CNN training\")\n        self.monitor.force_cleanup()\n        \n        return self.history\n    \n    def train_ensemble(self, X_train: np.ndarray, y_train: np.ndarray,\n                      X_val: np.ndarray, y_val: np.ndarray):\n        \"\"\"Train ensemble of classical ML models\"\"\"\n        print(\"\\n=== Training Ensemble Models ===\")\n        self.monitor.log_usage(\"Before ensemble training\")\n        \n        # Random Forest\n        print(\"Training Random Forest...\")\n        self.rf_model = RandomForestClassifier(\n            n_estimators=100,\n            max_depth=15,\n            min_samples_split=10,\n            min_samples_leaf=5,\n            n_jobs=-1,\n            random_state=42\n        )\n        self.rf_model.fit(X_train, y_train)\n        \n        rf_val_pred = self.rf_model.predict_proba(X_val)\n        rf_val_pred = np.array([pred[:, 1] for pred in rf_val_pred]).T\n        rf_score = calculate_competition_metric(y_val, rf_val_pred)\n        print(f\"Random Forest validation score: {rf_score:.4f}\")\n        \n        gc.collect()\n        \n        # Gradient Boosting\n        print(\"Training Gradient Boosting...\")\n        self.gb_model = GradientBoostingClassifier(\n            n_estimators=100,\n            max_depth=5,\n            learning_rate=0.1,\n            subsample=0.8,\n            random_state=42\n        )\n        \n        # Train on main target first\n        self.gb_model.fit(X_train, y_train[:, -1])\n        \n        gb_val_pred = self.gb_model.predict_proba(X_val)[:, 1]\n        \n        self.monitor.log_usage(\"After ensemble training\")\n        gc.collect()\n        \n        return rf_score\n    \n    def predict_ensemble(self, X: np.ndarray, X_img: np.ndarray = None) -> np.ndarray:\n        \"\"\"Generate ensemble predictions\"\"\"\n        predictions = []\n        weights = []\n        \n        # CNN predictions\n        if self.cnn_model is not None and X_img is not None:\n            if len(X_img.shape) == 3:\n                X_img = np.expand_dims(X_img, -1)\n            cnn_pred = self.cnn_model.predict(X_img, batch_size=BATCH_SIZE, verbose=0)\n            predictions.append(cnn_pred)\n            weights.append(0.5)\n        \n        # Random Forest predictions\n        if self.rf_model is not None:\n            rf_pred = self.rf_model.predict_proba(X)\n            rf_pred = np.array([pred[:, 1] for pred in rf_pred]).T\n            predictions.append(rf_pred)\n            weights.append(0.3)\n        \n        # Gradient Boosting predictions (for main target)\n        if self.gb_model is not None:\n            gb_pred = self.gb_model.predict_proba(X)[:, 1]\n            gb_pred_full = np.zeros((len(gb_pred), 14))\n            gb_pred_full[:, -1] = gb_pred\n            predictions.append(gb_pred_full)\n            weights.append(0.2)\n        \n        if not predictions:\n            return np.ones((len(X), 14)) * 0.5\n        \n        # Weighted average\n        weights = np.array(weights) / sum(weights)\n        ensemble_pred = sum(w * p for w, p in zip(weights, predictions))\n        \n        return ensemble_pred\n\n\nclass ModelEvaluator:\n    \"\"\"Comprehensive model evaluation and visualization\"\"\"\n    \n    def __init__(self, monitor: MemoryMonitor, save_path: str = WORKING_PATH):\n        self.monitor = monitor\n        self.save_path = Path(save_path)\n        self.save_path.mkdir(exist_ok=True)\n        \n    def evaluate_model(self, y_true: np.ndarray, y_pred: np.ndarray,\n                      dataset_name: str = \"Validation\") -> Dict[str, float]:\n        \"\"\"Comprehensive model evaluation\"\"\"\n        print(f\"\\n=== Evaluating on {dataset_name} Set ===\")\n        \n        metrics = {}\n        \n        # Competition metric\n        comp_score = calculate_competition_metric(y_true, y_pred)\n        metrics['competition_metric'] = comp_score\n        print(f\"Competition Metric: {comp_score:.4f}\")\n        \n        # Per-class AUC\n        for idx, col in enumerate(LABEL_COLS):\n            try:\n                auc = roc_auc_score(y_true[:, idx], y_pred[:, idx])\n                metrics[f'auc_{col}'] = auc\n                print(f\"{col}: AUC = {auc:.4f}\")\n            except:\n                metrics[f'auc_{col}'] = 0.0\n        \n        # Overall metrics\n        y_pred_binary = (y_pred > 0.5).astype(int)\n        \n        try:\n            overall_f1 = f1_score(y_true, y_pred_binary, average='weighted')\n            metrics['f1_weighted'] = overall_f1\n            print(f\"Weighted F1 Score: {overall_f1:.4f}\")\n        except:\n            metrics['f1_weighted'] = 0.0\n        \n        return metrics\n    \n    def plot_training_history(self, history):\n        \"\"\"Plot training history\"\"\"\n        if history is None:\n            return\n        \n        fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n        \n        # Loss\n        axes[0, 0].plot(history.history['loss'], label='Train Loss')\n        axes[0, 0].plot(history.history['val_loss'], label='Val Loss')\n        axes[0, 0].set_title('Model Loss')\n        axes[0, 0].set_xlabel('Epoch')\n        axes[0, 0].set_ylabel('Loss')\n        axes[0, 0].legend()\n        axes[0, 0].grid(True)\n        \n        # Accuracy\n        axes[0, 1].plot(history.history['accuracy'], label='Train Accuracy')\n        axes[0, 1].plot(history.history['val_accuracy'], label='Val Accuracy')\n        axes[0, 1].set_title('Model Accuracy')\n        axes[0, 1].set_xlabel('Epoch')\n        axes[0, 1].set_ylabel('Accuracy')\n        axes[0, 1].legend()\n        axes[0, 1].grid(True)\n        \n        # AUC\n        if 'auc' in history.history:\n            axes[1, 0].plot(history.history['auc'], label='Train AUC')\n            axes[1, 0].plot(history.history['val_auc'], label='Val AUC')\n            axes[1, 0].set_title('Model AUC')\n            axes[1, 0].set_xlabel('Epoch')\n            axes[1, 0].set_ylabel('AUC')\n            axes[1, 0].legend()\n            axes[1, 0].grid(True)\n        \n        plt.tight_layout()\n        plt.show()\n        plt.savefig(self.save_path / 'training_history.png', dpi=150, bbox_inches='tight')\n        plt.close()\n        \n        print(f\"Saved training history plot to {self.save_path / 'training_history.png'}\")\n    \n    def plot_roc_curves(self, y_true: np.ndarray, y_pred: np.ndarray,\n                       dataset_name: str = \"Validation\"):\n        \"\"\"Plot ROC curves for all classes\"\"\"\n        n_classes = y_true.shape[1]\n        n_cols = 4\n        n_rows = (n_classes + n_cols - 1) // n_cols\n        \n        fig, axes = plt.subplots(n_rows, n_cols, figsize=(20, 5 * n_rows))\n        axes = axes.flatten()\n        \n        for idx, col in enumerate(LABEL_COLS):\n            try:\n                fpr, tpr, _ = roc_curve(y_true[:, idx], y_pred[:, idx])\n                auc = roc_auc_score(y_true[:, idx], y_pred[:, idx])\n                \n                axes[idx].plot(fpr, tpr, label=f'AUC = {auc:.3f}')\n                axes[idx].plot([0, 1], [0, 1], 'k--', label='Random')\n                axes[idx].set_title(col, fontsize=10)\n                axes[idx].set_xlabel('False Positive Rate')\n                axes[idx].set_ylabel('True Positive Rate')\n                axes[idx].legend()\n                axes[idx].grid(True)\n            except:\n                axes[idx].text(0.5, 0.5, 'No valid data', ha='center', va='center')\n                axes[idx].set_title(col, fontsize=10)\n        \n        # Hide empty subplots\n        for idx in range(n_classes, len(axes)):\n            axes[idx].axis('off')\n        \n        plt.tight_layout()\n        plt.show()\n        plt.savefig(self.save_path / f'roc_curves_{dataset_name.lower()}.png',\n                   dpi=150, bbox_inches='tight')\n        plt.close()\n        \n        print(f\"Saved ROC curves to {self.save_path / f'roc_curves_{dataset_name.lower()}.png'}\")\n    \n    def plot_confusion_matrices(self, y_true: np.ndarray, y_pred: np.ndarray,\n                               dataset_name: str = \"Validation\"):\n        \"\"\"Plot confusion matrices for main target\"\"\"\n        y_pred_binary = (y_pred[:, -1] > 0.5).astype(int)\n#************************************************************************************************\n\n        y_true_binary = y_true[:, -1].astype(int)\n        \n        cm = confusion_matrix(y_true_binary, y_pred_binary)\n        \n        plt.figure(figsize=(8, 6))\n        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', \n                   xticklabels=['No Aneurysm', 'Aneurysm'],\n                   yticklabels=['No Aneurysm', 'Aneurysm'])\n        plt.title(f'Confusion Matrix - {dataset_name} Set\\nAneurysm Present')\n        plt.ylabel('True Label')\n        plt.xlabel('Predicted Label')\n        plt.tight_layout()\n        plt.show()\n        plt.savefig(self.save_path / f'confusion_matrix_{dataset_name.lower()}.png',\n                   dpi=150, bbox_inches='tight')\n        plt.close()\n        \n        print(f\"Saved confusion matrix to {self.save_path / f'confusion_matrix_{dataset_name.lower()}.png'}\")\n        \n        # Print classification report\n        print(\"\\nClassification Report (Aneurysm Present):\")\n        print(classification_report(y_true_binary, y_pred_binary, \n                                   target_names=['No Aneurysm', 'Aneurysm']))\n\n\nclass ProductionPipeline:\n    \"\"\"End-to-end production pipeline\"\"\"\n    \n    def __init__(self, base_path: str = BASE_PATH):\n        self.base_path = base_path\n        self.monitor = MemoryMonitor()\n        self.data_loader = DataLoader(base_path, self.monitor)\n        self.feature_eng = FeatureEngineering(self.monitor)\n        self.image_extractor = ImageFeatureExtractor(self.monitor)\n        self.model = AneurysmDetectionModel(self.monitor)\n        self.evaluator = ModelEvaluator(self.monitor)\n        \n        self.train_df = None\n        self.val_df = None\n        self.test_df = None\n        \n    def prepare_data(self, test_size: float = 0.2, val_size: float = 0.1):\n        \"\"\"Prepare training, validation, and test datasets\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"RSNA INTRACRANIAL ANEURYSM DETECTION PIPELINE\")\n        print(\"=\"*60)\n        \n        # Load metadata\n        train_df, localizers_df = self.data_loader.load_metadata()\n        \n        # Feature engineering\n        train_df = self.feature_eng.engineer_features(train_df, is_train=True)\n        \n        # Split data\n        train_val_df, test_df = train_test_split(\n            train_df, \n            test_size=test_size, \n            random_state=42,\n            stratify=train_df['Aneurysm Present'] if 'Aneurysm Present' in train_df.columns else None\n        )\n        \n        train_df, val_df = train_test_split(\n            train_val_df,\n            test_size=val_size / (1 - test_size),\n            random_state=42,\n            stratify=train_val_df['Aneurysm Present'] if 'Aneurysm Present' in train_val_df.columns else None\n        )\n        \n        self.train_df = train_df\n        self.val_df = val_df\n        self.test_df = test_df\n        \n        print(f\"\\nDataset splits:\")\n        print(f\"  Training: {len(train_df)} samples\")\n        print(f\"  Validation: {len(val_df)} samples\")\n        print(f\"  Test: {len(test_df)} samples\")\n        \n        self.monitor.log_usage(\"After data preparation\")\n        \n        return train_df, val_df, test_df\n    \n    def extract_features(self, df: pd.DataFrame, feature_cols: List[str]) -> np.ndarray:\n        \"\"\"Extract numerical features from dataframe\"\"\"\n        available_cols = [col for col in feature_cols if col in df.columns]\n        \n        if not available_cols:\n            # Return dummy features if no columns available\n            return np.random.randn(len(df), 10)\n        \n        X = df[available_cols].fillna(0).values\n        return X\n    \n    def prepare_image_data(self, df: pd.DataFrame, max_samples: int = None) -> np.ndarray:\n        \"\"\"Prepare image data for a dataset\"\"\"\n        if max_samples is not None:\n            df = df.head(max_samples)\n        \n        images = []\n        valid_indices = []\n        \n        print(f\"Loading images for {len(df)} samples...\")\n        for idx, row in tqdm(df.iterrows(), total=len(df)):\n            series_id = row[ID_COL]\n            volume = self.data_loader.load_dicom_series(series_id, max_slices=32)\n            \n            if volume is not None and len(volume) > 0:\n                # Take middle slice\n                mid_idx = len(volume) // 2\n                img = volume[mid_idx]\n                images.append(img)\n                valid_indices.append(idx)\n            \n            if not self.monitor.is_safe():\n                print(\"Memory limit approaching, stopping image loading\")\n                break\n        \n        if not images:\n            # Return dummy images if no real images loaded\n            print(\"WARNING: No images loaded, using synthetic data\")\n            return np.random.randn(len(df), IMG_SIZE, IMG_SIZE)\n        \n        images = np.array(images)\n        print(f\"Loaded {len(images)} valid images\")\n        \n        return images\n    \n    def run_training(self, use_images: bool = False):\n        \"\"\"Run the complete training pipeline\"\"\"\n        print(\"\\n=== Starting Training Pipeline ===\")\n        self.monitor.log_usage(\"Start of training\")\n        \n        # Prepare data\n        train_df, val_df, test_df = self.prepare_data()\n        \n        # Define feature columns\n        feature_cols = []\n        if 'PatientAge' in train_df.columns:\n            feature_cols.extend(['PatientAge', 'age_squared', 'age_log', 'age_group'])\n        if 'PatientSex_encoded' in train_df.columns:\n            feature_cols.append('PatientSex_encoded')\n        if 'Modality_encoded' in train_df.columns:\n            feature_cols.append('Modality_encoded')\n        \n        # Add engineered features\n        if 'total_aneurysm_locations' in train_df.columns:\n            feature_cols.extend([\n                'total_aneurysm_locations', 'left_aneurysms', 'right_aneurysms',\n                'anterior_aneurysms', 'posterior_aneurysms'\n            ])\n        \n        # Extract features\n        X_train = self.extract_features(train_df, feature_cols)\n        X_val = self.extract_features(val_df, feature_cols)\n        X_test = self.extract_features(test_df, feature_cols)\n        \n        # Extract labels\n        y_train = train_df[LABEL_COLS].values\n        y_val = val_df[LABEL_COLS].values\n        y_test = test_df[LABEL_COLS].values\n        \n        # Scale features\n        X_train, X_val, X_test = self.feature_eng.scale_features(X_train, X_val, X_test)\n        \n        print(f\"\\nFeature shapes:\")\n        print(f\"  X_train: {X_train.shape}\")\n        print(f\"  y_train: {y_train.shape}\")\n        \n        # Train ensemble models\n        self.model.train_ensemble(X_train, y_train, X_val, y_val)\n        \n        # Train CNN if using images\n        X_train_img, X_val_img, X_test_img = None, None, None\n        if use_images:\n            try:\n                X_train_img = self.prepare_image_data(train_df, max_samples=MAX_SERIES_SAMPLE)\n                X_val_img = self.prepare_image_data(val_df, max_samples=MAX_SERIES_SAMPLE // 2)\n                \n                # Align samples\n                min_train = min(len(X_train), len(X_train_img))\n                min_val = min(len(X_val), len(X_val_img))\n                \n                X_train = X_train[:min_train]\n                y_train = y_train[:min_train]\n                X_train_img = X_train_img[:min_train]\n                \n                X_val = X_val[:min_val]\n                y_val = y_val[:min_val]\n                X_val_img = X_val_img[:min_val]\n                \n                # Train CNN\n                self.model.train_cnn(X_train_img, y_train, X_val_img, y_val)\n                \n                # Evaluate with images\n                X_test_img = self.prepare_image_data(test_df, max_samples=MAX_SERIES_SAMPLE // 4)\n                min_test = min(len(X_test), len(X_test_img))\n                X_test = X_test[:min_test]\n                y_test = y_test[:min_test]\n                X_test_img = X_test_img[:min_test]\n                \n            except Exception as e:\n                print(f\"Error loading images: {e}\")\n                print(\"Continuing with tabular features only\")\n                use_images = False\n        \n        # Generate predictions\n        print(\"\\n=== Generating Predictions ===\")\n        \n        val_pred = self.model.predict_ensemble(X_val, X_val_img)\n        test_pred = self.model.predict_ensemble(X_test, X_test_img)\n        \n        # Evaluate\n        val_metrics = self.evaluator.evaluate_model(y_val, val_pred, \"Validation\")\n        test_metrics = self.evaluator.evaluate_model(y_test, test_pred, \"Test\")\n        \n        # Visualizations\n        if self.model.history:\n            self.evaluator.plot_training_history(self.model.history)\n        \n        self.evaluator.plot_roc_curves(y_val, val_pred, \"Validation\")\n        self.evaluator.plot_roc_curves(y_test, test_pred, \"Test\")\n        self.evaluator.plot_confusion_matrices(y_val, val_pred, \"Validation\")\n        self.evaluator.plot_confusion_matrices(y_test, test_pred, \"Test\")\n        \n        # Save results\n        self.save_results(val_metrics, test_metrics)\n        \n        self.monitor.log_usage(\"End of training\")\n        \n        return val_metrics, test_metrics\n    \n    def save_results(self, val_metrics: Dict, test_metrics: Dict):\n        \"\"\"Save evaluation results to JSON\"\"\"\n        results = {\n            'validation_metrics': val_metrics,\n            'test_metrics': test_metrics,\n            'model_config': {\n                'img_size': IMG_SIZE,\n                'batch_size': BATCH_SIZE,\n                'epochs': EPOCHS,\n                'max_series_sample': MAX_SERIES_SAMPLE\n            }\n        }\n        \n        results_path = Path(WORKING_PATH) / 'evaluation_results.json'\n        with open(results_path, 'w') as f:\n            json.dump(results, f, indent=4)\n        \n        print(f\"\\nSaved results to {results_path}\")\n    \n    def create_submission(self, test_df: pd.DataFrame, predictions: np.ndarray, \n                         output_path: str = None):\n        \"\"\"Create submission file\"\"\"\n        if output_path is None:\n            output_path = Path(WORKING_PATH) / 'submission.csv'\n        \n        submission = pd.DataFrame({\n            ID_COL: test_df[ID_COL].values\n        })\n        \n        for idx, col in enumerate(LABEL_COLS):\n            submission[col] = predictions[:, idx]\n        \n        submission.to_csv(output_path, index=False)\n        print(f\"\\nSubmission file saved to {output_path}\")\n        \n        return submission\n\n\ndef main():\n    \"\"\"Main execution function\"\"\"\n    try:\n        # Initialize pipeline\n        pipeline = ProductionPipeline(BASE_PATH)\n        \n        # Run training\n        val_metrics, test_metrics = pipeline.run_training(use_images=False)\n        \n        # Print final results\n        print(\"\\n\" + \"=\"*60)\n        print(\"FINAL RESULTS\")\n        print(\"=\"*60)\n        print(f\"\\nValidation Competition Metric: {val_metrics['competition_metric']:.4f}\")\n        print(f\"Test Competition Metric: {test_metrics['competition_metric']:.4f}\")\n        \n        print(\"\\n\" + \"=\"*60)\n        print(\"PIPELINE COMPLETED SUCCESSFULLY\")\n        print(\"=\"*60)\n        \n    except Exception as e:\n        print(f\"\\nERROR: Pipeline failed with exception:\")\n        print(f\"{type(e).__name__}: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n    \n    finally:\n        # Final cleanup\n        gc.collect()\n        if gpus:\n            try:\n                tf.keras.backend.clear_session()\n            except:\n                pass\n\n\nif __name__ == \"__main__\":\n    main()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-14T21:49:01.530884Z","iopub.execute_input":"2025-10-14T21:49:01.531201Z","iopub.status.idle":"2025-10-14T21:49:34.051695Z","shell.execute_reply.started":"2025-10-14T21:49:01.531177Z","shell.execute_reply":"2025-10-14T21:49:34.050841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}