{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":13451,"datasetId":654585,"databundleVersionId":1188070}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Kaggle notebook/script: build HemSeg-500 from the updated repo\n# and optionally derive RSNA-based volume labels (presence + extent)\n# for each reconstructed 3D CT volume.\n#\n# Sources:\n# - HemSeg repo / README: https://github.com/songchangwei/3DCT-SD-IVH-ICH\n# - Kaggle RSNA competition labels: stage_2_train.csv (long format: ID, Label)\n#\n# Output structure:\n#   /kaggle/working/hemseg500/\n#       ct_nifti/\n#       labels/\n#       splits/\n#       metadata/\n#           hemseg500_slice_to_volume_map.csv\n#           hemseg500_volume_labels_presence_extent.csv   <-- new\n#       build_log.csv\n\nimport os\nimport re\nimport json\nimport shutil\nimport zipfile\nimport subprocess\nfrom pathlib import Path\nfrom typing import Iterable, List, Dict, Optional\n\nimport numpy as np\nimport pandas as pd\n\nimport SimpleITK as sitk\n\n# Hide SimpleITK / ITK warning messages\nsitk.ProcessObject.GlobalWarningDisplayOff()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 1) CONFIG\n# ============================================================\n# Change these if your Kaggle input mount names differ.\n#\n# Competition Link:\n# https://www.kaggle.com/competitions/rsna-intracranial-hemorrhage-detection\n#\n# Common Kaggle paths are either:\n#   /kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\n# or\n#   /kaggle/input/competitions/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\nRSNA_DATASET_CANDIDATES = [\n    \"/kaggle/input/competitions/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\",\n    \"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\",\n]\n\nOUTPUT_ROOT = \"/kaggle/working/hemseg500\"\nREPO_DIR = \"/kaggle/working/3DCT-SD-IVH-ICH\"\nREPO_URL = \"https://github.com/songchangwei/3DCT-SD-IVH-ICH.git\"\n\n# Updated HemSeg-500 resources in the repo\nANNOT_JSONL_NAME = \"annotion_file_info_500.jsonl\"\nHEMSEG500_ZIP_NAME = \"HemSeg-500.zip\"\n\n# RSNA labels\nRSNA_LABELS_CSV_NAME = \"stage_2_train.csv\"\nLABELS = [\"epidural\", \"intraparenchymal\", \"intraventricular\", \"subarachnoid\", \"subdural\", \"any\"]\nMAKE_VOLUME_LABELS = True\nFORCE_DERIVE_ANY_FROM_5 = False  # set True if you want 'any' derived only from the 5 subtypes\n\n# Optional: final zip export of OUTPUT_ROOT\nMAKE_FINAL_ZIP = True\nFINAL_ZIP_BASENAME = \"/kaggle/working/hemseg500\"\n\nos.makedirs(OUTPUT_ROOT, exist_ok=True)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 2) HELPERS\n# ============================================================\n\ndef run(cmd: str, check: bool = True) -> None:\n    print(f\"\\n$ {cmd}\")\n    subprocess.run(cmd, shell=True, check=check)\n\n\ndef detect_rsna_root() -> str:\n    for p in RSNA_DATASET_CANDIDATES:\n        if os.path.exists(p):\n            return p\n    raise FileNotFoundError(\n        \"Could not find RSNA dataset root. Checked:\\n\" + \"\\n\".join(RSNA_DATASET_CANDIDATES)\n    )\n\n\ndef ensure_repo(repo_url: str, repo_dir: str) -> None:\n    if os.path.exists(repo_dir):\n        print(f\"Repo already exists: {repo_dir}\")\n    else:\n        run(f\"git clone {repo_url} {repo_dir}\")\n\n\ndef install_requirements() -> None:\n    # Minimal build dependencies; avoids pulling the full training stack.\n    run(\"python -m pip install --quiet --upgrade pip\")\n    run(\"python -m pip install --quiet jsonlines pydicom nibabel SimpleITK pandas\")\n\n\ndef ensure_file_exists(path: str, msg: str = \"\") -> None:\n    if not os.path.exists(path):\n        raise FileNotFoundError(f\"Missing: {path}. {msg}\".strip())\n\n\ndef maybe_download(url: str, out_path: str) -> None:\n    if os.path.exists(out_path):\n        print(f\"Already present: {out_path}\")\n        return\n    run(f'curl -L \"{url}\" -o \"{out_path}\"')\n\n\ndef extract_zip(zip_path: str, out_dir: str) -> None:\n    os.makedirs(out_dir, exist_ok=True)\n    with zipfile.ZipFile(zip_path, \"r\") as zf:\n        zf.extractall(out_dir)\n    print(f\"Extracted: {zip_path} -> {out_dir}\")\n\n\ndef load_jsonl(path: str) -> List[Dict]:\n    records = []\n    with open(path, \"r\", encoding=\"utf-8\") as f:\n        for line_num, line in enumerate(f, start=1):\n            line = line.strip()\n            if not line:\n                continue\n            try:\n                records.append(json.loads(line))\n            except json.JSONDecodeError as e:\n                raise ValueError(f\"Invalid JSON on line {line_num} of {path}: {e}\")\n    return records\n\n\ndef get_series_file_names(dicom_dir: str) -> List[str]:\n    import SimpleITK as sitk\n\n    series_ids = sitk.ImageSeriesReader.GetGDCMSeriesIDs(dicom_dir)\n    if not series_ids:\n        raise RuntimeError(f\"No DICOM series IDs found in {dicom_dir}\")\n    # Same behavior as the repo: use the first detected series\n    return list(sitk.ImageSeriesReader.GetGDCMSeriesFileNames(dicom_dir, series_ids[0]))\n\n\ndef dicom_dir_to_nii_gz(dicom_dir: str, out_path: str) -> None:\n    import SimpleITK as sitk\n\n    series_files = get_series_file_names(dicom_dir)\n    reader = sitk.ImageSeriesReader()\n    reader.SetFileNames(series_files)\n    image = reader.Execute()\n    sitk.WriteImage(image, out_path)\n\n\ndef normalize_slice_id(s: str) -> str:\n    \"\"\"\n    Convert inputs like:\n      'ID_xxx' -> 'ID_xxx'\n      'xxx'    -> 'ID_xxx'\n      'ID_xxx.dcm' -> 'ID_xxx'\n    \"\"\"\n    s = str(s).strip()\n    s = Path(s).name\n    if s.endswith(\".dcm\"):\n        s = s[:-4]\n    if not s.startswith(\"ID_\"):\n        s = f\"ID_{s}\"\n    return s\n\n\ndef copy_if_exists(src: str, dst: str) -> None:\n    if os.path.exists(src):\n        os.makedirs(os.path.dirname(dst), exist_ok=True)\n        shutil.copy2(src, dst)\n        print(f\"Copied: {src} -> {dst}\")\n\n\ndef find_existing_file(base_dir: str, relative_candidates: List[str]) -> Optional[str]:\n    for rel in relative_candidates:\n        path = os.path.join(base_dir, rel)\n        if os.path.exists(path):\n            return path\n    return None","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 3) RSNA LABEL HELPERS\n# ============================================================\n\ndef _find_col(df, candidates):\n    for c in candidates:\n        if c in df.columns:\n            return c\n    return None\n\n\ndef load_rsna_slice_labels(rsna_labels_csv: str) -> pd.DataFrame:\n    \"\"\"\n    Supports:\n      1) Kaggle RSNA format: columns ['ID','Label'] where ID looks like 'ID_xxx_epidural'\n      2) Wide format: columns include the 6 labels already\n    Returns a wide per-slice df with columns: ['image_id'] + LABELS\n    \"\"\"\n    df = pd.read_csv(rsna_labels_csv)\n\n    # Case 1: Kaggle long format (ID, Label)\n    if \"ID\" in df.columns and \"Label\" in df.columns:\n        tmp = df.copy()\n        parts = tmp[\"ID\"].astype(str).str.rsplit(\"_\", n=1, expand=True)\n        tmp[\"image_id\"] = parts[0]\n        tmp[\"diagnosis\"] = parts[1].str.lower()\n\n        tmp = tmp[tmp[\"diagnosis\"].isin(LABELS)]\n        wide = tmp.pivot_table(\n            index=\"image_id\", columns=\"diagnosis\", values=\"Label\", aggfunc=\"max\"\n        ).reset_index()\n\n        for c in LABELS:\n            if c not in wide.columns:\n                wide[c] = 0.0\n        wide = wide[[\"image_id\"] + LABELS]\n        return wide\n\n    # Case 2: already wide\n    img_col = _find_col(df, [\"image_id\", \"ImageID\", \"SOPInstanceUID\", \"SOPInstanceId\", \"instance_uid\", \"id\", \"ID\"])\n    if img_col is None:\n        raise ValueError(f\"Can't find image id column in {df.columns.tolist()}\")\n\n    out = df.rename(columns={img_col: \"image_id\"}).copy()\n    out[\"image_id\"] = out[\"image_id\"].astype(str).map(normalize_slice_id)\n    for c in LABELS:\n        if c not in out.columns:\n            out[c] = 0.0\n    out = out[[\"image_id\"] + LABELS]\n    return out\n\n\ndef make_volume_labels_from_slice_map(rsna_labels_csv: str,\n                                      slice_to_volume_csv: str,\n                                      out_csv: str,\n                                      force_derive_any_from_5: bool = False) -> pd.DataFrame:\n    \"\"\"\n    Build volume-level labels for reconstructed HemSeg volumes.\n\n    Expected mapping CSV columns:\n      - image_id   (e.g. ID_63eb1e259)\n      - volume_id  (HemSeg scan name used as output NIfTI stem)\n\n    Presence = max across slices\n    Extent   = mean across slices\n    \"\"\"\n    sl = load_rsna_slice_labels(rsna_labels_csv)\n    mp = pd.read_csv(slice_to_volume_csv)\n\n    if \"image_id\" not in mp.columns or \"volume_id\" not in mp.columns:\n        raise ValueError(\n            f\"slice_to_volume_csv must contain image_id and volume_id. Found: {mp.columns.tolist()}\"\n        )\n\n    mp = mp[[\"image_id\", \"volume_id\"]].dropna().copy()\n    mp[\"image_id\"] = mp[\"image_id\"].astype(str).map(normalize_slice_id)\n    mp[\"volume_id\"] = mp[\"volume_id\"].astype(str)\n    mp = mp.drop_duplicates()\n\n    df = mp.merge(sl, on=\"image_id\", how=\"left\")\n\n    # Fill missing label rows with zeros so the aggregation still works,\n    # while preserving an audit of how many slices matched the RSNA CSV.\n    for c in LABELS:\n        if c not in df.columns:\n            df[c] = 0.0\n        df[c] = df[c].fillna(0.0).astype(np.float32)\n\n    g = df.groupby(\"volume_id\", sort=False)\n    pres = g[LABELS].max().astype(np.float32)\n    ext = g[LABELS].mean().astype(np.float32)\n\n    if force_derive_any_from_5:\n        pres[\"any\"] = pres[LABELS[:-1]].max(axis=1).astype(np.float32)\n        # Simple mean of slice-wise subtype max, bounded in [0,1]\n        ext[\"any\"] = ext[LABELS[:-1]].max(axis=1).astype(np.float32)\n\n    out = pd.DataFrame({\"volume_id\": pres.index})\n    for c in LABELS:\n        out[f\"{c}_pres\"] = pres[c].values.astype(np.float32)\n    for c in LABELS:\n        out[f\"{c}_ext\"] = ext[c].values.astype(np.float32)\n\n    # Useful audit columns\n    matched_any = (df[LABELS].sum(axis=1) >= 0).astype(np.int32)  # all mapped rows count after fillna\n    out = out.merge(\n        g.size().rename(\"n_slices_total\").reset_index().rename(columns={\"volume_id\": \"volume_id\"}),\n        on=\"volume_id\", how=\"left\"\n    )\n\n    # Better audit: count rows whose image_id existed in label CSV before fillna\n    label_ids = set(sl[\"image_id\"].astype(str).tolist())\n    df[\"has_rsna_label_row\"] = df[\"image_id\"].astype(str).isin(label_ids).astype(np.int32)\n    audit = g[\"has_rsna_label_row\"].sum().rename(\"n_slices_with_rsna_labels\").reset_index()\n    out = out.merge(audit, on=\"volume_id\", how=\"left\")\n\n    out.to_csv(out_csv, index=False)\n    print(f\"Saved volume labels: {out_csv} | rows: {len(out)}\")\n    return out","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 4) BUILD ONE VOLUME + SLICE MAP ROWS\n# ============================================================\n\ndef build_selected_volume(raw_dicom_dir: str,\n                          scan_name: str,\n                          slice_ids: Iterable[str],\n                          temp_root: str,\n                          out_nii_path: str) -> Dict:\n    \"\"\"\n    Rebuild one RSNA study volume from the selected DICOM slices listed in the annotation JSONL.\n    Also returns a per-slice map so we can aggregate RSNA slice labels -> volume labels.\n    \"\"\"\n    slice_ids = list(slice_ids)\n    scan_temp_dir = os.path.join(temp_root, scan_name)\n    os.makedirs(scan_temp_dir, exist_ok=True)\n\n    found, missing = 0, []\n    slice_map_rows = []\n    for sid in slice_ids:\n        image_id = normalize_slice_id(sid)\n        src = os.path.join(raw_dicom_dir, f\"{image_id}.dcm\")\n        dst = os.path.join(scan_temp_dir, f\"{image_id}.dcm\")\n        if os.path.exists(src):\n            shutil.copy2(src, dst)\n            found += 1\n            slice_map_rows.append({\"volume_id\": scan_name, \"image_id\": image_id, \"filepath\": src})\n        else:\n            missing.append(src)\n\n    if found == 0:\n        return {\n            \"scan_name\": scan_name,\n            \"status\": \"failed_no_slices_found\",\n            \"n_expected\": len(slice_ids),\n            \"n_found\": 0,\n            \"n_missing\": len(missing),\n            \"out_nii_path\": out_nii_path,\n            \"slice_map_rows\": slice_map_rows,\n        }\n\n    try:\n        dicom_dir_to_nii_gz(scan_temp_dir, out_nii_path)\n        status = \"ok\"\n    except Exception as e:\n        status = f\"failed_convert: {e}\"\n\n    return {\n        \"scan_name\": scan_name,\n        \"status\": status,\n        \"n_expected\": len(slice_ids),\n        \"n_found\": found,\n        \"n_missing\": len(missing),\n        \"out_nii_path\": out_nii_path,\n        \"slice_map_rows\": slice_map_rows,\n    }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 5) PREPARE REPO + INPUTS\n# ============================================================\nRSNA_DATASET_DIR = detect_rsna_root()\nRAW_DICOM_DIR = os.path.join(RSNA_DATASET_DIR, \"stage_2_train\")\nRSNA_LABELS_CSV = os.path.join(RSNA_DATASET_DIR, RSNA_LABELS_CSV_NAME)\n\nprint(\"RSNA_DATASET_DIR:\", RSNA_DATASET_DIR)\nprint(\"RAW_DICOM_DIR:   \", RAW_DICOM_DIR)\nprint(\"RSNA_LABELS_CSV: \", RSNA_LABELS_CSV)\nprint(\"OUTPUT_ROOT:     \", OUTPUT_ROOT)\nprint(\"REPO_DIR:        \", REPO_DIR)\n\nensure_file_exists(RAW_DICOM_DIR, \"Check your Kaggle RSNA mount path.\")\nensure_file_exists(RSNA_LABELS_CSV, \"RSNA stage_2_train.csv is required for volume label aggregation.\")\ninstall_requirements()\nensure_repo(REPO_URL, REPO_DIR)\n\nannot_jsonl = os.path.join(REPO_DIR, ANNOT_JSONL_NAME)\nensure_file_exists(annot_jsonl, \"The updated repo should contain annotion_file_info_500.jsonl\")\n\nhemseg500_zip = os.path.join(REPO_DIR, HEMSEG500_ZIP_NAME)\nif not os.path.exists(hemseg500_zip):\n    maybe_download(\n        f\"https://github.com/songchangwei/3DCT-SD-IVH-ICH/raw/main/{HEMSEG500_ZIP_NAME}\",\n        hemseg500_zip,\n    )\nensure_file_exists(hemseg500_zip, \"HemSeg-500.zip is required for labels/splits packaged by the updated repo.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 6) EXTRACT LABEL PACKAGE\n# ============================================================\nextracted_dir = os.path.join(OUTPUT_ROOT, \"repo_package\")\nextract_zip(hemseg500_zip, extracted_dir)\n\nct_nifti_dir = os.path.join(OUTPUT_ROOT, \"ct_nifti\")\nlabels_dir = os.path.join(OUTPUT_ROOT, \"labels\")\nmetadata_dir = os.path.join(OUTPUT_ROOT, \"metadata\")\nsplits_dir = os.path.join(OUTPUT_ROOT, \"splits\")\ntemp_dir = os.path.join(OUTPUT_ROOT, \"_tmp_selected_dicoms\")\n\nfor d in [ct_nifti_dir, labels_dir, metadata_dir, splits_dir, temp_dir]:\n    os.makedirs(d, exist_ok=True)\n\nlabel_exts = {\".nii\", \".nii.gz\", \".nrrd\", \".mha\", \".mhd\"}\nsplit_names = {\"train_set.csv\", \"val_set.csv\", \"test_set.csv\"}\n\nfor root, _, files in os.walk(extracted_dir):\n    for fname in files:\n        fpath = os.path.join(root, fname)\n        lower = fname.lower()\n\n        if fname in split_names:\n            copy_if_exists(fpath, os.path.join(splits_dir, fname))\n            continue\n\n        if lower.endswith(\".nii.gz\") or Path(lower).suffix in label_exts:\n            rel = os.path.relpath(fpath, extracted_dir)\n            dst = os.path.join(labels_dir, rel)\n            os.makedirs(os.path.dirname(dst), exist_ok=True)\n            shutil.copy2(fpath, dst)\n\nfor csv_name in split_names:\n    dst = os.path.join(splits_dir, csv_name)\n    if not os.path.exists(dst):\n        repo_csv = os.path.join(REPO_DIR, csv_name)\n        copy_if_exists(repo_csv, dst)\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 7) BUILD CT NIFTI VOLUMES FOR HEMSEG-500 + SAVE SLICE MAP\n# ============================================================\nrecords = load_jsonl(annot_jsonl)\nprint(f\"Loaded {len(records)} annotation records from {annot_jsonl}\")\n\ndef extract_slice_ids(slice_obj):\n    \"\"\"\n    HemSeg-500 stores slice_list as a dict:\n      {\"0\": \"7624b0593\", \"1\": \"5d1566dc6\", ...}\n    Older formats may use a plain list.\n    \"\"\"\n    if slice_obj is None:\n        return []\n\n    if isinstance(slice_obj, dict):\n        # sort by numeric key to preserve slice order\n        items = sorted(slice_obj.items(), key=lambda kv: int(kv[0]))\n        return [v for _, v in items]\n\n    if isinstance(slice_obj, list):\n        return list(slice_obj)\n\n    raise ValueError(f\"Unsupported slice_list type: {type(slice_obj)}\")\n\nbuild_log = []\nslice_to_volume_rows = []\n\nfor i, item in enumerate(records, start=1):\n    scan_name = item.get(\"scan_\") or item.get(\"scan\") or item.get(\"scan_name\") or item.get(\"name\")\n    raw_slice_obj = item.get(\"slice_list\") or item.get(\"slices\") or item.get(\"slice_ids\")\n    slice_list = extract_slice_ids(raw_slice_obj)\n\n    if not scan_name or not slice_list:\n        build_log.append({\n            \"scan_name\": scan_name or f\"record_{i}\",\n            \"status\": \"failed_bad_jsonl_record\",\n            \"n_expected\": 0,\n            \"n_found\": 0,\n            \"n_missing\": 0,\n            \"out_nii_path\": \"\",\n        })\n        continue\n\n    out_nii = os.path.join(ct_nifti_dir, f\"{scan_name}.nii.gz\")\n\n    if os.path.exists(out_nii):\n        build_log.append({\n            \"scan_name\": scan_name,\n            \"status\": \"skipped_exists\",\n            \"n_expected\": len(slice_list),\n            \"n_found\": len(slice_list),\n            \"n_missing\": 0,\n            \"out_nii_path\": out_nii,\n        })\n        for sid in slice_list:\n            image_id = normalize_slice_id(sid)\n            src = os.path.join(RAW_DICOM_DIR, f\"{image_id}.dcm\")\n            if os.path.exists(src):\n                slice_to_volume_rows.append({\n                    \"volume_id\": scan_name,\n                    \"image_id\": image_id,\n                    \"filepath\": src\n                })\n        continue\n\n    log_row = build_selected_volume(\n        raw_dicom_dir=RAW_DICOM_DIR,\n        scan_name=scan_name,\n        slice_ids=slice_list,\n        temp_root=temp_dir,\n        out_nii_path=out_nii,\n    )\n    slice_to_volume_rows.extend(log_row.pop(\"slice_map_rows\", []))\n    build_log.append(log_row)\n\n    if i % 25 == 0 or i == len(records):\n        ok_count = sum(r[\"status\"] == \"ok\" for r in build_log)\n        no_slice_count = sum(r[\"status\"] == \"failed_no_slices_found\" for r in build_log)\n        print(f\"Processed {i}/{len(records)} | ok={ok_count} | no_slices={no_slice_count}\")\n\nslice_to_volume_csv = os.path.join(metadata_dir, \"hemseg500_slice_to_volume_map.csv\")\npd.DataFrame(slice_to_volume_rows).drop_duplicates().to_csv(slice_to_volume_csv, index=False)\nprint(f\"Saved slice-to-volume map: {slice_to_volume_csv}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-04-22T08:25:46.428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"build_log_df = pd.DataFrame(build_log)\nprint(build_log_df[\"status\"].value_counts(dropna=False))\ndisplay(build_log_df.head(20))","metadata":{"trusted":true,"execution":{"execution_failed":"2026-04-22T08:25:46.428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport pydicom\n\ndef get_z_from_ds(ds):\n    ipp = getattr(ds, \"ImagePositionPatient\", None)\n    if ipp is not None and len(ipp) >= 3:\n        try:\n            return float(ipp[2])\n        except:\n            return None\n    return None\n\nqc_rows = []\n\nslice_map = pd.read_csv(\"/kaggle/working/hemseg500/metadata/hemseg500_slice_to_volume_map.csv\")\n\nfor vol_id, g in slice_map.groupby(\"volume_id\"):\n    dcm_paths = g[\"filepath\"].tolist()\n    z_vals = []\n\n    for p in dcm_paths:\n        try:\n            ds = pydicom.dcmread(p, stop_before_pixels=True, force=True)\n            z = get_z_from_ds(ds)\n            if z is not None:\n                z_vals.append(z)\n        except:\n            pass\n\n    z_vals = sorted(z_vals)\n    if len(z_vals) >= 3:\n        diffs = np.diff(z_vals)\n        med = np.median(np.abs(diffs))\n        nonuniformity = float(np.max(np.abs(np.abs(diffs) - med)))\n    else:\n        med = np.nan\n        nonuniformity = np.nan\n\n    qc_rows.append({\n        \"volume_id\": vol_id,\n        \"n_slices\": len(dcm_paths),\n        \"z_spacing_median\": med,\n        \"z_nonuniformity_max\": nonuniformity,\n    })\n\nqc_df = pd.DataFrame(qc_rows)\nqc_df.to_csv(\"/kaggle/working/hemseg500/metadata/hemseg500_spacing_qc.csv\", index=False)\n\nprint(qc_df[\"z_nonuniformity_max\"].describe())\ndisplay(qc_df.sort_values(\"z_nonuniformity_max\", ascending=False).head(20))","metadata":{"trusted":true,"execution":{"execution_failed":"2026-04-22T08:25:46.428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 8) SAVE BUILD REPORT + BUILD VOLUME LABELS\n# ============================================================\nbuild_csv = os.path.join(OUTPUT_ROOT, \"build_log.csv\")\npd.DataFrame(build_log).to_csv(build_csv, index=False)\nprint(f\"Saved build log: {build_csv}\")\n\nif MAKE_VOLUME_LABELS:\n    vol_labels_csv = os.path.join(metadata_dir, \"hemseg500_volume_labels_presence_extent.csv\")\n    _vol_df = make_volume_labels_from_slice_map(\n        rsna_labels_csv=RSNA_LABELS_CSV,\n        slice_to_volume_csv=slice_to_volume_csv,\n        out_csv=vol_labels_csv,\n        force_derive_any_from_5=FORCE_DERIVE_ANY_FROM_5,\n    )\n\n# Optional cleanup of temporary copied DICOM folders\nif os.path.exists(temp_dir):\n    shutil.rmtree(temp_dir)\n    print(f\"Removed temp dir: {temp_dir}\")","metadata":{"trusted":true,"execution":{"execution_failed":"2026-04-22T08:25:46.428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 9) SUMMARY\n# ============================================================\nct_files = list(Path(ct_nifti_dir).rglob(\"*.nii.gz\"))\nlabel_files = [p for p in Path(labels_dir).rglob(\"*\") if p.is_file()]\nsplit_csvs = [p.name for p in Path(splits_dir).glob(\"*.csv\")]\n\nprint(\"\\n=== SUMMARY ===\")\nprint(\"CT NIfTI volumes:\", len(ct_files))\nprint(\"Label package files:\", len(label_files))\nprint(\"Split CSVs:\", split_csvs)\nprint(\"Slice->volume map:\", slice_to_volume_csv)\nif MAKE_VOLUME_LABELS:\n    print(\"Volume labels CSV:\", vol_labels_csv)\n","metadata":{"trusted":true,"execution":{"execution_failed":"2026-04-22T08:25:46.428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 10) HemSeg500 final folder cleanup — adjusted paths\n#   /kaggle/working/hemseg500/ct_nifti  ->  /kaggle/working/hemseg500/ct_scans\n#   /kaggle/working/hemseg500/labels/HemSeg-500/{train,val,test}/label/*.nii.gz\n#       -> /kaggle/working/hemseg500/masks\n#   delete /kaggle/working/hemseg500/labels after successful verification\n# ============================================================\n\nfrom pathlib import Path\nimport shutil\nimport pandas as pd\n\nHEMSEG_ROOT = Path(\"/kaggle/working/hemseg500\")\n\nCT_OLD_DIR = HEMSEG_ROOT / \"ct_nifti\"\nCT_NEW_DIR = HEMSEG_ROOT / \"ct_scans\"\n\nLABELS_ROOT = HEMSEG_ROOT / \"labels\"\nLABELS_HEMSEG_DIR = LABELS_ROOT / \"HemSeg-500\"\n\nMASKS_DIR = HEMSEG_ROOT / \"masks\"\n\nREPORT_CSV = HEMSEG_ROOT / \"hemseg500_final_folder_cleanup_report.csv\"\nSUMMARY_CSV = HEMSEG_ROOT / \"hemseg500_final_folder_cleanup_summary.csv\"\nMISMATCH_CSV = HEMSEG_ROOT / \"hemseg500_final_folder_cleanup_mismatches.csv\"\n\ndef nii_stem(p: Path) -> str:\n    name = p.name\n    if name.endswith(\".nii.gz\"):\n        return name[:-7]\n    if name.endswith(\".nii\"):\n        return name[:-4]\n    return p.stem\n\ndef list_niigz(folder: Path):\n    if not folder.exists():\n        return []\n    return sorted([\n        p for p in folder.iterdir()\n        if p.is_file() and p.name.endswith(\".nii.gz\")\n    ])\n\n# ------------------------------------------------------------\n# 1) Basic folder checks\n# ------------------------------------------------------------\nif not HEMSEG_ROOT.exists():\n    raise FileNotFoundError(f\"HEMSEG_ROOT does not exist: {HEMSEG_ROOT}\")\n\nif not CT_OLD_DIR.exists() and not CT_NEW_DIR.exists():\n    raise FileNotFoundError(\n        f\"Neither ct_nifti nor ct_scans exists under: {HEMSEG_ROOT}\"\n    )\n\nif not LABELS_HEMSEG_DIR.exists():\n    raise FileNotFoundError(f\"Missing labels/HemSeg-500 folder: {LABELS_HEMSEG_DIR}\")\n\n# ------------------------------------------------------------\n# 2) Rename ct_nifti -> ct_scans\n# ------------------------------------------------------------\nif CT_OLD_DIR.exists():\n    if CT_NEW_DIR.exists():\n        raise FileExistsError(\n            f\"Both ct_nifti and ct_scans exist. Please inspect manually:\\n\"\n            f\"{CT_OLD_DIR}\\n{CT_NEW_DIR}\"\n        )\n    CT_OLD_DIR.rename(CT_NEW_DIR)\n    print(f\"Renamed: {CT_OLD_DIR} -> {CT_NEW_DIR}\")\nelse:\n    print(\"ct_scans already exists; no rename needed.\")\n\n# ------------------------------------------------------------\n# 3) Create masks folder\n# ------------------------------------------------------------\nMASKS_DIR.mkdir(parents=True, exist_ok=True)\n\nexisting_masks = list_niigz(MASKS_DIR)\nif len(existing_masks) > 0:\n    raise RuntimeError(\n        f\"masks folder already contains {len(existing_masks)} .nii.gz files:\\n\"\n        f\"{MASKS_DIR}\\n\"\n        \"Please clear it first if you want to rebuild masks.\"\n    )\n\n# ------------------------------------------------------------\n# 4) Copy masks from labels/HemSeg-500/{train,val,test}/label\n# ------------------------------------------------------------\nsplit_names = [\"train\", \"val\", \"test\"]\ncopy_rows = []\nseen_mask_names = set()\n\nfor split in split_names:\n    label_dir = LABELS_HEMSEG_DIR / split / \"label\"\n\n    if not label_dir.exists():\n        raise FileNotFoundError(f\"Missing expected label folder: {label_dir}\")\n\n    mask_files = list_niigz(label_dir)\n    print(f\"{split}: found {len(mask_files)} mask files\")\n\n    for src in mask_files:\n        if src.name in seen_mask_names:\n            raise RuntimeError(f\"Duplicate mask filename found across splits: {src.name}\")\n\n        dst = MASKS_DIR / src.name\n        shutil.copy2(src, dst)\n        seen_mask_names.add(src.name)\n\n        copy_rows.append({\n            \"split\": split,\n            \"mask_filename\": src.name,\n            \"mask_stem\": nii_stem(src),\n            \"src_path\": str(src),\n            \"dst_path\": str(dst),\n            \"status\": \"copied\",\n        })\n\ncopy_df = pd.DataFrame(copy_rows)\ncopy_df.to_csv(REPORT_CSV, index=False)\n\n# ------------------------------------------------------------\n# 5) Verify CT/mask pairing\n# ------------------------------------------------------------\nct_files = list_niigz(CT_NEW_DIR)\nmask_files = list_niigz(MASKS_DIR)\n\nct_stems = {nii_stem(p): p.name for p in ct_files}\nmask_stems = {nii_stem(p): p.name for p in mask_files}\n\nct_only = sorted(set(ct_stems) - set(mask_stems))\nmask_only = sorted(set(mask_stems) - set(ct_stems))\npaired = sorted(set(ct_stems) & set(mask_stems))\n\nsummary_df = pd.DataFrame([\n    {\"metric\": \"n_ct_files\", \"value\": len(ct_files)},\n    {\"metric\": \"n_mask_files\", \"value\": len(mask_files)},\n    {\"metric\": \"n_paired\", \"value\": len(paired)},\n    {\"metric\": \"n_ct_only\", \"value\": len(ct_only)},\n    {\"metric\": \"n_mask_only\", \"value\": len(mask_only)},\n])\n\nsummary_df.to_csv(SUMMARY_CSV, index=False)\n\nprint(\"\\nFinal verification:\")\nprint(\"CT files     :\", len(ct_files))\nprint(\"Mask files   :\", len(mask_files))\nprint(\"Paired files :\", len(paired))\nprint(\"CT only      :\", len(ct_only))\nprint(\"Mask only    :\", len(mask_only))\n\nprint(\"\\nSaved report :\", REPORT_CSV)\nprint(\"Saved summary:\", SUMMARY_CSV)\n\n# ------------------------------------------------------------\n# 6) If mismatched, save mismatch report and stop\n# ------------------------------------------------------------\nif len(ct_only) > 0 or len(mask_only) > 0 or len(ct_files) != len(mask_files):\n    mismatch_rows = []\n\n    for s in ct_only:\n        mismatch_rows.append({\n            \"stem\": s,\n            \"issue\": \"ct_without_mask\",\n            \"ct_filename\": ct_stems.get(s),\n            \"mask_filename\": None,\n        })\n\n    for s in mask_only:\n        mismatch_rows.append({\n            \"stem\": s,\n            \"issue\": \"mask_without_ct\",\n            \"ct_filename\": None,\n            \"mask_filename\": mask_stems.get(s),\n        })\n\n    pd.DataFrame(mismatch_rows).to_csv(MISMATCH_CSV, index=False)\n\n    raise RuntimeError(\n        f\"Pairing check failed. labels folder was NOT deleted.\\n\"\n        f\"Mismatch report saved to: {MISMATCH_CSV}\"\n    )\n\n# ------------------------------------------------------------\n# 7) Delete labels folder only after successful verification\n# ------------------------------------------------------------\nshutil.rmtree(LABELS_ROOT)\n\nprint(\"\\nPairing verified successfully.\")\nprint(f\"Deleted labels folder: {LABELS_ROOT}\")\n\nprint(\"\\nFinal structure:\")\nprint(f\"{HEMSEG_ROOT}/\")\nprint(\"├── ct_scans/\")\nprint(\"└── masks/\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# 11) OPTIONAL ZIP EXPORT\n# ============================================================\nif MAKE_FINAL_ZIP:\n    archive_path = shutil.make_archive(FINAL_ZIP_BASENAME, \"zip\", OUTPUT_ROOT)\n    print(f\"Created final archive: {archive_path}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%cd /kaggle/working\n\nimport os\nfrom IPython.display import FileLink, display\n\nfname = \"hemseg500.zip\"\nprint(\"exists:\", os.path.exists(fname))\nif os.path.exists(fname):\n    print(\"size_gb:\", round(os.path.getsize(fname) / (1024**3), 3))\n    display(FileLink(fname))\nelse:\n    print(\"File not found in /kaggle/working\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}