{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":99552,"databundleVersionId":13851420,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os, sys, gc, time, json, math, glob, shutil, tempfile, warnings, subprocess, random, ast\nfrom typing import List, Tuple, Dict, Any\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport cv2\nimport torch\nfrom tqdm import tqdm\n\ndef now():\n    return time.strftime(\"%Y-%m-%d %H:%M:%S\", time.localtime())\n\ndef log(msg):\n    print(f\"[{now()}] {msg}\", flush=True)\n\ndef disk_gb(path=\"/kaggle/working\") -> Tuple[float,float,float]:\n    total, used, free = shutil.disk_usage(path)\n    return tuple(x/(1024**3) for x in (total, used, free))\n\ndef rm_rf(path):\n    shutil.rmtree(path, ignore_errors=True)\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nos.environ[\"KAGGLE_USERNAME\"] = user_secrets.get_secret(\"KAGGLE_USERNAME\")\nos.environ[\"KAGGLE_KEY\"] = user_secrets.get_secret(\"KAGGLE_KEY\")\nlog(f\"KAGGLE_USERNAME visible? {bool(os.getenv('KAGGLE_USERNAME'))}\")\nlog(f\"KAGGLE_KEY visible? {bool(os.getenv('KAGGLE_KEY'))}\")\n\ncfg_dir = os.path.expanduser(\"~/.config/kaggle\")\nos.makedirs(cfg_dir, exist_ok=True)\nkj_path = os.path.join(cfg_dir, \"kaggle.json\")\nif not os.path.isfile(kj_path):\n    with open(kj_path, \"w\") as f:\n        json.dump({\"username\": os.getenv(\"KAGGLE_USERNAME\"), \"key\": os.getenv(\"KAGGLE_KEY\")}, f)\n    os.chmod(kj_path, 0o600)\n\ntry:\n    out = subprocess.run([\"kaggle\",\"--version\"], capture_output=True, text=True)\n    if out.returncode == 0:\n        log(out.stdout.strip())\nexcept Exception as e:\n    log(f\"[WARN] Kaggle CLI check failed: {e}\")\n\nDATA_ROOT = \"/kaggle/input/rsna-intracranial-aneurysm-detection\"\nSERIES_ROOT = f\"{DATA_ROOT}/series\"\nTRAIN_CSV = f\"{DATA_ROOT}/train.csv\"\nLOC_CSV = f\"{DATA_ROOT}/train_localizers.csv\"\nSEGM_DIR = f\"{DATA_ROOT}/segmentations\"\n\n\nOWNER = os.getenv(\"KAGGLE_USERNAME\")\nDATASET_PREFIX = \"rsna-preprocessed-v2-shard\"  # each shard gets -{shard_id:02d}\nUSE_TIMESTAMP_SUFFIX = False  # set True to make brand-new datasets each run\nTIMESTAMP = time.strftime(\"%Y%m%d%H%M%S\", time.gmtime())\n\ndef shard_dataset_ref(shard_id: int) -> str:\n    base = f\"{DATASET_PREFIX}-{shard_id:02d}\"\n    if USE_TIMESTAMP_SUFFIX:\n        base = f\"{base}-{TIMESTAMP}\"\n    return f\"{OWNER}/{base}\"\n\ndef ensure_shard_dataset(shard_id: int, out_dir: str, meta_note: str=\"\"):\n    ref = shard_dataset_ref(shard_id)\n    st = subprocess.run([\"kaggle\",\"datasets\",\"status\",\"-d\",ref], capture_output=True, text=True)\n    if st.returncode == 0:\n        log(f\"[OK] Shard dataset exists: https://www.kaggle.com/datasets/{ref}\")\n        return ref\n\n    os.makedirs(out_dir, exist_ok=True)\n    meta = {\n        \"title\": f\"RSNA Preprocessed v2 — Shard {shard_id:02d}\",\n        \"id\": ref,\n        \"licenses\":[{\"name\":\"CC0-1.0\"}],\n        \"subtitle\":\"K=9 float16 for all; full uint8 for positives + 20% negatives\",\n        \"description\": \"One dataset per shard to avoid cross-shard overwrites. \" + meta_note,\n    }\n    with open(os.path.join(out_dir, \"dataset-metadata.json\"), \"w\") as f:\n        json.dump(meta, f, indent=2)\n    # at least one file must exist\n    with open(os.path.join(out_dir, \"hello.txt\"), \"w\") as f:\n        f.write(f\"Shard {shard_id:02d} scaffold\")\n\n    cr = subprocess.run([\"kaggle\",\"datasets\",\"create\",\"-p\",out_dir,\"-r\",\"zip\",\"-q\"], capture_output=True, text=True)\n    if cr.returncode != 0:\n        log(\"Dataset creation error: \" + (cr.stderr or \"\").strip())\n    else:\n        log(f\"[OK] Created shard dataset: https://www.kaggle.com/datasets/{ref}\")\n    return ref\n\nos.environ[\"OMP_NUM_THREADS\"] = \"1\"\nos.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\"\nos.environ[\"MKL_NUM_THREADS\"] = \"1\"\nos.environ[\"VECLIB_MAXIMUM_THREADS\"] = \"1\"\nos.environ[\"NUMEXPR_NUM_THREADS\"] = \"1\"\ncv2.setNumThreads(0)\ntorch.set_num_threads(1)\nrandom.seed(42)\n\nSAVE_K_SLICES = True\nSAVE_FULL = True\n\nK_SLICES = 9\nIMG_SIZE = 224\nKS_DTYPE = \"float16\"   \n\nFULL_FOR_POSITIVES = True\nNEG_FULL_FRAC = 0.20  \nFULL_DTYPE = \"uint8\"\nFULL_KEEP_EVERY_NTH = 2  \n\nTARGET_FREE_HEADROOM_GB = 2.5   \nHARD_MAX_SERIES_PER_SHARD = 480\n\nPER_SERIES_K_MB_EST = 40.0      \nPER_SLICE_MB_FULL_U8 = (IMG_SIZE*IMG_SIZE*3) / (1024**2) \nTOTAL_SHARDS = 16\n\nAPPLY_WINDOWING = True\nWINDOW_MIN, WINDOW_MAX = -100.0, 300.0\nAPPLY_DENOISE = False  # optional wavelet\n\nLABEL_COLS = [\n    'Left Infraclinoid Internal Carotid Artery',\n    'Right Infraclinoid Internal Carotid Artery',\n    'Left Supraclinoid Internal Carotid Artery',\n    'Right Supraclinoid Internal Carotid Artery',\n    'Left Middle Cerebral Artery',\n    'Right Middle Cerebral Artery',\n    'Anterior Communicating Artery',\n    'Left Anterior Cerebral Artery',\n    'Right Anterior Cerebral Artery',\n    'Left Posterior Communicating Artery',\n    'Right Posterior Communicating Artery',\n    'Basilar Tip',\n    'Other Posterior Circulation',\n    'Aneurysm Present',\n]\n\ndef pick_k_indices(n: int, k: int) -> np.ndarray:\n    if n <= 0:\n        return np.zeros(k, dtype=int)\n    idxs = np.linspace(0, n-1, num=k)\n    return np.clip(np.round(idxs).astype(int), 0, n-1)\n\ndef normalize01(x: np.ndarray) -> np.ndarray:\n    mn, mx = float(x.min()), float(x.max())\n    if (mx - mn) < 1e-5:\n        return np.zeros_like(x, dtype=np.float32)\n    return ((x - mn)/((mx - mn)+1e-5)).astype(np.float32)\n\ndef dicom_to_gray(path: str) -> np.ndarray:\n    with warnings.catch_warnings():\n        warnings.simplefilter(\"ignore\")\n        try:\n            ds = pydicom.dcmread(path, force=True)\n        except Exception:\n            return np.zeros((512,512), dtype=np.float32)\n\n    try:\n        arr = ds.pixel_array.astype(np.float32)\n    except Exception:\n        return np.zeros((512,512), dtype=np.float32)\n\n    slope = float(getattr(ds, \"RescaleSlope\", 1.0) or 1.0)\n    intercept = float(getattr(ds, \"RescaleIntercept\", 0.0) or 0.0)\n    arr = arr * slope + intercept\n\n    if arr.ndim == 3:\n        if arr.shape[-1] in (3,4):  # RGB(A) → gray\n            if arr.shape[-1] == 4:\n                arr = arr[..., :3]\n            arr = cv2.cvtColor(arr.astype(np.float32), cv2.COLOR_RGB2GRAY)\n        else:  # [frames, H, W] → take middle slice\n            arr = arr[arr.shape[0]//2]\n\n    arr = np.squeeze(arr).astype(np.float32)\n    if arr.ndim != 2:\n        return np.zeros((512,512), dtype=np.float32)\n\n    if str(getattr(ds, \"PhotometricInterpretation\", \"\")).upper() == \"MONOCHROME1\":\n        arr = -arr\n\n    return arr\n\ndef apply_window(img: np.ndarray, wmin=WINDOW_MIN, wmax=WINDOW_MAX) -> np.ndarray:\n    return np.clip(img, wmin, wmax)\n\ndef to_tensor_3ch(img: np.ndarray, size=IMG_SIZE, out=\"float32\") -> torch.Tensor:\n    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)\n    t = torch.from_numpy(img).float().unsqueeze(0).expand(3,-1,-1).contiguous()\n    if out == \"float16\":\n        return t.half()\n    if out == \"uint8\":\n        return (t.clamp(0,1)*255).byte()\n    return t.float()\n\ndef estimate_full_mb(n_slices: int, keep_every_n: int = 1) -> float:\n    kept = max(1, (n_slices + keep_every_n - 1)//keep_every_n)\n    return kept * PER_SLICE_MB_FULL_U8\n\ndef parse_coord_xyf(cell: str):\n    try:\n        d = ast.literal_eval(cell)\n        if isinstance(d, dict):\n            return d.get('x', None), d.get('y', None), d.get('f', None)\n    except Exception:\n        pass\n    return None, None, None\n\ntrain_df = pd.read_csv(TRAIN_CSV).reset_index(drop=True)\n\ntrain_df = train_df.sample(frac=1.0, random_state=42).reset_index(drop=True)\n\ntotal_series = len(train_df)\nlog(f\"Planning: TOTAL_SHARDS={TOTAL_SHARDS}\")\nt,u,f = disk_gb(\"/kaggle/working\")\nlog(f\"Initial disk GB total/used/free = {t:.1f}/{u:.1f}/{f:.1f}\")\n\ndef initial_cap(free_gb: float, headroom_gb: float, per_series_mb: float, hard_max: int) -> int:\n    usable_gb = max(0.0, free_gb - headroom_gb)\n    if usable_gb <= 0:\n        return min(120, hard_max)\n    est_series = int((usable_gb*1024.0)/per_series_mb)\n    return max(120, min(hard_max, est_series))\n\nBASE_CAP = initial_cap(f, TARGET_FREE_HEADROOM_GB, PER_SERIES_K_MB_EST, HARD_MAX_SERIES_PER_SHARD)\n\nfor shard_id in range(TOTAL_SHARDS):\n    log(\"=\"*74)\n    log(f\"Starting shard {shard_id}/{TOTAL_SHARDS-1}\")\n    t,u,f = disk_gb(\"/kaggle/working\")\n    log(f\"Disk GB (total/used/free) = {t:.1f}/{u:.1f}/{f:.1f}\")\n\n    shard_idxs = np.arange(total_series)[shard_id::TOTAL_SHARDS]\n    shard_idxs = shard_idxs[:BASE_CAP]\n    shard_df = train_df.iloc[shard_idxs].reset_index(drop=True)\n\n    if 'Aneurysm Present' in shard_df.columns:\n        shard_df = (shard_df\n                    .assign(_y=shard_df['Aneurysm Present'].fillna(0).astype(int),\n                            _m=shard_df['Modality'].fillna('Other'))\n                    .sort_values(by=['_y','_m'])\n                    .drop(columns=['_y','_m'])\n                    .reset_index(drop=True))\n\n    log(f\"Series to process in this shard: {len(shard_df)} (base cap={BASE_CAP})\")\n\n    OUT_ROOT = f\"/kaggle/working/shard{shard_id:02d}\"\n    PRE_K = os.path.join(OUT_ROOT, \"pre_k\")\n    PRE_FULL = os.path.join(OUT_ROOT, \"pre_full\")\n    os.makedirs(OUT_ROOT, exist_ok=True)\n    if SAVE_K_SLICES: os.makedirs(PRE_K, exist_ok=True)\n    if SAVE_FULL   : os.makedirs(PRE_FULL, exist_ok=True)\n\n    meta_note = (f\"K={K_SLICES} {KS_DTYPE}; FULL={SAVE_FULL} {FULL_DTYPE} keep{FULL_KEEP_EVERY_NTH}; \"\n                 f\"IMG={IMG_SIZE}; NegFullFrac={NEG_FULL_FRAC}; Headroom={TARGET_FREE_HEADROOM_GB} GB\")\n    DATASET_REF = ensure_shard_dataset(shard_id, OUT_ROOT, meta_note=meta_note)\n\n    try:\n        if os.path.isfile(LOC_CSV):\n            loc_df = pd.read_csv(LOC_CSV)\n            loc_df.to_csv(os.path.join(OUT_ROOT, \"train_localizers_merged.csv\"), index=False)\n            log(\"Wrote train_localizers_merged.csv\")\n\n            try:\n                tmp = loc_df.copy()\n                xyf = tmp['coordinates'].apply(parse_coord_xyf)\n                tmp[['x','y','f']] = pd.DataFrame(xyf.tolist(), index=tmp.index)\n                loc_grp = (tmp.groupby('SeriesInstanceUID')\n                              .agg(n_localizers=('SOPInstanceUID','count'),\n                                   any_frame=('f', lambda x: x.notna().any()))\n                              .reset_index())\n                loc_grp.to_csv(os.path.join(OUT_ROOT, \"localizer_per_series.csv\"), index=False)\n                log(\"Wrote localizer_per_series.csv\")\n            except Exception as e:\n                log(f\"[WARN] localizer detail failed: {e}\")\n    except Exception as e:\n        log(f\"[WARN] localizers export failed: {e}\")\n\n    try:\n        seg_paths = sorted(glob.glob(os.path.join(SEGM_DIR, \"*.nii*\")))\n        pd.DataFrame({\"path\": seg_paths}).to_csv(os.path.join(OUT_ROOT, \"segmentations_index.csv\"), index=False)\n        log(f\"Wrote segmentations_index.csv ({len(seg_paths)} rows)\")\n    except Exception as e:\n        log(f\"[WARN] segmentations scan failed: {e}\")\n\n    rows = []\n    saved_full = 0\n    saved_k = 0\n    skipped_full_lowdisk = 0\n\n    pbar = tqdm(total=len(shard_df), desc=f\"Shard{shard_id} preprocessing\")\n    for _, r in shard_df.iterrows():\n        sid = str(r[\"SeriesInstanceUID\"])\n        sdir = os.path.join(SERIES_ROOT, sid)\n        label = int(r[\"Aneurysm Present\"]) if \"Aneurysm Present\" in r and pd.notna(r[\"Aneurysm Present\"]) else 0\n\n        if not os.path.isdir(sdir):\n            pbar.update(1); continue\n        files = sorted([f for f in os.listdir(sdir) if f.lower().endswith(\".dcm\")])\n        if not files:\n            pbar.update(1); continue\n\n        if SAVE_K_SLICES:\n            try:\n                idxs = pick_k_indices(len(files), K_SLICES)\n                k_imgs = []\n                for i in idxs:\n                    arr = dicom_to_gray(os.path.join(sdir, files[i]))\n                    if APPLY_WINDOWING: arr = apply_window(arr)\n                    if APPLY_DENOISE:\n                        from skimage.restoration import denoise_wavelet\n                        arr = denoise_wavelet(arr, method='BayesShrink', mode='soft', rescale_sigma=True).astype(np.float32)\n                    arr = normalize01(arr)\n                    k_imgs.append(to_tensor_3ch(arr, IMG_SIZE, out=\"float32\"))\n                k_vol = torch.stack(k_imgs, dim=0)\n                if KS_DTYPE == \"float16\":\n                    k_vol = k_vol.half()\n                elif KS_DTYPE == \"uint8\":\n                    k_vol = (k_vol.clamp(0,1)*255).byte()\n                torch.save(k_vol, os.path.join(PRE_K, f\"{sid}.pt\"))\n                saved_k += 1\n            except Exception as e:\n                log(f\"[WARN] K-slices failed for {sid}: {e}\")\n\n        want_full = SAVE_FULL and ((FULL_FOR_POSITIVES and label==1) or (random.random() < NEG_FULL_FRAC))\n        if want_full:\n            est_mb = estimate_full_mb(len(files), keep_every_n=FULL_KEEP_EVERY_NTH)\n            _,_,free_gb = disk_gb(\"/kaggle/working\")\n            if (free_gb*1024) - est_mb < TARGET_FREE_HEADROOM_GB*1024:\n                skipped_full_lowdisk += 1\n                pbar.set_postfix_str(\"skip-full:low-disk\")\n            else:\n                try:\n                    frames = []\n                    for idx, fname in enumerate(files):\n                        if (idx % FULL_KEEP_EVERY_NTH) != 0:\n                            continue\n                        arr = dicom_to_gray(os.path.join(sdir, fname))\n                        if APPLY_WINDOWING: arr = apply_window(arr)\n                        arr = normalize01(arr)\n                        frames.append(to_tensor_3ch(arr, IMG_SIZE, out=\"uint8\"))\n                    if frames:\n                        full_vol = torch.stack(frames, dim=0)\n                        torch.save(full_vol, os.path.join(PRE_FULL, f\"{sid}.pt\"))\n                        saved_full += 1\n                except OSError as e:\n                    log(f(\"[WARN] FULL OSError for {sid}: {e}\"))\n                except Exception as e:\n                    log(f\"[WARN] FULL failed for {sid}: {e}\")\n\n        slice_count = len(files)\n        rows.append(\n            [sid, slice_count, r.get(\"Modality\",\"\"), r.get(\"PatientAge\",\"\"), r.get(\"PatientSex\",\"\")]\n            + [r.get(c, 0) for c in LABEL_COLS]\n        )\n\n        pbar.update(1)\n    pbar.close()\n\n    log(f\"[STATS] shard {shard_id}: saved_k={saved_k}, saved_full={saved_full}, skipped_full_lowdisk={skipped_full_lowdisk}\")\n\n    if rows:\n        ci = pd.DataFrame(\n            rows,\n            columns=[\"SeriesInstanceUID\",\"slice_count\",\"Modality\",\"PatientAge\",\"PatientSex\"] + LABEL_COLS\n        )\n        ci.to_csv(os.path.join(OUT_ROOT,\"cache_index.csv\"), index=False)\n        log(f\"[OK] cache_index.csv written ({len(ci)} rows)\")\n    else:\n        log(f\"[WARN] No rows; cleaning and moving on.\")\n        rm_rf(OUT_ROOT); continue\n\n    with open(os.path.join(OUT_ROOT, \"dataset-metadata.json\"), \"w\") as fmeta:\n        json.dump({\n            \"title\": f\"RSNA Preprocessed v2 — Shard {shard_id:02d}\",\n            \"id\": DATASET_REF,\n            \"licenses\":[{\"name\":\"CC0-1.0\"}],\n            \"subtitle\": f\"K={K_SLICES} {KS_DTYPE} | FULL={SAVE_FULL} {FULL_DTYPE} keep{FULL_KEEP_EVERY_NTH} | IMG={IMG_SIZE}\",\n            \"description\": (\n                f\"Shard {shard_id:02d}. K-slices for all series ({KS_DTYPE}). \"\n                f\"Full volumes saved as {FULL_DTYPE} (windowed/normalized) for all positives \"\n                f\"and ~{int(NEG_FULL_FRAC*100)}% negatives, keeping every {FULL_KEEP_EVERY_NTH} slice, \"\n                f\"with disk-aware skipping when free space < {TARGET_FREE_HEADROOM_GB} GB. \"\n                f\"Includes rescale slope/intercept, MONOCHROME1 handling, richer per-shard index, \"\n                f\"and localizer per-series summary.\"\n            )\n        }, fmeta)\n\n    _,_,free_gb = disk_gb(\"/kaggle/working\")\n    log(f\"Pre-upload free disk: {free_gb:.1f} GB\")\n\n    res = subprocess.run(\n        [\"kaggle\",\"datasets\",\"version\",\"-p\",OUT_ROOT,\"-m\",f\"shard {shard_id:02d} update\",\"-r\",\"zip\"],\n        capture_output=True, text=True\n    )\n    log(f\"version rc={res.returncode}\\nstdout:\\n{res.stdout}\\nstderr:\\n{res.stderr}\")\n    if res.returncode != 0:\n        log(\"[WARN] Version failed, retry in 6s …\")\n        time.sleep(6)\n        res2 = subprocess.run(\n            [\"kaggle\",\"datasets\",\"version\",\"-p\",OUT_ROOT,\"-m\",f\"retry shard {shard_id:02d}\",\"-r\",\"zip\"],\n            capture_output=True, text=True\n        )\n        log(f\"retry rc={res2.returncode}\\nstdout:\\n{res2.stdout}\\nstderr:\\n{res2.stderr}\")\n        if res2.returncode != 0:\n            log(f\"[FATAL] Retry failed at shard {shard_id:02d}; leaving folder for inspection.\")\n            break\n        else:\n            log(\"[OK] Retry succeeded\")\n\n    # Clean local working dir to reclaim space between shards\n    rm_rf(OUT_ROOT)\n    t,u,f = disk_gb(\"/kaggle/working\")\n    log(f\"Post-clean disk GB total/used/free = {t:.1f}/{u:.1f}/{f:.1f}\")\n\nlog(\"=\"*74)\nlog(\"All shards done. Each shard has its own private dataset (separate IDs).\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-14T04:05:32.47246Z","iopub.execute_input":"2025-10-14T04:05:32.472791Z","iopub.status.idle":"2025-10-14T07:43:03.657253Z","shell.execute_reply.started":"2025-10-14T04:05:32.472769Z","shell.execute_reply":"2025-10-14T07:43:03.653968Z"}},"outputs":[],"execution_count":null}]}