{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"fd03c86a-cd6a-4016-90fe-43cab42e5aca","cell_type":"markdown","source":"# RSNA Knee Abnormality Detection - Hybrid DINO/Radiomics v8\n\nThis notebook is meant as a different direction from the submitted report-teacher/MRI-student v7.  The key changes are:\n\n- soft multilingual report supervision instead of hard pseudo-labels;\n- DINOv2 features from ordered 2.5D MRI views, with foreground cropping;\n- per-slot intensity/radiomics features added beside DINO embeddings;\n- conservative target-wise ensembling across logistic, ExtraTrees, HistGradientBoosting, and exact-label correction models;\n- no aggressive prediction inversion from tiny OOF folds.","metadata":{}},{"id":"2235bbb2-047c-490d-81ca-975e997bedf5","cell_type":"code","source":"import os, gc, re, json, math, random, warnings, unicodedata, zipfile\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom concurrent.futures import ThreadPoolExecutor\nfrom functools import lru_cache\nfrom copy import deepcopy\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\nfrom scipy.stats import rankdata\nfrom scipy.sparse import hstack\nfrom sklearn.decomposition import IncrementalPCA\nfrom sklearn.ensemble import ExtraTreesClassifier\nfrom sklearn.ensemble import HistGradientBoostingClassifier\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import balanced_accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pydicom\n\nwarnings.filterwarnings(\"ignore\")\ntorch.set_float32_matmul_precision(\"high\")\nif torch.cuda.is_available():\n    torch.backends.cudnn.benchmark = True\n\n\nclass CFG:\n    seed = 20260807\n    debug = False\n\n    img_size = 224\n    n_positions = 5\n    n_slots = 6\n    feature_batch_size = 64\n    study_batch_size = 6\n    dicom_workers = 8\n    cache_tag = \"v8_hybrid_dino224_6slot_5pos_radiomics\"\n    rebuild_features = False\n    rebuild_pca = False\n\n    pca_components = 192\n    pca_batch_size = 1024\n\n    text_cs = (0.35, 1.2, 4.0, 12.0)\n    lr_cs = (0.015, 0.05, 0.16, 0.5)\n    exact_lr_cs = (0.01, 0.03, 0.10, 0.30)\n    oof_folds = 4\n\n    exact_weight = 7.0\n    exact_correction_weight = 3.0\n    pseudo_min_score = 0.20\n    pseudo_soft_floor = 0.18\n    max_pseudo_per_class = 2200\n\n    default_family_weights = {\n        \"lr\": 0.46,\n        \"et\": 0.24,\n        \"hgb\": 0.18,\n        \"exact_lr\": 0.12,\n    }\n\n    competition_dir = None\n    dinov2_dir = None\n\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    amp = torch.cuda.is_available()\n    n_gpus = torch.cuda.device_count()\n\n\nTARGETS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\"\n]\nPLANES = [\"Sagittal\", \"Coronal\", \"Axial\"]\nCONTRASTS = [\"Fluid\", \"Structural\"]\nSLOT_NAMES = [f\"{plane}_{contrast}\" for plane in PLANES for contrast in CONTRASTS]\nSLOT_META_DIM = 8\nSTUDY_META_DIM = 13\nRAD_DIM = 14\nRAD_AGG_DIM = 4 * RAD_DIM\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n\n\ndef find_competition_dir():\n    if CFG.competition_dir:\n        path = Path(CFG.competition_dir)\n        if path.exists():\n            return path\n\n    preferred = [\n        Path(\"/kaggle/input/rsna-knee-abnormality-detection\"),\n        Path(\"/kaggle/input/competitions/rsna-knee-abnormality-detection\"),\n        Path(\"Data\"),\n        Path(\".\"),\n    ]\n    for path in preferred:\n        if all((path / name).exists() for name in [\"train.csv\", \"train_series.csv\", \"test.csv\", \"test_series.csv\", \"sample_submission.csv\"]):\n            return path\n\n    for root in [Path(\"/kaggle/input\"), Path(\".\")]:\n        if not root.exists():\n            continue\n        for train_csv in root.glob(\"**/train.csv\"):\n            path = train_csv.parent\n            if all((path / name).exists() for name in [\"train_series.csv\", \"test.csv\", \"test_series.csv\", \"sample_submission.csv\"]):\n                return path\n    raise FileNotFoundError(\"Could not find RSNA Knee competition CSV files.\")\n\n\ndef find_dinov2_dir():\n    if CFG.dinov2_dir:\n        path = Path(CFG.dinov2_dir)\n        if (path / \"config.json\").exists():\n            return path\n\n    preferred = [\n        Path(\"/kaggle/input/models/metaresearch/dinov2/pytorch/small/1\"),\n        Path(\"/kaggle/input/dinov2/pytorch/small/1\"),\n        Path(\"/kaggle/input/dinov2-small/pytorch/small/1\"),\n        Path(\"/kaggle/input/dinov2-small\"),\n    ]\n    for path in preferred:\n        if (path / \"config.json\").exists():\n            return path\n\n    candidates = []\n    root = Path(\"/kaggle/input\")\n    if root.exists():\n        for current, dirs, files in os.walk(root):\n            current_path = Path(current)\n            if any(part in {\"train_series\", \"test_series\"} for part in current_path.parts):\n                dirs[:] = []\n                continue\n            if \"config.json\" not in files:\n                continue\n            if not any((current_path / name).exists() for name in [\"model.safetensors\", \"pytorch_model.bin\"]):\n                continue\n            try:\n                cfg = json.loads((current_path / \"config.json\").read_text())\n            except Exception:\n                continue\n            hint = (str(cfg.get(\"model_type\", \"\")) + \" \" + str(current_path)).lower()\n            if \"dinov2\" in hint:\n                hidden = int(cfg.get(\"hidden_size\", 10000))\n                candidates.append((abs(hidden - 384), current_path))\n    if candidates:\n        return sorted(candidates, key=lambda x: x[0])[0][1]\n\n    raise FileNotFoundError(\n        \"DINOv2 weights were not found. Add the Kaggle model input \"\n        \"'Meta Research / DINOv2 / PyTorch / small / 1' before running.\"\n    )\n\n\nseed_everything(CFG.seed)\nBASE = find_competition_dir()\nWORK = Path(\"/kaggle/working\") if Path(\"/kaggle/working\").exists() else Path(\".\")\nWORK.mkdir(parents=True, exist_ok=True)\n\nprint(\"Competition directory:\", BASE)\nprint(\"Working directory:\", WORK.resolve())\nprint(\"Device:\", CFG.device)\nprint(\"GPUs:\", CFG.n_gpus, [torch.cuda.get_device_name(i) for i in range(CFG.n_gpus)])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:30:50.296597Z","iopub.execute_input":"2026-08-07T08:30:50.296875Z","iopub.status.idle":"2026-08-07T08:30:57.951054Z","shell.execute_reply.started":"2026-08-07T08:30:50.296844Z","shell.execute_reply":"2026-08-07T08:30:57.950193Z"}},"outputs":[],"execution_count":null},{"id":"5774e9c7-ecd5-401e-ac5a-410a232a8f57","cell_type":"code","source":"train = pd.read_csv(BASE / \"train.csv\")\ntrain_series = pd.read_csv(BASE / \"train_series.csv\")\ntest = pd.read_csv(BASE / \"test.csv\")\ntest_series = pd.read_csv(BASE / \"test_series.csv\")\nsample_submission = pd.read_csv(BASE / \"sample_submission.csv\")\n\nfor frame in [train, train_series, test, test_series, sample_submission]:\n    frame[\"StudyInstanceUID\"] = frame[\"StudyInstanceUID\"].astype(str)\n\nHAS_TEST_REPORT = \"Report\" in test.columns\nHAS_SEX = \"PatientSex\" in train.columns\n\nexact_mask = train[TARGETS].notna().values\nfully_labeled_mask = exact_mask.all(axis=1)\nexact_indices = np.where(exact_mask.any(axis=1))[0]\nexact_targets = train[TARGETS].fillna(0.5).values.astype(np.float32)\n\nIMAGE_AVAILABLE = (BASE / \"train_series\").exists() and (BASE / \"test_series\").exists()\nif CFG.debug:\n    train = train.iloc[:180].reset_index(drop=True)\n    keep = set(train[\"StudyInstanceUID\"])\n    train_series = train_series[train_series[\"StudyInstanceUID\"].isin(keep)].reset_index(drop=True)\n    exact_mask = train[TARGETS].notna().values\n    fully_labeled_mask = exact_mask.all(axis=1)\n    exact_indices = np.where(exact_mask.any(axis=1))[0]\n    exact_targets = train[TARGETS].fillna(0.5).values.astype(np.float32)\n\nprint(\"train:\", train.shape, \"test:\", test.shape)\nprint(\"train_series:\", train_series.shape, \"test_series:\", test_series.shape)\nprint(\"exact labeled studies:\", len(exact_indices))\nprint(\"DICOM folders available:\", IMAGE_AVAILABLE)\ndisplay(pd.DataFrame({\n    \"target\": TARGETS,\n    \"exact_count\": exact_mask.sum(axis=0),\n    \"exact_positive_rate\": train[TARGETS].mean(skipna=True).values,\n}))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:31:06.196657Z","iopub.execute_input":"2026-08-07T08:31:06.197742Z","iopub.status.idle":"2026-08-07T08:31:06.486963Z","shell.execute_reply.started":"2026-08-07T08:31:06.1977Z","shell.execute_reply":"2026-08-07T08:31:06.486332Z"}},"outputs":[],"execution_count":null},{"id":"bc5d432b-321f-4e0c-a596-3e24f9df0286","cell_type":"markdown","source":"## 1. Soft Multilingual Report Teacher\n\nThe report teacher is used only to create image-training supervision. Exact labels overwrite teacher labels for final training, but the pseudo-labels remain soft and confidence-weighted because report language and exact labels are both noisy in a few edge cases.\n","metadata":{}},{"id":"5e068ebe","cell_type":"code","source":"LATIN_TRANSLATION = str.maketrans({\n    \"\\u0131\": \"i\", \"\\u0130\": \"i\", \"\\u011f\": \"g\", \"\\u011e\": \"g\",\n    \"\\u015f\": \"s\", \"\\u015e\": \"s\", \"\\u00e7\": \"c\", \"\\u00c7\": \"c\",\n    \"\\u00f6\": \"o\", \"\\u00d6\": \"o\", \"\\u00fc\": \"u\", \"\\u00dc\": \"u\",\n    \"\\u00df\": \"ss\", \"\\u00f8\": \"o\", \"\\u00d8\": \"o\", \"\\u00e6\": \"ae\",\n    \"\\u00c6\": \"ae\",\n})\n\n\ndef normalize_text(text):\n    text = \"\" if pd.isna(text) else str(text)\n    text = text.translate(LATIN_TRANSLATION)\n    text = unicodedata.normalize(\"NFKD\", text)\n    text = \"\".join(ch for ch in text if not unicodedata.combining(ch))\n    text = text.lower()\n    replacements = {\n        \"\\u00e2\\u20ac\\u2122\": \"'\", \"\\u00e2\\u20ac\\u02dc\": \"'\",\n        \"\\u00e2\\u20ac\\u201c\": \"-\", \"\\u00e2\\u20ac\\u201d\": \"-\",\n        \"\\u00b5\": \"u\",\n        \"lca\": \"acl\", \"lcm\": \"mcl\", \"vkb\": \"acl\",\n    }\n    for old, new in replacements.items():\n        text = text.replace(old, new)\n    text = re.sub(r\"([a-z])([0-9])\", r\"\\1 \\2\", text)\n    text = re.sub(r\"([0-9])([a-z])\", r\"\\1 \\2\", text)\n    text = re.sub(r\"\\s+\", \" \", text)\n    return text.strip()\n\n\nNEGATION = re.compile(\n    r\"\\b(?:no|not|without|absent|absence of|negative for|free of|denies|\"\n    r\"sin|ausencia de|no se observa|no se evidencia|no hay|\"\n    r\"sem|ausencia|nao ha|nao evidencia|\"\n    r\"kein|keine|keinen|ohne|nicht nachweisbar|\"\n    r\"pas de|sans|absence de|aucun|aucune|\"\n    r\"senza|assenza di|non si osserva|\"\n    r\"geen|zonder|afwezig|normalmente|normal)\\b\"\n)\nUNCERTAIN = re.compile(\n    r\"\\b(?:possible|possibly|probable|probably|suspect|suspected|suggestive|\"\n    r\"cannot exclude|may represent|questionable|equivocal|favor|in favor|\"\n    r\"posible|probable|sugestivo|sospecha|no se descarta|\"\n    r\"possivel|provavel|suspeita|moglich|verdacht|suspecte|\"\n    r\"possibile|sospetto|olasilikla|olasi)\\b\"\n)\nNORMAL = re.compile(\n    r\"\\b(?:intact|normal|preserved|unremarkable|stable|continuous|\"\n    r\"no tear|not torn|without tear|\"\n    r\"integro|integros|conservado|conservados|intacto|intactos|preservado|\"\n    r\"intakt|unauffallig|erhalten|intacte|intacts|integri|conservato|\"\n    r\"korunmus|saglam)\\b\"\n)\n\nHIGH_GRADE = re.compile(\n    r\"\\b(?:complete|full thickness|full-thickness|grade ?(?:2|ii|3|iii)|\"\n    r\"high grade|high-grade|total|near complete|near-complete|\"\n    r\"espesor total|alto grado|grado ?(?:2|ii|3|iii)|\"\n    r\"komplett|vollstandig|complet|complete|tam kat)\\b\"\n)\nLOW_GRADE = re.compile(r\"\\b(?:grade ?1|grade ?i|low grade|low-grade|mild|leve|discreto|trace|minimal|grade i)\\b\")\n\nLIGAMENT_DAMAGE = re.compile(\n    r\"\\b(?:tear|torn|rupture|ruptured|sprain|injury|disruption|laxity|\"\n    r\"partial tear|attenuat|discontinu|thicken|edema|\"\n    r\"rotura|ruptura|desgarro|lesion|esguince|distension|\"\n    r\"rottura|lesione|dechirure|entorse|riss|zerrung|verletzung|scheur|\"\n    r\"yirtik|yaralanma)\\w*\"\n)\nMENISCUS_TEAR = re.compile(\n    r\"\\b(?:tear|torn|rupture|fissure|cleavage|radial|horizontal|vertical|complex|\"\n    r\"bucket.?handle|root tear|posterior root|surfacing|surface|extrusion|macerat|truncat|\"\n    r\"rotura|desgarro|fisura|lesion|extrusion|\"\n    r\"rottura|fissurazione|lesione|dechirure|einriss|scheur|yirtik|riss)\\w*\"\n)\nMENISCUS_NEG = re.compile(\n    r\"\\b(?:intrasubstance degeneration|degenerative signal|without surfacing|without extension to the articular surface|\"\n    r\"no frank tear|not torn|sin rotura|no rotura)\\b\"\n)\nOA_DAMAGE = re.compile(\n    r\"\\b(?:osteoarthrit|osteoarthros|arthros|gonarthros|degenerative joint|chondrosis|chondropathy|\"\n    r\"cartilage (?:loss|thinning|defect|fissur)|chondral (?:loss|defect|fissur)|\"\n    r\"joint space narrowing|chondromalacia|osteophyte|subchondral sclerosis|full thickness cartilage|\"\n    r\"artrosis|osteoartritis|gonartrosis|degeneracion condral|condropatia|ulcera condral|\"\n    r\"osteoartrose|artrose|arthrose|knorpelschaden|chondropathie|artrosi|gonartrosi|kraakbeen)\\w*\"\n)\n\nALIASES = {\n    \"ACL\": re.compile(r\"\\b(?:acl|anterior cruciate ligament|voorste kruisband|vorderes kreuzband|ligamento cruzado anterior|ligamento crociato anteriore|ligament croise anterieur|on capraz bag)\\b\"),\n    \"MCL\": re.compile(r\"\\b(?:mcl|medial collateral ligament|medial collateral|ligamento colateral medial|ligamento collaterale mediale|ligament collateral medial|mediales kollateralband|mediale band|ic yan bag)\\b\"),\n    \"Medial Meniscus\": re.compile(r\"\\b(?:medial menisc\\w*|menisc\\w* medial\\w*|menisco medial|menisco interno|menisque medial|innenmeniskus|menisco mediale|medyal meniskus)\\b\"),\n    \"Lateral Meniscus\": re.compile(r\"\\b(?:lateral menisc\\w*|menisc\\w* lateral\\w*|menisco lateral|menisco externo|menisque lateral|aussenmeniskus|menisco laterale|lateral meniskus)\\b\"),\n    \"Medial OA\": re.compile(r\"\\b(?:medial compartment|medial femorotibial|compartimento medial|compartimento interno|femorotibial medial|mediales kompartiment|compartiment medial|mediale femorotibiaal|medial tibiofemoral)\\b\"),\n    \"Lateral OA\": re.compile(r\"\\b(?:lateral compartment|lateral femorotibial|compartimento lateral|compartimento externo|femorotibial lateral|laterales kompartiment|compartiment lateral|laterale femorotibiaal|lateral tibiofemoral)\\b\"),\n    \"PF OA\": re.compile(r\"\\b(?:patellofemoral|patello.?femoral|femoropatellar|femoro.?patelar|femororotulian|retropatellar|femoropatellaire|femoropatellare|trochlea|trochlear|patellar facet|rotuliana|rotula)\\w*\"),\n    \"Effusion\": re.compile(r\"\\b(?:joint effusion|effusion|hydarthrosis|hydrops|derrame articular|derrame|liquido articular|erguss|gelenkerguss|epanchement|versamento articolare|efuzyon|sivi)\\b\"),\n    \"Synovitis\": re.compile(r\"\\b(?:synovitis|sinovitis|synovite|synovial (?:thickening|proliferation|inflammation|hypertrophy|tissue)|synoviale entzundung|sinovit)\\b\"),\n    \"Baker's\": re.compile(r\"\\b(?:baker'?s? cyst|popliteal cyst|popliteale cyste|quiste de baker|quiste popliteo|cisto de baker|kyste de baker|baker zyste|cisti di baker|popliteal kist)\\b\"),\n    \"Contusion\": re.compile(r\"\\b(?:bone contusion|bone bruise|marrow contusion|traumatic marrow edema|bone marrow edema|marrow edema|bone oedema|\"\n                            r\"contusion osea|contusao ossea|edema ose\\w*|edema de medula|contusion osseuse|oedeme osseux|knochenmarkodem|knochenkontusion|kemik iligi odemi|kontuzyon)\\b\"),\n    \"Fracture\": re.compile(r\"\\b(?:fracture|fractura|fratura|fraktur|frattura|insufficiency fracture|stress fracture|subchondral fracture|tibial plateau fracture|kirik)\\w*\"),\n}\n\nGROUP_NORMAL = {\n    \"ACL\": re.compile(r\"\\b(?:cruciate ligaments?|ligamentos cruzados|kreuzbander|chiasti)\\b\"),\n    \"MCL\": re.compile(r\"\\b(?:collateral ligaments?|ligamentos colaterales|kollateralbander|yan baglar)\\b\"),\n    \"Medial Meniscus\": re.compile(r\"\\b(?:menisci|meniscos|menisken|menisques|meniskusler)\\b\"),\n    \"Lateral Meniscus\": re.compile(r\"\\b(?:menisci|meniscos|menisken|menisques|meniskusler)\\b\"),\n}\n\nGLOBAL_NEGATIVE = {\n    \"Fracture\": re.compile(r\"\\b(?:no acute osseous abnormality|no acute bony abnormality|no fracture|no fracture is seen|without fracture)\\b\"),\n    \"Contusion\": re.compile(r\"\\b(?:normal marrow signal|no bone marrow edema|no marrow edema|no osseous contusion|no bone bruise)\\b\"),\n    \"Effusion\": re.compile(r\"\\b(?:no joint effusion|without joint effusion|no significant effusion|no knee effusion)\\b\"),\n    \"Synovitis\": re.compile(r\"\\b(?:no synovitis|without synovitis)\\b\"),\n    \"Baker's\": re.compile(r\"\\b(?:no baker'?s? cyst|no popliteal cyst|without baker'?s? cyst|no hay quistes popliteos)\\b\"),\n}\n\n\ndef split_report(text):\n    text = normalize_text(text)\n    pieces = re.split(r\"(?<=[\\.\\!\\?;])\\s+|\\n+|\\s+-\\s+\", text)\n    return [piece.strip() for piece in pieces if piece.strip()]\n\n\ndef context_window(sentence, start, end, radius=130):\n    return sentence[max(0, start - radius): min(len(sentence), end + radius)]\n\n\ndef impression_bonus(sentence_index, n_sentences, sentence):\n    bonus = 0.05 * sentence_index / max(1, n_sentences - 1)\n    if re.search(r\"\\b(?:impression|conclusion|impresion|besluit|conclusie|schlussfolgerung|impressao|conclusao)\\b\", sentence):\n        bonus += 0.04\n    return bonus\n\n\ndef evidence_for_target(sentence, target):\n    evidences = []\n    full_sentence = sentence\n\n    if target in (\"Medial OA\", \"Lateral OA\", \"PF OA\"):\n        if re.search(r\"\\b(?:tricompartmental|tri-compartmental|tricompartimental|all three compartments|gonartrose)\\b\", sentence) and OA_DAMAGE.search(sentence):\n            evidences.append((0.97, 0.86))\n\n    if target == \"Fracture\" and re.search(r\"\\b(?:osteochondral defect|osteochondral body|ocd fragment)\\b\", sentence):\n        evidences.append((0.42, 0.38))\n\n    for match in ALIASES[target].finditer(full_sentence):\n        window = context_window(full_sentence, match.start(), match.end())\n        neg = bool(NEGATION.search(window))\n        uncertain = bool(UNCERTAIN.search(window))\n        normal = bool(NORMAL.search(window))\n\n        if target in (\"ACL\", \"MCL\"):\n            damaged = bool(LIGAMENT_DAMAGE.search(window))\n            severe = bool(HIGH_GRADE.search(window))\n            low = bool(LOW_GRADE.search(window))\n            if neg or (normal and not damaged):\n                evidences.append((0.025, 0.93))\n            elif damaged and severe:\n                evidences.append((0.965, 0.86))\n            elif damaged and low:\n                evidences.append((0.58, 0.34))\n            elif damaged:\n                evidences.append((0.78 if uncertain else 0.90, 0.48 if uncertain else 0.65))\n\n        elif target in (\"Medial Meniscus\", \"Lateral Meniscus\"):\n            tear = bool(MENISCUS_TEAR.search(window))\n            soft_negative = bool(MENISCUS_NEG.search(window))\n            if neg or (normal and not tear) or soft_negative:\n                evidences.append((0.035, 0.90))\n            elif tear:\n                confidence = 0.72\n                if re.search(r\"\\b(?:root|radial|horizontal|vertical|complex|bucket|surfacing|surface|extrusion|macerat)\\w*\", window):\n                    confidence = 0.86\n                if uncertain:\n                    confidence *= 0.60\n                evidences.append((0.96 if not uncertain else 0.70, confidence))\n\n        elif target in (\"Medial OA\", \"Lateral OA\", \"PF OA\"):\n            oa = bool(OA_DAMAGE.search(window))\n            high = bool(HIGH_GRADE.search(window) or re.search(r\"\\b(?:grade ?(?:3|iii|4|iv)|full thickness|bone-on-bone|complete cartilage loss)\\b\", window))\n            if neg or (normal and not oa):\n                evidences.append((0.04, 0.82))\n            elif oa:\n                evidences.append((0.96 if high else 0.82, 0.82 if high else 0.58))\n\n        elif target == \"Effusion\":\n            if neg:\n                evidences.append((0.035, 0.91))\n            else:\n                large = re.search(r\"\\b(?:large|moderate|massive|marked|severe|moyen|grave|moderado|abundante)\\b\", window)\n                trace = re.search(r\"\\b(?:trace|minimal|small|tiny|leve|discreto)\\b\", window)\n                evidences.append((0.94 if large else 0.78 if trace else 0.88, 0.84 if large else 0.52 if trace else 0.70))\n\n        elif target == \"Synovitis\":\n            if neg:\n                evidences.append((0.035, 0.90))\n            else:\n                chronic = re.search(r\"\\b(?:thickening|hypertrophy|proliferation|chronic|compatible)\\b\", window)\n                evidences.append((0.94 if chronic else 0.78, 0.82 if chronic else 0.55))\n\n        elif target == \"Baker's\":\n            if neg:\n                evidences.append((0.025, 0.92))\n            else:\n                trace = re.search(r\"\\b(?:trace|tiny|minimal|small)\\b\", window)\n                evidences.append((0.76 if trace else 0.96, 0.50 if trace else 0.88))\n\n        elif target == \"Contusion\":\n            if neg:\n                evidences.append((0.035, 0.88))\n            else:\n                traumatic = re.search(r\"\\b(?:contusion|bruise|traumatic|pivot shift|kissing)\\b\", window)\n                fracture_nearby = ALIASES[\"Fracture\"].search(window)\n                confidence = 0.82 if traumatic else 0.54\n                if fracture_nearby and not traumatic:\n                    confidence *= 0.50\n                evidences.append((0.91 if traumatic else 0.72, confidence))\n\n        elif target == \"Fracture\":\n            if neg:\n                evidences.append((0.025, 0.91))\n            else:\n                excluded = re.search(r\"\\b(?:osteochondral defect|osteochondral body|ocd)\\b\", window)\n                confidence = 0.86\n                if excluded:\n                    confidence = 0.38\n                evidences.append((0.95, confidence))\n\n    if target in GROUP_NORMAL:\n        for match in GROUP_NORMAL[target].finditer(full_sentence):\n            window = context_window(full_sentence, match.start(), match.end())\n            if NEGATION.search(window) or NORMAL.search(window):\n                evidences.append((0.035, 0.78))\n\n    return evidences\n\n\ndef report_rule_scores(report):\n    sentences = split_report(report)\n    probs = np.full(len(TARGETS), 0.5, dtype=np.float32)\n    confs = np.zeros(len(TARGETS), dtype=np.float32)\n    full = \" \".join(sentences)\n\n    for j, target in enumerate(TARGETS):\n        candidates = []\n        if target in GLOBAL_NEGATIVE and GLOBAL_NEGATIVE[target].search(full):\n            candidates.append((0.90, 0.035))\n\n        for i, sentence in enumerate(sentences):\n            bonus = impression_bonus(i, len(sentences), sentence)\n            for prob, conf in evidence_for_target(sentence, target):\n                candidates.append((min(0.98, conf + bonus), prob))\n\n        if candidates:\n            candidates.sort(key=lambda x: (x[0] + (0.06 if x[1] > 0.5 else 0.0), abs(x[1] - 0.5)), reverse=True)\n            confs[j], probs[j] = candidates[0]\n    return probs, confs\n\n\ntrain_reports = train[\"Report\"].fillna(\"\").astype(str).tolist()\ntest_reports = test[\"Report\"].fillna(\"\").astype(str).tolist() if HAS_TEST_REPORT else [\"\"] * len(test)\nnormalized_train_reports = [normalize_text(x) for x in train_reports]\n\nrule_probs = np.zeros((len(train), len(TARGETS)), dtype=np.float32)\nrule_conf = np.zeros_like(rule_probs)\nfor i, report in enumerate(tqdm(train_reports, desc=\"Report rules\")):\n    rule_probs[i], rule_conf[i] = report_rule_scores(report)\n\n\ndef estimate_rule_reliability(excluded_indices=None):\n    excluded = np.zeros(len(train), dtype=bool)\n    if excluded_indices is not None:\n        excluded[np.asarray(excluded_indices, dtype=int)] = True\n    quality = np.full(len(TARGETS), 0.40, dtype=np.float32)\n    rows = []\n    for j, target in enumerate(TARGETS):\n        known = exact_mask[:, j] & (~excluded)\n        covered = known & (rule_conf[:, j] >= 0.45)\n        y = train.loc[covered, target].astype(int).values\n        p = (rule_probs[covered, j] >= 0.5).astype(int)\n        if len(y) >= 8 and len(np.unique(y)) == 2:\n            bacc = balanced_accuracy_score(y, p)\n            rel = np.clip((bacc - 0.50) / 0.34, 0.12, 1.00)\n        elif len(y) > 0:\n            bacc = float((y == p).mean())\n            rel = np.clip((bacc - 0.50) / 0.42, 0.12, 0.75)\n        else:\n            bacc = np.nan\n            rel = 0.32\n        quality[j] = rel\n        rows.append({\n            \"target\": target,\n            \"covered_exact\": int(covered.sum()),\n            \"rule_balanced_accuracy\": bacc,\n            \"rule_reliability\": rel,\n        })\n    return quality, pd.DataFrame(rows)\n\n\nrule_reliability, rule_diag = estimate_rule_reliability()\ndisplay(rule_diag)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:31:13.255413Z","iopub.execute_input":"2026-08-07T08:31:13.256216Z","iopub.status.idle":"2026-08-07T08:31:22.531348Z","shell.execute_reply.started":"2026-08-07T08:31:13.256174Z","shell.execute_reply":"2026-08-07T08:31:22.530442Z"}},"outputs":[],"execution_count":null},{"id":"a957e80a-2e72-4508-a6d5-ddc524208292","cell_type":"code","source":"char_vectorizer = TfidfVectorizer(\n    analyzer=\"char_wb\",\n    ngram_range=(3, 6),\n    min_df=2,\n    max_features=90_000,\n    sublinear_tf=True,\n    strip_accents=\"unicode\",\n    dtype=np.float32,\n)\nword_vectorizer = TfidfVectorizer(\n    analyzer=\"word\",\n    ngram_range=(1, 3),\n    min_df=1,\n    max_features=45_000,\n    sublinear_tf=True,\n    strip_accents=\"unicode\",\n    dtype=np.float32,\n)\nX_char = char_vectorizer.fit_transform(normalized_train_reports)\nX_word = word_vectorizer.fit_transform(normalized_train_reports)\nX_text = hstack([X_char, X_word], format=\"csr\")\nprint(\"Text matrix:\", X_text.shape)\n\n\ndef fit_lr_average_sparse(X_fit, y_fit, X_pred, sample_weight=None, cs=CFG.text_cs, seed=0):\n    y_fit = np.asarray(y_fit, dtype=int)\n    if len(y_fit) == 0:\n        return np.full(X_pred.shape[0], 0.5, dtype=np.float32)\n    if len(np.unique(y_fit)) < 2:\n        return np.full(X_pred.shape[0], float(y_fit.mean()), dtype=np.float32)\n    preds = []\n    for k, c_value in enumerate(cs):\n        model = LogisticRegression(\n            C=c_value,\n            solver=\"liblinear\",\n            class_weight=\"balanced\",\n            max_iter=2500,\n            random_state=seed + 97 * k,\n        )\n        model.fit(X_fit, y_fit, sample_weight=sample_weight)\n        preds.append(model.predict_proba(X_pred)[:, 1])\n    return np.mean(preds, axis=0).astype(np.float32)\n\n\ndef build_text_predictions(target_index, excluded_indices=None, seed=0):\n    target = TARGETS[target_index]\n    excluded = np.zeros(len(train), dtype=bool)\n    if excluded_indices is not None:\n        excluded[np.asarray(excluded_indices, dtype=int)] = True\n\n    known = exact_mask[:, target_index] & (~excluded)\n    exact_idx = np.where(known)[0]\n    exact_y = train.loc[exact_idx, target].astype(int).values\n\n    rule_score = rule_conf[:, target_index] * (2.0 * np.abs(rule_probs[:, target_index] - 0.5))\n    pseudo_pool = (~exact_mask[:, target_index]) & (~excluded)\n    pseudo_idx = np.where(\n        pseudo_pool\n        & (rule_score >= 0.42)\n        & (rule_conf[:, target_index] >= 0.70)\n        & (np.abs(rule_probs[:, target_index] - 0.5) >= 0.28)\n    )[0]\n    pseudo_y = (rule_probs[pseudo_idx, target_index] >= 0.5).astype(int)\n\n    fit_idx = np.concatenate([exact_idx, pseudo_idx])\n    fit_y = np.concatenate([exact_y, pseudo_y])\n    weights = np.concatenate([\n        np.ones(len(exact_idx), dtype=np.float32),\n        np.full(len(pseudo_idx), 0.32 * rule_reliability[target_index], dtype=np.float32),\n    ])\n\n    return fit_lr_average_sparse(\n        X_text[fit_idx],\n        fit_y,\n        X_text,\n        sample_weight=weights,\n        cs=CFG.text_cs,\n        seed=seed,\n    )\n\n\ntext_oof = np.full((len(train), len(TARGETS)), np.nan, dtype=np.float32)\ntext_auc = np.full(len(TARGETS), np.nan, dtype=np.float32)\n\nfor j, target in enumerate(TARGETS):\n    idx = np.where(exact_mask[:, j])[0]\n    y = train.loc[idx, target].astype(int).values\n    minority = int(np.bincount(y).min()) if len(np.unique(y)) == 2 else 0\n    n_splits = min(CFG.oof_folds, minority)\n    if n_splits >= 2:\n        skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=CFG.seed + j)\n        for fold, (_, va_local) in enumerate(skf.split(idx, y)):\n            va_idx = idx[va_local]\n            pred_all = build_text_predictions(j, excluded_indices=va_idx, seed=CFG.seed + 200 * j + fold)\n            text_oof[va_idx, j] = pred_all[va_idx]\n        valid = np.isfinite(text_oof[idx, j])\n        if valid.sum() and len(np.unique(y[valid])) == 2:\n            text_auc[j] = roc_auc_score(y[valid], text_oof[idx[valid], j])\n\ntext_reliability = np.asarray([\n    np.clip((auc - 0.50) / 0.28, 0.0, 1.0) if np.isfinite(auc) else 0.0\n    for auc in text_auc\n], dtype=np.float32)\n\ndisplay(pd.DataFrame({\n    \"target\": TARGETS,\n    \"text_oof_auc\": text_auc,\n    \"text_reliability\": text_reliability,\n}))\n\n\ndef make_teacher_for_target(target_index, excluded_indices=None, overwrite_exact=True):\n    excluded = np.zeros(len(train), dtype=bool)\n    if excluded_indices is not None:\n        excluded[np.asarray(excluded_indices, dtype=int)] = True\n\n    local_rule_reliability, _ = estimate_rule_reliability(excluded_indices=excluded_indices)\n    text_probs = build_text_predictions(\n        target_index,\n        excluded_indices=excluded_indices,\n        seed=CFG.seed + 2000 + target_index,\n    )\n\n    rule_strength = rule_conf[:, target_index] * local_rule_reliability[target_index]\n    text_strength = (2.0 * np.abs(text_probs - 0.5)) * text_reliability[target_index]\n    denom = rule_strength + 0.72 * text_strength\n    teacher = np.full(len(train), 0.5, dtype=np.float32)\n    valid = denom > 1e-8\n    teacher[valid] = (\n        rule_strength[valid] * rule_probs[valid, target_index]\n        + 0.72 * text_strength[valid] * text_probs[valid]\n    ) / denom[valid]\n    teacher_conf = np.maximum(rule_strength, text_strength).astype(np.float32)\n\n    if overwrite_exact:\n        use_exact = exact_mask[:, target_index] & (~excluded)\n        teacher[use_exact] = exact_targets[use_exact, target_index]\n        teacher_conf[use_exact] = 1.0\n\n    teacher = np.clip(teacher, 1e-4, 1.0 - 1e-4).astype(np.float32)\n    teacher_conf = np.clip(teacher_conf, 0.0, 1.0).astype(np.float32)\n    return teacher, teacher_conf\n\n\ndef make_teacher(excluded_indices=None, overwrite_exact=True):\n    teacher = np.zeros((len(train), len(TARGETS)), dtype=np.float32)\n    teacher_conf = np.zeros_like(teacher)\n    for j in tqdm(range(len(TARGETS)), desc=\"Text teacher\", leave=False):\n        teacher[:, j], teacher_conf[:, j] = make_teacher_for_target(\n            j,\n            excluded_indices=excluded_indices,\n            overwrite_exact=overwrite_exact,\n        )\n    return teacher, teacher_conf\n\n\nteacher_train, teacher_conf_train = make_teacher(excluded_indices=None, overwrite_exact=True)\ndisplay(pd.DataFrame({\n    \"target\": TARGETS,\n    \"teacher_mean\": teacher_train.mean(axis=0),\n    \"mean_confidence\": teacher_conf_train.mean(axis=0),\n    \"selected_pseudo_rate\": ((teacher_conf_train * (2 * np.abs(teacher_train - 0.5))) >= CFG.pseudo_min_score).mean(axis=0),\n}))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:31:24.858277Z","iopub.execute_input":"2026-08-07T08:31:24.858912Z","iopub.status.idle":"2026-08-07T08:34:40.079752Z","shell.execute_reply.started":"2026-08-07T08:31:24.858882Z","shell.execute_reply":"2026-08-07T08:34:40.079178Z"}},"outputs":[],"execution_count":null},{"id":"5eef316a-1198-44e7-a1cb-cbe20f1c90f1","cell_type":"markdown","source":"## 2. MRI Slot Selection and DICOM Loading\n\nEach study is represented by six ordered slots: fluid-sensitive and structural acquisitions for sagittal, coronal, and axial planes.  Each selected series contributes five 2.5D views sampled across the slice stack.\n","metadata":{}},{"id":"8118f355-a9cf-4d15-bb28-54ce48ecba6c","cell_type":"code","source":"def numeric(value, default=0.0):\n    try:\n        value = float(value)\n        return value if np.isfinite(value) else default\n    except Exception:\n        return default\n\n\ndef sex_to_id(row):\n    value = str(row.get(\"PatientSex\", \"\")).strip().lower()\n    if value.startswith(\"m\"):\n        return 1\n    if value.startswith(\"f\"):\n        return 2\n    return 0\n\n\ndef choose_series(part, contrast, used_ids):\n    if len(part) == 0:\n        return None\n    fluid = part[\"Fluid_Sensitive\"].fillna(0).astype(float)\n    fat = part[\"Fat_Suppression\"].fillna(0).astype(float)\n    if contrast == \"Fluid\":\n        score = 4.0 * fluid + 2.0 * fat\n    else:\n        score = 3.5 * (1.0 - fluid) + 1.5 * (1.0 - fat)\n    ordered = part.assign(_slot_score=score).sort_values(\"_slot_score\", ascending=False)\n    for _, row in ordered.iterrows():\n        series_id = str(row[\"SeriesInstanceUID\"])\n        if series_id not in used_ids:\n            return row\n    return ordered.iloc[0]\n\n\ndef build_slots(df_series):\n    slots = {}\n    study_meta = {}\n    for study_id, rows in tqdm(\n        df_series.groupby(\"StudyInstanceUID\", sort=False),\n        total=df_series[\"StudyInstanceUID\"].nunique(),\n        desc=\"Selecting MRI slots\",\n    ):\n        study_id = str(study_id)\n        selected = {}\n        plane_lower = rows[\"Anatomical_Plane\"].astype(str).str.lower()\n        meta = []\n        for plane in PLANES:\n            part = rows[plane_lower == plane.lower()]\n            count = len(part)\n            fluid_mean = part[\"Fluid_Sensitive\"].fillna(0).astype(float).mean() if count else 0.0\n            fat_mean = part[\"Fat_Suppression\"].fillna(0).astype(float).mean() if count else 0.0\n            meta.extend([np.log1p(count) / 3.0, fluid_mean, fat_mean])\n            used = set()\n            for contrast in CONTRASTS:\n                row = choose_series(part, contrast, used)\n                if row is None:\n                    continue\n                sid = str(row[\"SeriesInstanceUID\"])\n                used.add(sid)\n                selected[f\"{plane}_{contrast}\"] = {\n                    \"series_id\": sid,\n                    \"plane\": plane,\n                    \"contrast\": contrast,\n                    \"fluid\": numeric(row.get(\"Fluid_Sensitive\", 0)),\n                    \"fat\": numeric(row.get(\"Fat_Suppression\", 0)),\n                }\n        total = len(rows)\n        meta.extend([\n            np.log1p(total) / 4.0,\n            rows[\"Fluid_Sensitive\"].fillna(0).astype(float).mean() if total else 0.0,\n            rows[\"Fat_Suppression\"].fillna(0).astype(float).mean() if total else 0.0,\n            rows[\"SeriesInstanceUID\"].nunique() / 12.0 if total else 0.0,\n        ])\n        slots[study_id] = selected\n        study_meta[study_id] = np.asarray(meta, dtype=np.float32)\n    return slots, study_meta\n\n\ntrain_slots, train_study_meta = build_slots(train_series)\ntest_slots, test_study_meta = build_slots(test_series)\ndisplay(pd.DataFrame({\n    \"slot\": SLOT_NAMES,\n    \"train_count\": [sum(slot in values for values in train_slots.values()) for slot in SLOT_NAMES],\n    \"test_count\": [sum(slot in values for values in test_slots.values()) for slot in SLOT_NAMES],\n}))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:34:40.080926Z","iopub.execute_input":"2026-08-07T08:34:40.081216Z","iopub.status.idle":"2026-08-07T08:35:28.473175Z","shell.execute_reply.started":"2026-08-07T08:34:40.081187Z","shell.execute_reply":"2026-08-07T08:35:28.470736Z"}},"outputs":[],"execution_count":null},{"id":"ad41dadf-93eb-424a-a414-1812e63aae90","cell_type":"code","source":"try:\n    import cv2\nexcept Exception:\n    cv2 = None\n\ntry:\n    from PIL import Image\nexcept Exception:\n    Image = None\n\n\ndef read_dicom_header(path):\n    try:\n        return pydicom.dcmread(str(path), stop_before_pixels=True, force=True)\n    except Exception:\n        return None\n\n\ndef header_position_value(ds):\n    if ds is None:\n        return None\n    try:\n        ipp = np.asarray([float(x) for x in ds.ImagePositionPatient], dtype=np.float64)\n        iop = np.asarray([float(x) for x in ds.ImageOrientationPatient], dtype=np.float64)\n        row = iop[:3]\n        col = iop[3:]\n        normal = np.cross(row, col)\n        return float(np.dot(ipp, normal))\n    except Exception:\n        pass\n    for name in [\"SliceLocation\", \"InstanceNumber\"]:\n        try:\n            return float(getattr(ds, name))\n        except Exception:\n            continue\n    return None\n\n\n@lru_cache(maxsize=8192)\ndef ordered_series_files(folder_str):\n    folder = Path(folder_str)\n    files = sorted(folder.glob(\"*.dcm\"))\n    if not files:\n        return tuple()\n\n    keyed = []\n    ok = 0\n    for fallback_index, path in enumerate(files):\n        ds = read_dicom_header(path)\n        key = header_position_value(ds)\n        if key is None:\n            key = fallback_index\n        else:\n            ok += 1\n        keyed.append((key, str(path)))\n    if ok >= max(3, len(files) // 3):\n        keyed.sort(key=lambda x: x[0])\n    return tuple(path for _, path in keyed)\n\n\ndef dicom_spacing(ds):\n    spacing_x, spacing_y, thickness = 0.0, 0.0, 0.0\n    try:\n        ps = [float(x) for x in ds.PixelSpacing]\n        spacing_y, spacing_x = ps[0], ps[1]\n    except Exception:\n        pass\n    for name in [\"SliceThickness\", \"SpacingBetweenSlices\"]:\n        try:\n            thickness = float(getattr(ds, name))\n            break\n        except Exception:\n            continue\n    return spacing_x, spacing_y, thickness\n\n\ndef read_pixel_float(path):\n    ds = pydicom.dcmread(str(path), force=True)\n    arr = ds.pixel_array.astype(np.float32)\n    slope = numeric(getattr(ds, \"RescaleSlope\", 1.0), 1.0)\n    intercept = numeric(getattr(ds, \"RescaleIntercept\", 0.0), 0.0)\n    arr = arr * slope + intercept\n    if str(getattr(ds, \"PhotometricInterpretation\", \"\")).upper() == \"MONOCHROME1\":\n        arr = arr.max() - arr\n    return arr, ds\n\n\ndef robust_uint8(stack):\n    stack = np.asarray(stack, dtype=np.float32)\n    finite = stack[np.isfinite(stack)]\n    if finite.size == 0:\n        return np.zeros(stack.shape, dtype=np.uint8)\n    low, high = np.percentile(finite, [1.0, 99.4])\n    if high <= low:\n        low, high = float(finite.min()), float(finite.max())\n    if high <= low:\n        return np.zeros(stack.shape, dtype=np.uint8)\n    stack = np.clip((stack - low) / (high - low), 0.0, 1.0)\n    return (255.0 * stack).astype(np.uint8)\n\n\ndef crop_foreground(image):\n    gray = image.max(axis=2)\n    mask = gray > max(8, np.percentile(gray, 55) * 0.18)\n    if mask.sum() < 64:\n        return image\n    ys, xs = np.where(mask)\n    y0, y1 = int(ys.min()), int(ys.max()) + 1\n    x0, x1 = int(xs.min()), int(xs.max()) + 1\n    pad_y = int(0.08 * (y1 - y0 + 1))\n    pad_x = int(0.08 * (x1 - x0 + 1))\n    y0 = max(0, y0 - pad_y)\n    y1 = min(image.shape[0], y1 + pad_y)\n    x0 = max(0, x0 - pad_x)\n    x1 = min(image.shape[1], x1 + pad_x)\n    if (y1 - y0) < 32 or (x1 - x0) < 32:\n        return image\n    return image[y0:y1, x0:x1]\n\n\ndef resize_image(image, size):\n    if cv2 is not None:\n        return cv2.resize(image, (size, size), interpolation=cv2.INTER_AREA)\n    if Image is not None:\n        return np.asarray(Image.fromarray(image).resize((size, size), resample=Image.BILINEAR))\n    y_idx = np.linspace(0, image.shape[0] - 1, size).astype(int)\n    x_idx = np.linspace(0, image.shape[1] - 1, size).astype(int)\n    return image[y_idx][:, x_idx]\n\n\ndef view_radiomics(image):\n    gray = image.astype(np.float32).mean(axis=2) / 255.0\n    h, w = gray.shape\n    q = np.percentile(gray, [1, 5, 10, 25, 50, 75, 90, 95, 99])\n    cy0, cy1 = h // 4, 3 * h // 4\n    cx0, cx1 = w // 4, 3 * w // 4\n    center = gray[cy0:cy1, cx0:cx1]\n    gy, gx = np.gradient(gray)\n    grad = np.sqrt(gx * gx + gy * gy)\n    foreground = gray > 0.08\n    return np.asarray([\n        gray.mean(), gray.std(), q[0], q[2], q[4], q[6], q[8],\n        center.mean(), center.std(),\n        grad.mean(), grad.std(),\n        foreground.mean(),\n        gray[foreground].mean() if foreground.any() else 0.0,\n        gray[foreground].std() if foreground.any() else 0.0,\n    ], dtype=np.float32)\n\n\ndef make_2p5d(paths):\n    arrays = []\n    first_ds = None\n    for path in paths:\n        try:\n            arr, ds = read_pixel_float(path)\n            if first_ds is None:\n                first_ds = ds\n            arrays.append(arr)\n        except Exception:\n            return None, np.zeros(RAD_DIM, dtype=np.float32), (0.0, 0.0, 0.0)\n    stack = robust_uint8(np.stack(arrays, axis=0))\n    image = np.transpose(stack, (1, 2, 0))\n    image = crop_foreground(image)\n    image = resize_image(image, CFG.img_size)\n    rad = view_radiomics(image)\n    spacing = dicom_spacing(first_ds) if first_ds is not None else (0.0, 0.0, 0.0)\n    return np.transpose(image, (2, 0, 1)), rad, spacing\n\n\ndef sampled_triplets(folder, n_positions):\n    files = list(ordered_series_files(str(folder)))\n    n = len(files)\n    if n == 0:\n        return [], 0\n    centers = np.round(np.linspace(0.08 * (n - 1), 0.92 * (n - 1), n_positions)).astype(int)\n    centers = np.clip(centers, 0, n - 1)\n    triplets = []\n    for center in centers:\n        indices = [max(0, center - 1), center, min(n - 1, center + 1)]\n        triplets.append([files[i] for i in indices])\n    return triplets, n\n\n\nclass StudySampler:\n    def __init__(self, studies, slots, study_meta, series_root):\n        self.studies = studies.reset_index(drop=True)\n        self.slots = slots\n        self.study_meta = study_meta\n        self.series_root = Path(series_root)\n\n    def __len__(self):\n        return len(self.studies)\n\n    def get(self, index):\n        row = self.studies.iloc[index]\n        study_id = str(row[\"StudyInstanceUID\"])\n        images = np.zeros((CFG.n_slots, CFG.n_positions, 3, CFG.img_size, CFG.img_size), dtype=np.uint8)\n        view_mask = np.zeros((CFG.n_slots, CFG.n_positions), dtype=bool)\n        slot_meta = np.zeros((CFG.n_slots, SLOT_META_DIM), dtype=np.float32)\n        radiomics = np.zeros((CFG.n_slots, RAD_AGG_DIM), dtype=np.float32)\n\n        selected = self.slots.get(study_id, {})\n        for slot_index, slot_name in enumerate(SLOT_NAMES):\n            info = selected.get(slot_name)\n            if info is None:\n                continue\n            folder = self.series_root / study_id / str(info[\"series_id\"])\n            triplets, n_files = sampled_triplets(folder, CFG.n_positions)\n            rad_values = []\n            spacings = []\n            for pos_index, paths in enumerate(triplets[:CFG.n_positions]):\n                image, rad, spacing = make_2p5d(paths)\n                if image is None:\n                    continue\n                images[slot_index, pos_index] = image\n                view_mask[slot_index, pos_index] = True\n                rad_values.append(rad)\n                spacings.append(spacing)\n\n            if rad_values:\n                rad_arr = np.stack(rad_values).astype(np.float32)\n                radiomics[slot_index] = np.concatenate([\n                    rad_arr.mean(axis=0),\n                    rad_arr.std(axis=0),\n                    rad_arr.min(axis=0),\n                    rad_arr.max(axis=0),\n                ]).astype(np.float32)\n                spacing_arr = np.asarray(spacings, dtype=np.float32)\n                spacing_mean = spacing_arr.mean(axis=0)\n            else:\n                spacing_mean = np.zeros(3, dtype=np.float32)\n\n            slot_meta[slot_index] = np.asarray([\n                info[\"fluid\"],\n                info[\"fat\"],\n                1.0 if info[\"contrast\"] == \"Structural\" else 0.0,\n                np.log1p(n_files) / 6.0,\n                float(view_mask[slot_index].mean()),\n                spacing_mean[0] / 2.5 if spacing_mean[0] else 0.0,\n                spacing_mean[1] / 2.5 if spacing_mean[1] else 0.0,\n                spacing_mean[2] / 8.0 if spacing_mean[2] else 0.0,\n            ], dtype=np.float32)\n\n        return (\n            images,\n            view_mask,\n            slot_meta,\n            radiomics,\n            self.study_meta.get(study_id, np.zeros(STUDY_META_DIM, dtype=np.float32)),\n            sex_to_id(row),\n            study_id,\n        )\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:35:28.476662Z","iopub.execute_input":"2026-08-07T08:35:28.477042Z","iopub.status.idle":"2026-08-07T08:35:28.834424Z","shell.execute_reply.started":"2026-08-07T08:35:28.476969Z","shell.execute_reply":"2026-08-07T08:35:28.833487Z"}},"outputs":[],"execution_count":null},{"id":"7a1729f1-324e-4979-a81e-230fef82fe9e","cell_type":"markdown","source":"## 3. Frozen DINOv2 Features plus Radiomics\n\nDINOv2 is loaded from a Kaggle model input.  The image transformer is frozen; only compact downstream classifiers are trained in this notebook.\n","metadata":{}},{"id":"4e97785d-1576-4056-a6d0-1c221f6416df","cell_type":"code","source":"from transformers import AutoModel\n\n\nclass DinoEncoder(nn.Module):\n    def __init__(self, backbone):\n        super().__init__()\n        self.backbone = backbone\n\n    def forward(self, pixel_values):\n        output = self.backbone(pixel_values=pixel_values)\n        tokens = output.last_hidden_state\n        patches = tokens[:, 1:]\n        cls = F.normalize(tokens[:, 0], dim=1)\n        mean = F.normalize(patches.mean(dim=1), dim=1)\n        max_tokens = F.normalize(patches.amax(dim=1), dim=1)\n        return torch.cat([cls, mean, max_tokens], dim=1)\n\n\ndef load_dino_model():\n    model_dir = find_dinov2_dir()\n    backbone = AutoModel.from_pretrained(str(model_dir), local_files_only=True, trust_remote_code=False)\n    for parameter in backbone.parameters():\n        parameter.requires_grad = False\n    hidden = int(backbone.config.hidden_size)\n    model = DinoEncoder(backbone).eval().to(CFG.device)\n    if CFG.n_gpus > 1:\n        model = nn.DataParallel(model, device_ids=list(range(CFG.n_gpus)))\n    print(\"DINOv2:\", model_dir)\n    print(\"view feature dimension:\", 3 * hidden)\n    return model, 3 * hidden\n\n\n@torch.inference_mode()\ndef encode_views(model, images):\n    if len(images) == 0:\n        return torch.empty((0, 0), dtype=torch.float32)\n    mean = torch.tensor([0.485, 0.456, 0.406], device=CFG.device).view(1, 3, 1, 1)\n    std = torch.tensor([0.229, 0.224, 0.225], device=CFG.device).view(1, 3, 1, 1)\n    outputs = []\n    for start in range(0, len(images), CFG.feature_batch_size):\n        batch = images[start:start + CFG.feature_batch_size].to(CFG.device, non_blocking=True).float().div_(255.0)\n        batch = (batch - mean) / std\n        with torch.autocast(\"cuda\", dtype=torch.float16, enabled=CFG.amp):\n            features = model(batch)\n        outputs.append(features.float().cpu())\n    return torch.cat(outputs, dim=0)\n\n\ndef create_dummy_feature_files(studies, slots, study_meta, prefix):\n    stem = f\"{prefix}_{CFG.cache_tag}_dummy\"\n    feature_path = WORK / f\"{stem}_slot_features.npy\"\n    mask_path = WORK / f\"{stem}_slot_mask.npy\"\n    slot_meta_path = WORK / f\"{stem}_slot_meta.npy\"\n    radiomics_path = WORK / f\"{stem}_radiomics.npy\"\n    study_meta_path = WORK / f\"{stem}_study_meta.npy\"\n    sex_path = WORK / f\"{stem}_sex.npy\"\n    ids_path = WORK / f\"{stem}_ids.npy\"\n\n    n = len(studies)\n    np.save(feature_path, np.zeros((n, CFG.n_slots, 16), dtype=np.float16))\n    np.save(mask_path, np.zeros((n, CFG.n_slots), dtype=bool))\n    np.save(radiomics_path, np.zeros((n, CFG.n_slots, RAD_AGG_DIM), dtype=np.float16))\n\n    slot_meta = np.zeros((n, CFG.n_slots, SLOT_META_DIM), dtype=np.float16)\n    meta = np.zeros((n, STUDY_META_DIM), dtype=np.float16)\n    sex = np.zeros(n, dtype=np.int8)\n    ids = np.empty(n, dtype=object)\n    for i, row in studies.reset_index(drop=True).iterrows():\n        study_id = str(row[\"StudyInstanceUID\"])\n        ids[i] = study_id\n        sex[i] = sex_to_id(row)\n        meta[i] = study_meta.get(study_id, np.zeros(STUDY_META_DIM, dtype=np.float32)).astype(np.float16)\n        selected = slots.get(study_id, {})\n        for s, slot_name in enumerate(SLOT_NAMES):\n            info = selected.get(slot_name)\n            if info is not None:\n                slot_meta[i, s, :4] = np.asarray([\n                    info[\"fluid\"],\n                    info[\"fat\"],\n                    1.0 if info[\"contrast\"] == \"Structural\" else 0.0,\n                    0.0,\n                ], dtype=np.float16)\n    np.save(slot_meta_path, slot_meta)\n    np.save(study_meta_path, meta)\n    np.save(sex_path, sex)\n    np.save(ids_path, ids, allow_pickle=True)\n    return feature_path, mask_path, slot_meta_path, radiomics_path, study_meta_path, sex_path, ids_path\n\n\ndef extract_study_features(studies, slots, study_meta, series_root, prefix, model=None, view_dim=None):\n    if not IMAGE_AVAILABLE:\n        print(f\"No DICOM folder found for {prefix}; creating metadata-only dummy features.\")\n        return create_dummy_feature_files(studies, slots, study_meta, prefix)\n\n    stem = f\"{prefix}_{CFG.cache_tag}\"\n    feature_path = WORK / f\"{stem}_slot_features.npy\"\n    mask_path = WORK / f\"{stem}_slot_mask.npy\"\n    slot_meta_path = WORK / f\"{stem}_slot_meta.npy\"\n    radiomics_path = WORK / f\"{stem}_radiomics.npy\"\n    study_meta_path = WORK / f\"{stem}_study_meta.npy\"\n    sex_path = WORK / f\"{stem}_sex.npy\"\n    ids_path = WORK / f\"{stem}_ids.npy\"\n    done_path = WORK / f\"{stem}.done\"\n    aggregate_dim = 3 * view_dim\n    expected = (len(studies), CFG.n_slots, aggregate_dim)\n\n    cache_ok = all(path.exists() for path in [\n        feature_path, mask_path, slot_meta_path, radiomics_path, study_meta_path, sex_path, ids_path, done_path\n    ])\n    if cache_ok and not CFG.rebuild_features:\n        try:\n            cached = np.load(feature_path, mmap_mode=\"r\")\n            cached_mask = np.load(mask_path, mmap_mode=\"r\")\n            if cached.shape == expected and cached_mask.shape == (len(studies), CFG.n_slots):\n                print(f\"Using cached {prefix} features:\", cached.shape)\n                return feature_path, mask_path, slot_meta_path, radiomics_path, study_meta_path, sex_path, ids_path\n        except Exception:\n            pass\n\n    done_path.unlink(missing_ok=True)\n    sampler = StudySampler(studies, slots, study_meta, series_root)\n    features = np.lib.format.open_memmap(feature_path, mode=\"w+\", dtype=np.float16, shape=expected)\n    masks = np.lib.format.open_memmap(mask_path, mode=\"w+\", dtype=np.bool_, shape=(len(studies), CFG.n_slots))\n    slot_meta_mm = np.lib.format.open_memmap(slot_meta_path, mode=\"w+\", dtype=np.float16, shape=(len(studies), CFG.n_slots, SLOT_META_DIM))\n    radiomics_mm = np.lib.format.open_memmap(radiomics_path, mode=\"w+\", dtype=np.float16, shape=(len(studies), CFG.n_slots, RAD_AGG_DIM))\n    study_meta_mm = np.lib.format.open_memmap(study_meta_path, mode=\"w+\", dtype=np.float16, shape=(len(studies), STUDY_META_DIM))\n    sexes = np.lib.format.open_memmap(sex_path, mode=\"w+\", dtype=np.int8, shape=(len(studies),))\n    ids = np.empty(len(studies), dtype=object)\n\n    workers = max(1, min(CFG.dicom_workers, os.cpu_count() or CFG.dicom_workers))\n\n    def safe_get(index):\n        try:\n            return sampler.get(index)\n        except Exception as exc:\n            study_id = str(studies.iloc[index][\"StudyInstanceUID\"])\n            if CFG.debug:\n                print(\"Study failed:\", study_id, repr(exc))\n            return (\n                np.zeros((CFG.n_slots, CFG.n_positions, 3, CFG.img_size, CFG.img_size), dtype=np.uint8),\n                np.zeros((CFG.n_slots, CFG.n_positions), dtype=bool),\n                np.zeros((CFG.n_slots, SLOT_META_DIM), dtype=np.float32),\n                np.zeros((CFG.n_slots, RAD_AGG_DIM), dtype=np.float32),\n                np.zeros(STUDY_META_DIM, dtype=np.float32),\n                0,\n                study_id,\n            )\n\n    cursor = 0\n    starts = range(0, len(sampler), CFG.study_batch_size)\n    with ThreadPoolExecutor(max_workers=workers) as pool:\n        for start in tqdm(starts, total=math.ceil(len(sampler) / CFG.study_batch_size), desc=f\"{prefix} DINO features\"):\n            stop = min(start + CFG.study_batch_size, len(sampler))\n            indices = list(range(start, stop))\n            items = list(pool.map(safe_get, indices)) if workers > 1 else [safe_get(i) for i in indices]\n\n            image_batch = torch.from_numpy(np.stack([item[0] for item in items]))\n            view_mask = torch.from_numpy(np.stack([item[1] for item in items]))\n            batch_size = len(items)\n            valid_views = view_mask.reshape(-1)\n\n            view_features = torch.zeros(batch_size * CFG.n_slots * CFG.n_positions, view_dim, dtype=torch.float32)\n            if valid_views.any():\n                flat_images = image_batch.reshape(-1, 3, CFG.img_size, CFG.img_size)\n                view_features[valid_views] = encode_views(model, flat_images[valid_views])\n\n            view_features = view_features.reshape(batch_size, CFG.n_slots, CFG.n_positions, view_dim)\n            aggregate = torch.zeros(batch_size, CFG.n_slots, 3 * view_dim, dtype=torch.float32)\n            slot_mask = view_mask.any(dim=2)\n            for b in range(batch_size):\n                for s in range(CFG.n_slots):\n                    present = view_mask[b, s]\n                    if present.any():\n                        values = view_features[b, s, present]\n                        aggregate[b, s] = torch.cat([\n                            values.mean(dim=0),\n                            values.amax(dim=0),\n                            values.std(dim=0, unbiased=False),\n                        ], dim=0)\n\n            end = cursor + batch_size\n            features[cursor:end] = aggregate.numpy().astype(np.float16)\n            masks[cursor:end] = slot_mask.numpy()\n            slot_meta_mm[cursor:end] = np.stack([item[2] for item in items]).astype(np.float16)\n            radiomics_mm[cursor:end] = np.stack([item[3] for item in items]).astype(np.float16)\n            study_meta_mm[cursor:end] = np.stack([item[4] for item in items]).astype(np.float16)\n            sexes[cursor:end] = np.asarray([item[5] for item in items], dtype=np.int8)\n            ids[cursor:end] = [item[6] for item in items]\n            cursor = end\n\n    for arr in [features, masks, slot_meta_mm, radiomics_mm, study_meta_mm, sexes]:\n        arr.flush()\n    np.save(ids_path, ids, allow_pickle=True)\n    done_path.write_text(\"complete\")\n    print(f\"Completed {prefix} features:\", expected)\n    return feature_path, mask_path, slot_meta_path, radiomics_path, study_meta_path, sex_path, ids_path\n\n\nif IMAGE_AVAILABLE:\n    dino_model, view_dim = load_dino_model()\nelse:\n    dino_model, view_dim = None, 16\n\ntrain_feature_files = extract_study_features(\n    train, train_slots, train_study_meta, BASE / \"train_series\", \"train\", dino_model, view_dim\n)\ntest_feature_files = extract_study_features(\n    test, test_slots, test_study_meta, BASE / \"test_series\", \"test\", dino_model, view_dim\n)\n\ndel dino_model\ngc.collect()\nif torch.cuda.is_available():\n    torch.cuda.empty_cache()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T08:35:28.836607Z","iopub.execute_input":"2026-08-07T08:35:28.837084Z","iopub.status.idle":"2026-08-07T10:02:30.919605Z","shell.execute_reply.started":"2026-08-07T08:35:28.837049Z","shell.execute_reply":"2026-08-07T10:02:30.916209Z"}},"outputs":[],"execution_count":null},{"id":"4a358dd1-8a75-4c7a-9777-5019d6e291de","cell_type":"markdown","source":"## 4. Compact Study Matrix\n\nThe frozen image embeddings are reduced with IncrementalPCA fitted on valid train and test slots.  Radiomics, sequence metadata, study protocol metadata, slot masks, and sex are appended after scaling.\n","metadata":{}},{"id":"bdd70cb2-408f-4dd7-8e76-806ec0d7bb00","cell_type":"code","source":"import joblib\n\n\ndef load_feature_bundle(files):\n    feat, mask, slot_meta, radiomics, study_meta, sex, ids = files\n    return (\n        np.load(feat, mmap_mode=\"r\"),\n        np.load(mask, mmap_mode=\"r\"),\n        np.load(slot_meta, mmap_mode=\"r\"),\n        np.load(radiomics, mmap_mode=\"r\"),\n        np.load(study_meta, mmap_mode=\"r\"),\n        np.load(sex, mmap_mode=\"r\"),\n        np.load(ids, allow_pickle=True),\n    )\n\n\ntr_feat, tr_mask, tr_slot_meta, tr_rad, tr_study_meta, tr_sex, tr_ids = load_feature_bundle(train_feature_files)\nte_feat, te_mask, te_slot_meta, te_rad, te_study_meta, te_sex, te_ids = load_feature_bundle(test_feature_files)\nslot_dim = int(tr_feat.shape[-1])\ntotal_valid_slots = int(np.asarray(tr_mask).sum() + np.asarray(te_mask).sum())\n\n\ndef fit_ipca_if_possible():\n    if total_valid_slots < 32 or slot_dim < 4:\n        print(\"Skipping PCA because image slots are unavailable.\")\n        return None, 0\n    n_components = min(CFG.pca_components, slot_dim - 1, max(8, total_valid_slots - 1))\n    model_path = WORK / f\"{CFG.cache_tag}_ipca_{n_components}.joblib\"\n    if model_path.exists() and not CFG.rebuild_pca:\n        print(\"Loaded cached IncrementalPCA:\", n_components)\n        return joblib.load(model_path), n_components\n\n    ipca = IncrementalPCA(n_components=n_components, batch_size=CFG.pca_batch_size)\n    buffer = []\n    buffered = 0\n\n    def feed(features, masks, label):\n        nonlocal buffer, buffered\n        for start in tqdm(range(0, len(features), 96), desc=f\"PCA fit {label}\", leave=False):\n            stop = min(start + 96, len(features))\n            block = np.asarray(features[start:stop], dtype=np.float32)\n            block_mask = np.asarray(masks[start:stop]).reshape(-1)\n            valid = block.reshape(-1, slot_dim)[block_mask]\n            if len(valid):\n                buffer.append(valid)\n                buffered += len(valid)\n            if buffered >= max(CFG.pca_batch_size, n_components * 4):\n                data = np.concatenate(buffer, axis=0)\n                if len(data) >= n_components:\n                    ipca.partial_fit(data)\n                buffer, buffered = [], 0\n\n    feed(tr_feat, tr_mask, \"train\")\n    feed(te_feat, te_mask, \"test\")\n    if buffered:\n        data = np.concatenate(buffer, axis=0)\n        if len(data) >= n_components:\n            ipca.partial_fit(data)\n    joblib.dump(ipca, model_path)\n    print(\"Fitted IncrementalPCA:\", n_components)\n    return ipca, n_components\n\n\nipca, n_components = fit_ipca_if_possible()\n\n\ndef transform_studies(features, masks, slot_meta, radiomics, study_meta, sex):\n    n = len(features)\n    if ipca is None or n_components == 0:\n        pca_flat = np.zeros((n, 0), dtype=np.float32)\n    else:\n        result = np.zeros((n, CFG.n_slots, n_components), dtype=np.float32)\n        for start in tqdm(range(0, n, 96), desc=\"PCA transform\", leave=False):\n            stop = min(start + 96, n)\n            block = np.asarray(features[start:stop], dtype=np.float32)\n            block_mask = np.asarray(masks[start:stop]).reshape(-1)\n            flat = block.reshape(-1, slot_dim)\n            transformed = np.zeros((len(flat), n_components), dtype=np.float32)\n            if block_mask.any():\n                transformed[block_mask] = ipca.transform(flat[block_mask]).astype(np.float32)\n            result[start:stop] = transformed.reshape(stop - start, CFG.n_slots, n_components)\n        pca_flat = result.reshape(n, -1)\n\n    sex_onehot = np.eye(3, dtype=np.float32)[np.asarray(sex, dtype=int).clip(0, 2)]\n    meta = np.concatenate([\n        np.asarray(masks, dtype=np.float32),\n        np.asarray(slot_meta, dtype=np.float32).reshape(n, -1),\n        np.asarray(radiomics, dtype=np.float32).reshape(n, -1),\n        np.asarray(study_meta, dtype=np.float32),\n        sex_onehot,\n    ], axis=1)\n    return np.nan_to_num(np.concatenate([pca_flat, meta], axis=1), nan=0.0, posinf=0.0, neginf=0.0)\n\n\nX_train_raw = transform_studies(tr_feat, tr_mask, tr_slot_meta, tr_rad, tr_study_meta, tr_sex)\nX_test_raw = transform_studies(te_feat, te_mask, te_slot_meta, te_rad, te_study_meta, te_sex)\n\nscaler = StandardScaler()\nscaler.fit(np.vstack([X_train_raw, X_test_raw]))\nX_train = scaler.transform(X_train_raw).astype(np.float32)\nX_test = scaler.transform(X_test_raw).astype(np.float32)\n\nprint(\"Study matrices:\", X_train.shape, X_test.shape)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:02:30.929017Z","iopub.execute_input":"2026-08-07T10:02:30.929945Z","iopub.status.idle":"2026-08-07T10:03:10.8438Z","shell.execute_reply.started":"2026-08-07T10:02:30.929918Z","shell.execute_reply":"2026-08-07T10:03:10.843208Z"}},"outputs":[],"execution_count":null},{"id":"d92c52cf-3af8-4b08-8cbe-4b79bc18bb83","cell_type":"markdown","source":"## 5. Conservative Student Ensemble\n\nAll student branches learn from the same soft teacher, but with different inductive biases.  OOF AUC on the tiny exact subset is used only for modest weight adjustment, not for high-variance prediction flips.\n","metadata":{}},{"id":"11f0e2fe-6b20-45cf-b31c-e00cbad1be9e","cell_type":"code","source":"def safe_rank(values):\n    values = np.asarray(values, dtype=np.float64)\n    if len(values) <= 1:\n        return np.full(len(values), 0.5, dtype=np.float32)\n    if np.nanmax(values) - np.nanmin(values) < 1e-12:\n        return np.full(len(values), 0.5, dtype=np.float32)\n    return (rankdata(values, method=\"average\") / (len(values) + 1.0)).astype(np.float32)\n\n\ndef select_top(indices, scores, labels, class_value, cap):\n    cls = indices[labels == class_value]\n    if len(cls) <= cap:\n        return cls\n    cls_scores = scores[labels == class_value]\n    order = np.argsort(-cls_scores)[:cap]\n    return cls[order]\n\n\ndef student_training_arrays(target_index, excluded_indices, teacher_probs, teacher_conf):\n    excluded = np.zeros(len(train), dtype=bool)\n    if excluded_indices is not None:\n        excluded[np.asarray(excluded_indices, dtype=int)] = True\n\n    exact_known = exact_mask[:, target_index] & (~excluded)\n    pseudo_pool = (~exact_mask[:, target_index]) & (~excluded)\n\n    probs = teacher_probs[:, target_index] if np.ndim(teacher_probs) == 2 else np.asarray(teacher_probs)\n    confs = teacher_conf[:, target_index] if np.ndim(teacher_conf) == 2 else np.asarray(teacher_conf)\n    confidence = confs * (2.0 * np.abs(probs - 0.5))\n    pseudo_candidate = pseudo_pool & (confidence >= CFG.pseudo_min_score)\n    pseudo_idx_all = np.where(pseudo_candidate)[0]\n    pseudo_labels_all = (probs[pseudo_idx_all] >= 0.5).astype(int)\n    pseudo_scores_all = confidence[pseudo_idx_all]\n\n    pos_idx = select_top(pseudo_idx_all, pseudo_scores_all, pseudo_labels_all, 1, CFG.max_pseudo_per_class)\n    neg_idx = select_top(pseudo_idx_all, pseudo_scores_all, pseudo_labels_all, 0, CFG.max_pseudo_per_class)\n    pseudo_idx = np.concatenate([pos_idx, neg_idx])\n    pseudo_labels = (probs[pseudo_idx] >= 0.5).astype(int)\n    pseudo_scores = confidence[pseudo_idx]\n\n    exact_idx = np.where(exact_known)[0]\n    exact_y = train.loc[exact_idx, TARGETS[target_index]].astype(int).values\n\n    fit_idx = np.concatenate([exact_idx, pseudo_idx]).astype(int)\n    fit_y = np.concatenate([exact_y, pseudo_labels]).astype(int)\n    pseudo_weight = CFG.pseudo_soft_floor + 1.15 * np.power(np.clip(pseudo_scores, 0, 1), 1.4)\n    weights = np.concatenate([\n        np.full(len(exact_idx), CFG.exact_weight, dtype=np.float32),\n        pseudo_weight.astype(np.float32),\n    ])\n\n    if len(np.unique(fit_y)) < 2 and len(exact_y) and len(np.unique(exact_y)) == 2:\n        fit_idx = exact_idx\n        fit_y = exact_y\n        weights = np.full(len(exact_idx), CFG.exact_weight, dtype=np.float32)\n    return fit_idx, fit_y, weights\n\n\ndef exact_training_arrays(target_index, excluded_indices):\n    excluded = np.zeros(len(train), dtype=bool)\n    if excluded_indices is not None:\n        excluded[np.asarray(excluded_indices, dtype=int)] = True\n    idx = np.where(exact_mask[:, target_index] & (~excluded))[0]\n    y = train.loc[idx, TARGETS[target_index]].astype(int).values\n    w = np.full(len(idx), CFG.exact_correction_weight, dtype=np.float32)\n    return idx, y, w\n\n\ndef fit_lr_family(X_fit, y_fit, X_pred, sample_weight, cs, seed):\n    y_fit = np.asarray(y_fit, dtype=int)\n    if len(y_fit) == 0:\n        return np.full(len(X_pred), 0.5, dtype=np.float32)\n    if len(np.unique(y_fit)) < 2:\n        return np.full(len(X_pred), float(y_fit.mean()), dtype=np.float32)\n    preds = []\n    for k, c_value in enumerate(cs):\n        model = LogisticRegression(\n            C=c_value,\n            solver=\"liblinear\",\n            class_weight=\"balanced\",\n            max_iter=3000,\n            random_state=seed + 31 * k,\n        )\n        model.fit(X_fit, y_fit, sample_weight=sample_weight)\n        preds.append(model.predict_proba(X_pred)[:, 1])\n    return np.mean(preds, axis=0).astype(np.float32)\n\n\ndef fit_et_family(X_fit, y_fit, X_pred, sample_weight, seed):\n    y_fit = np.asarray(y_fit, dtype=int)\n    if len(y_fit) == 0:\n        return np.full(len(X_pred), 0.5, dtype=np.float32)\n    if len(np.unique(y_fit)) < 2:\n        return np.full(len(X_pred), float(y_fit.mean()), dtype=np.float32)\n    model = ExtraTreesClassifier(\n        n_estimators=420 if not CFG.debug else 60,\n        max_features=\"sqrt\",\n        min_samples_leaf=4,\n        min_samples_split=8,\n        bootstrap=False,\n        class_weight=\"balanced\",\n        random_state=seed,\n        n_jobs=-1,\n    )\n    model.fit(X_fit, y_fit, sample_weight=sample_weight)\n    return model.predict_proba(X_pred)[:, 1].astype(np.float32)\n\n\ndef fit_hgb_family(X_fit, y_fit, X_pred, sample_weight, seed):\n    y_fit = np.asarray(y_fit, dtype=int)\n    if len(y_fit) == 0:\n        return np.full(len(X_pred), 0.5, dtype=np.float32)\n    if len(np.unique(y_fit)) < 2:\n        return np.full(len(X_pred), float(y_fit.mean()), dtype=np.float32)\n    model = HistGradientBoostingClassifier(\n        learning_rate=0.035,\n        max_iter=180 if not CFG.debug else 40,\n        max_leaf_nodes=15,\n        min_samples_leaf=18,\n        l2_regularization=0.25,\n        early_stopping=True,\n        validation_fraction=0.15,\n        random_state=seed,\n    )\n    model.fit(X_fit, y_fit, sample_weight=sample_weight)\n    return model.predict_proba(X_pred)[:, 1].astype(np.float32)\n\n\nFAMILY_ORDER = [\"lr\", \"et\", \"hgb\", \"exact_lr\"]\n\n\ndef fit_family_predictions(family, target_index, train_indices, y, weights, pred_indices_or_matrix, seed):\n    X_pred = X_train[pred_indices_or_matrix] if isinstance(pred_indices_or_matrix, np.ndarray) and pred_indices_or_matrix.dtype.kind in \"iu\" else pred_indices_or_matrix\n    X_fit = X_train[train_indices]\n    if family == \"lr\":\n        return fit_lr_family(X_fit, y, X_pred, weights, CFG.lr_cs, seed)\n    if family == \"et\":\n        return fit_et_family(X_fit, y, X_pred, weights, seed)\n    if family == \"hgb\":\n        return fit_hgb_family(X_fit, y, X_pred, weights, seed)\n    if family == \"exact_lr\":\n        return fit_lr_family(X_fit, y, X_pred, weights, CFG.exact_lr_cs, seed)\n    raise ValueError(f\"Unknown family: {family}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:10:38.124895Z","iopub.execute_input":"2026-08-07T10:10:38.125595Z","iopub.status.idle":"2026-08-07T10:10:38.146459Z","shell.execute_reply.started":"2026-08-07T10:10:38.125563Z","shell.execute_reply":"2026-08-07T10:10:38.145652Z"}},"outputs":[],"execution_count":null},{"id":"58af11be-82a0-4527-812c-5ec0f54709e1","cell_type":"code","source":"oof_preds = {\n    family: np.full((len(train), len(TARGETS)), np.nan, dtype=np.float32)\n    for family in FAMILY_ORDER\n}\noof_rows = []\n\nfor j, target in enumerate(TARGETS):\n    idx = np.where(exact_mask[:, j])[0]\n    y_exact = train.loc[idx, target].astype(int).values\n    minority = int(np.bincount(y_exact).min()) if len(np.unique(y_exact)) == 2 else 0\n    n_splits = min(CFG.oof_folds, minority)\n    if n_splits < 2:\n        continue\n\n    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=CFG.seed + 3000 + j)\n    for fold, (_, va_local) in enumerate(skf.split(idx, y_exact)):\n        va_idx = idx[va_local]\n        fold_teacher, fold_conf = make_teacher_for_target(j, excluded_indices=va_idx, overwrite_exact=True)\n        fit_idx, fit_y, fit_w = student_training_arrays(j, va_idx, fold_teacher, fold_conf)\n        exact_idx, exact_y, exact_w = exact_training_arrays(j, va_idx)\n\n        for family in [\"lr\", \"et\", \"hgb\"]:\n            oof_preds[family][va_idx, j] = fit_family_predictions(\n                family,\n                j,\n                fit_idx,\n                fit_y,\n                fit_w,\n                va_idx,\n                CFG.seed + 5000 + 101 * j + 17 * fold + len(family),\n            )\n        oof_preds[\"exact_lr\"][va_idx, j] = fit_family_predictions(\n            \"exact_lr\",\n            j,\n            exact_idx,\n            exact_y,\n            exact_w,\n            va_idx,\n            CFG.seed + 7000 + 101 * j + 17 * fold,\n        )\n\n    row = {\"target\": target}\n    valid = np.isfinite(oof_preds[\"lr\"][idx, j])\n    if valid.sum() and len(np.unique(y_exact[valid])) == 2:\n        for family in FAMILY_ORDER:\n            pred = oof_preds[family][idx[valid], j]\n            row[f\"{family}_auc\"] = roc_auc_score(y_exact[valid], pred) if np.isfinite(pred).all() else np.nan\n    oof_rows.append(row)\n\noof_table = pd.DataFrame(oof_rows)\ndisplay(oof_table)\n\n\ndef family_weights_from_oof(oof_table):\n    defaults = np.asarray([CFG.default_family_weights[name] for name in FAMILY_ORDER], dtype=np.float32)\n    defaults = defaults / defaults.sum()\n    weights = np.tile(defaults[None, :], (len(TARGETS), 1)).astype(np.float32)\n    rows_by_target = {row[\"target\"]: row for _, row in oof_table.iterrows()} if len(oof_table) else {}\n\n    for j, target in enumerate(TARGETS):\n        row = rows_by_target.get(target)\n        if row is None:\n            continue\n        aucs = np.asarray([row.get(f\"{name}_auc\", np.nan) for name in FAMILY_ORDER], dtype=np.float32)\n        if not np.isfinite(aucs).any():\n            continue\n        gains = np.maximum(np.nan_to_num(aucs, nan=0.5) - 0.50, 0.0)\n        if gains.sum() <= 1e-8:\n            continue\n        learned = defaults * (0.20 + gains ** 1.7)\n        learned = learned / learned.sum()\n        weights[j] = 0.72 * defaults + 0.28 * learned\n        weights[j] = weights[j] / weights[j].sum()\n    return weights\n\n\ntarget_family_weights = family_weights_from_oof(oof_table)\ndisplay(pd.DataFrame(target_family_weights, columns=FAMILY_ORDER).assign(target=TARGETS)[[\"target\"] + FAMILY_ORDER])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:10:41.853457Z","iopub.execute_input":"2026-08-07T10:10:41.85426Z","iopub.status.idle":"2026-08-07T10:23:10.776025Z","shell.execute_reply.started":"2026-08-07T10:10:41.854228Z","shell.execute_reply":"2026-08-07T10:23:10.775384Z"}},"outputs":[],"execution_count":null},{"id":"b3c083ab-b514-4e1a-9127-a4ed6d871a7a","cell_type":"markdown","source":"## 6. Final Training and Submission\n","metadata":{}},{"id":"3970689d-cadd-400d-8f1a-a450a84e6118","cell_type":"code","source":"final_family_preds = {\n    family: np.zeros((len(test), len(TARGETS)), dtype=np.float32)\n    for family in FAMILY_ORDER\n}\n\nfor j, target in enumerate(tqdm(TARGETS, desc=\"Final target models\")):\n    fit_idx, fit_y, fit_w = student_training_arrays(j, [], teacher_train, teacher_conf_train)\n    exact_idx, exact_y, exact_w = exact_training_arrays(j, [])\n\n    for family in [\"lr\", \"et\", \"hgb\"]:\n        final_family_preds[family][:, j] = fit_family_predictions(\n            family,\n            j,\n            fit_idx,\n            fit_y,\n            fit_w,\n            X_test,\n            CFG.seed + 9000 + 113 * j + len(family),\n        )\n    final_family_preds[\"exact_lr\"][:, j] = fit_family_predictions(\n        \"exact_lr\",\n        j,\n        exact_idx,\n        exact_y,\n        exact_w,\n        X_test,\n        CFG.seed + 11000 + 113 * j,\n    )\n\n\ntest_probs = np.zeros((len(test), len(TARGETS)), dtype=np.float32)\nfor j, target in enumerate(TARGETS):\n    weighted_rank = np.zeros(len(test), dtype=np.float32)\n    weighted_prob = np.zeros(len(test), dtype=np.float32)\n    for k, family in enumerate(FAMILY_ORDER):\n        pred = np.clip(final_family_preds[family][:, j], 1e-5, 1.0 - 1e-5)\n        weight = target_family_weights[j, k]\n        weighted_rank += weight * safe_rank(pred)\n        weighted_prob += weight * pred\n    test_probs[:, j] = 0.90 * weighted_rank + 0.10 * weighted_prob\n\ntest_probs = np.clip(test_probs, 1e-5, 1.0 - 1e-5)\n\nprediction_frame = pd.DataFrame(test_probs, columns=TARGETS)\nprediction_frame.insert(0, \"StudyInstanceUID\", te_ids.astype(str))\n\nsubmission = sample_submission[[\"StudyInstanceUID\"]].copy()\nsubmission[\"StudyInstanceUID\"] = submission[\"StudyInstanceUID\"].astype(str)\nsubmission = submission.merge(prediction_frame, on=\"StudyInstanceUID\", how=\"left\", validate=\"one_to_one\")\n\nfor target in TARGETS:\n    if target not in submission.columns:\n        submission[target] = 0.5\n    submission[target] = pd.to_numeric(submission[target], errors=\"coerce\").fillna(0.5).clip(1e-5, 1.0 - 1e-5)\n\nsubmission = submission[[\"StudyInstanceUID\"] + TARGETS]\nassert submission.shape[0] == sample_submission.shape[0], \"Row count changed.\"\nassert submission[\"StudyInstanceUID\"].is_unique, \"Duplicate StudyInstanceUID values.\"\nassert submission.columns.tolist() == [\"StudyInstanceUID\"] + TARGETS, \"Bad submission columns.\"\nassert np.isfinite(submission[TARGETS].values).all(), \"Non-finite predictions.\"\n\nsubmission_path = WORK / \"submission.csv\"\nsubmission.to_csv(submission_path, index=False)\nwith zipfile.ZipFile(WORK / \"submission.zip\", \"w\", compression=zipfile.ZIP_DEFLATED) as zf:\n    zf.write(submission_path, arcname=\"submission.csv\")\n\nprint(\"Saved:\", submission_path)\nprint(\"Shape:\", submission.shape)\ndisplay(pd.DataFrame(target_family_weights, columns=FAMILY_ORDER).assign(target=TARGETS)[[\"target\"] + FAMILY_ORDER])\ndisplay(submission.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:23:10.777412Z","iopub.execute_input":"2026-08-07T10:23:10.777656Z","iopub.status.idle":"2026-08-07T10:25:48.955095Z","shell.execute_reply.started":"2026-08-07T10:23:10.777634Z","shell.execute_reply":"2026-08-07T10:25:48.954359Z"}},"outputs":[],"execution_count":null},{"id":"32807423-271f-46a2-93ba-243f0eda6776","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}