{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:01:31.132678Z","iopub.execute_input":"2026-08-07T10:01:31.133016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nRSNA Knee - dump sample reports to inspect real phrasing patterns\nbefore building the multilingual weak-label extraction pipeline.\n\"\"\"\nimport os\nimport pandas as pd\n\nDATA_DIR = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\ntrain = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\n\nLABEL_COLS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\",\n]\npresent_labels = [c for c in LABEL_COLS if c in train.columns]\nhas_any_label = train[present_labels].notna().any(axis=1)\n\n# 1) Show a few of the 58 GOLD-labeled reports, so we can see what\n#    phrasing maps to a positive vs negative label for each finding.\nprint(\"=\" * 70)\nprint(\"GOLD-LABELED SAMPLE (label present) - first 5\")\nprint(\"=\" * 70)\ngold = train.loc[has_any_label].head(5)\nfor _, row in gold.iterrows():\n    print(\"\\nStudyInstanceUID:\", row[\"StudyInstanceUID\"])\n    print(\"Labels:\", {c: row[c] for c in present_labels})\n    print(\"Report:\\n\", row[\"Report\"])\n    print(\"-\" * 60)\n\n# 2) Show a random sample of UNLABELED reports across the length /\n#    character-set spectrum to eyeball language diversity.\nprint(\"\\n\" + \"=\" * 70)\nprint(\"RANDOM UNLABELED REPORTS - 8 samples\")\nprint(\"=\" * 70)\nunlabeled_sample = train.loc[~has_any_label].sample(8, random_state=0)\nfor _, row in unlabeled_sample.iterrows():\n    print(\"\\nStudyInstanceUID:\", row[\"StudyInstanceUID\"])\n    print(\"Report:\\n\", row[\"Report\"])\n    print(\"-\" * 60)\n\n# 3) Rough character-set based language signal (no external libs needed):\n#    count non-ASCII heavy reports vs ASCII-only, as a proxy.\ndef ascii_ratio(s):\n    if not isinstance(s, str) or len(s) == 0:\n        return 1.0\n    ascii_chars = sum(1 for ch in s if ord(ch) < 128)\n    return ascii_chars / len(s)\n\ntrain[\"ascii_ratio\"] = train[\"Report\"].apply(ascii_ratio)\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ASCII RATIO DISTRIBUTION (proxy for non-English reports)\")\nprint(\"=\" * 70)\nprint(train[\"ascii_ratio\"].describe())\nprint(\"\\nReports with ascii_ratio < 0.9 (likely non-English), count:\",\n      (train[\"ascii_ratio\"] < 0.9).sum())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T10:53:17.744888Z","iopub.execute_input":"2026-08-07T10:53:17.745616Z","iopub.status.idle":"2026-08-07T10:53:18.257023Z","shell.execute_reply.started":"2026-08-07T10:53:17.745587Z","shell.execute_reply":"2026-08-07T10:53:18.255881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nRSNA Knee - Report -> Weak Label Extraction (v1)\n==================================================\nRule-based (keyword + negation) extractor for the 12 findings, built to\nhandle both structured reports (section headers) and free-narrative\nreports, across the primary Latin-alphabet languages first (English,\nSpanish). Extensible LEXICON dict for more languages later.\n\nRun this AFTER loading train.csv. It will:\n1. Extract labels from every report.\n2. Compare extracted labels to the 58 gold-labeled studies.\n3. Print per-finding precision / recall / F1 so we know where the\n   extractor is weak and needs more terms or negation handling.\n\"\"\"\n\nimport os\nimport re\nimport pandas as pd\n\nDATA_DIR = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\ntrain = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\n\nLABEL_COLS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\",\n]\n\n# ---------------------------------------------------------------------\n# LEXICON\n# Each finding maps to:\n#   \"positive\": list of regex patterns (case-insensitive) that indicate\n#               the finding is present when NOT negated\n#   \"negation\": list of regex patterns that, if found near a positive\n#               match (same sentence), flip it to negative\n# This is v1 - English + Spanish only. Extend per language once we\n# see enough of each language in the sample dump.\n# ---------------------------------------------------------------------\n\nNEGATION_EN = [\n    r\"\\bno\\b\", r\"\\bnot\\b\", r\"\\bwithout\\b\", r\"\\bintact\\b\", r\"\\bnormal\\b\",\n    r\"\\bpreserved\\b\", r\"\\bunremarkable\\b\", r\"\\bnegative for\\b\",\n    r\"\\bno evidence of\\b\", r\"\\bno significant\\b\", r\"\\bno acute\\b\",\n]\nNEGATION_ES = [\n    r\"\\bsin\\b\", r\"\\bno hay\\b\", r\"\\bno se\\b\", r\"\\bnormal(?:es)?\\b\",\n    r\"\\bdentro de límites normales\\b\", r\"\\bconservad[ao]s?\\b\",\n    r\"\\bíntegr[ao]s?\\b\", r\"\\bintact[ao]s?\\b\",\n]\n\nLEXICON = {\n    \"ACL\": {\n        \"positive\": [r\"\\bacl\\b.{0,40}\\btear\\b\", r\"\\bacl\\b.{0,40}\\brupture\\b\",\n                     r\"anterior cruciate ligament.{0,40}(tear|rupture|disrupt)\",\n                     r\"ligamento cruzado anterior.{0,40}(rotura|desgarro)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"MCL\": {\n        \"positive\": [r\"\\bmcl\\b.{0,40}\\b(tear|sprain|injury)\\b\",\n                     r\"medial collateral ligament.{0,40}(tear|sprain|injury)\",\n                     r\"ligamento colateral medial.{0,40}(rotura|lesión|desgarro)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Medial Meniscus\": {\n        \"positive\": [r\"medial meniscus.{0,60}(tear|torn|extrusion|complex tear)\",\n                     r\"menisco medial.{0,60}(rotura|desgarro|extrusión)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Lateral Meniscus\": {\n        \"positive\": [r\"lateral meniscus.{0,60}(tear|torn|extrusion|complex tear)\",\n                     r\"menisco lateral.{0,60}(rotura|desgarro|extrusión)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Medial OA\": {\n        \"positive\": [r\"medial compartment.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\",\n                     r\"medial (tibiofemoral)?.{0,20}osteoarthrit\",\n                     r\"condropat[ií]a.{0,40}medial\", r\"compartimento (femorotibial )?medial.{0,60}(úlcera|condral|degenerativ)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Lateral OA\": {\n        \"positive\": [r\"lateral compartment.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\",\n                     r\"lateral (tibiofemoral)?.{0,20}osteoarthrit\",\n                     r\"condropat[ií]a.{0,40}lateral\", r\"compartimento (femorotibial )?lateral.{0,60}(úlcera|condral|degenerativ)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"PF OA\": {\n        \"positive\": [r\"patellofemoral.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\",\n                     r\"compartimento femorotibial.{0,20}patel\",\n                     r\"r[óo]tula.{0,40}(condropat[ií]a|úlceras condrales)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Effusion\": {\n        \"positive\": [r\"\\beffusion\\b\", r\"\\bderrame articular\\b\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Synovitis\": {\n        \"positive\": [r\"\\bsynovitis\\b\", r\"\\bsinovitis\\b\", r\"synovial (thickening|hypertrophy)\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Baker's\": {\n        \"positive\": [r\"baker'?s? cyst\", r\"quiste (de baker|popl[ií]teo)\", r\"popliteal cyst\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Contusion\": {\n        \"positive\": [r\"bone (bruise|contusion)\", r\"marrow edema\", r\"edema ó?seo\",\n                     r\"contusi[oó]n\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n    \"Fracture\": {\n        \"positive\": [r\"\\bfracture\\b\", r\"\\bfractura\\b\"],\n        \"negation\": NEGATION_EN + NEGATION_ES,\n    },\n}\n\n\ndef split_sentences(text):\n    \"\"\"Cheap sentence/line splitter that works for both narrative and\n    section-header style reports.\"\"\"\n    if not isinstance(text, str):\n        return []\n    # split on newlines and on '.' followed by space+capital, keep it simple\n    chunks = re.split(r\"[\\n\\r]+|(?<=[.;])\\s+(?=[A-ZÁÉÍÓÚÑ])\", text)\n    return [c.strip() for c in chunks if c.strip()]\n\n\ndef extract_labels(report_text):\n    \"\"\"Return dict finding -> 0/1/None (None = not mentioned at all).\"\"\"\n    sentences = split_sentences(report_text)\n    result = {f: None for f in LABEL_COLS}\n\n    for finding, rules in LEXICON.items():\n        pos_patterns = [re.compile(p, re.IGNORECASE) for p in rules[\"positive\"]]\n        neg_patterns = [re.compile(p, re.IGNORECASE) for p in rules[\"negation\"]]\n\n        found_positive = False\n        found_any_mention = False\n\n        for sent in sentences:\n            for pat in pos_patterns:\n                m = pat.search(sent)\n                if m:\n                    found_any_mention = True\n                    # check negation within the same sentence, prioritizing\n                    # negation words that appear BEFORE the match\n                    window = sent[: m.start()]\n                    negated = any(neg.search(window) for neg in neg_patterns)\n                    if not negated:\n                        found_positive = True\n\n        if found_positive:\n            result[finding] = 1\n        elif found_any_mention:\n            result[finding] = 0\n        # else stays None (finding never mentioned -> skip / abstain)\n\n    return result\n\n\n# ---------------------------------------------------------------------\n# Apply to all reports\n# ---------------------------------------------------------------------\nprint(\"Extracting weak labels from all reports...\")\nextracted = train[\"Report\"].apply(extract_labels).apply(pd.Series)\nextracted.columns = [f\"pred_{c}\" for c in extracted.columns]\ntrain_ext = pd.concat([train, extracted], axis=1)\n\ncoverage = extracted.notna().mean()\nprint(\"\\n\" + \"=\" * 70)\nprint(\"EXTRACTION COVERAGE (fraction of ALL reports where finding was mentioned)\")\nprint(\"=\" * 70)\nprint(coverage.sort_values(ascending=False).to_string())\n\n# ---------------------------------------------------------------------\n# Validate against the 58 gold-labeled studies\n# ---------------------------------------------------------------------\npresent_labels = [c for c in LABEL_COLS if c in train.columns]\nhas_any_label = train[present_labels].notna().any(axis=1)\ngold = train_ext.loc[has_any_label]\n\nprint(\"\\n\" + \"=\" * 70)\nprint(f\"VALIDATION AGAINST {len(gold)} GOLD-LABELED STUDIES\")\nprint(\"=\" * 70)\n\nrows = []\nfor finding in LABEL_COLS:\n    y_true = gold[finding]\n    y_pred = gold[f\"pred_{finding}\"]\n\n    mask = y_pred.notna()  # only score where extractor made a call\n    n_scored = mask.sum()\n    if n_scored == 0:\n        rows.append((finding, 0, None, None, None))\n        continue\n\n    yt = y_true[mask].astype(int)\n    yp = y_pred[mask].astype(int)\n\n    tp = ((yt == 1) & (yp == 1)).sum()\n    fp = ((yt == 0) & (yp == 1)).sum()\n    fn = ((yt == 1) & (yp == 0)).sum()\n    tn = ((yt == 0) & (yp == 0)).sum()\n\n    precision = tp / (tp + fp) if (tp + fp) > 0 else None\n    recall = tp / (tp + fn) if (tp + fn) > 0 else None\n    accuracy = (tp + tn) / n_scored\n\n    rows.append((finding, n_scored, precision, recall, accuracy))\n\nreport_df = pd.DataFrame(rows, columns=[\"finding\", \"n_scored_of_58\", \"precision\", \"recall\", \"accuracy\"])\nprint(report_df.to_string(index=False, float_format=lambda x: f\"{x:.2f}\" if x is not None else \"NA\"))\n\nprint(\"\\nAlso check: how many of the 58 gold studies got ZERO extracted mentions at all (extractor blind spot)?\")\nno_mentions = extracted.loc[has_any_label].isna().all(axis=1).sum()\nprint(f\"-> {no_mentions} / {len(gold)} gold studies had no findings extracted at all\")\n\nprint(\"\\nNext: inspect false positives/negatives per finding to refine regex + add more languages.\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q pylibjpeg pylibjpeg-libjpeg pylibjpeg-openjpeg python-gdcm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T12:05:10.421127Z","iopub.execute_input":"2026-08-07T12:05:10.421396Z","iopub.status.idle":"2026-08-07T12:05:17.036046Z","shell.execute_reply.started":"2026-08-07T12:05:10.421362Z","shell.execute_reply":"2026-08-07T12:05:17.035291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nRSNA Knee Abnormality Detection - Baseline v1 (image model + submission)\n==========================================================================\nRun this in a Kaggle Notebook with GPU enabled (Settings -> Accelerator -> GPU).\n\nIf DICOM decoding fails with \"Unable to decompress\", run this in a cell\nfirst:\n    !pip install -q pylibjpeg pylibjpeg-libjpeg pylibjpeg-openjpeg python-gdcm\n\nPipeline:\n1. Build a label table: gold labels (58 studies) take priority; everywhere\n   else, use the regex-extracted weak labels from the report-parsing\n   script (only where the extractor actually made a call - masked).\n2. For every study, pick up to 3 series (prefer one Sagittal, one\n   Coronal, one Axial, fluid-sensitive if available).\n3. For each chosen series, sample 3 evenly spaced slices -> stack as a\n   3-channel \"2.5D\" image -> feed into a ResNet18 backbone.\n4. Multi-label head (12 sigmoid outputs), masked BCE loss (only\n   backprop on labels we actually know).\n5. Validate AUC on the gold-labeled holdout only (the only labels we\n   fully trust).\n6. Run inference on test_series/, average predictions across the\n   selected series per study, write submission.csv.\n\"\"\"\n\nimport os\nimport re\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport torchvision.models as models\nfrom sklearn.metrics import roc_auc_score\n\n# ---------------------------------------------------------------------\n# CONFIG\n# ---------------------------------------------------------------------\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\nDATA_DIR = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\nTRAIN_SERIES_DIR = os.path.join(DATA_DIR, \"train_series\")\nTEST_SERIES_DIR = os.path.join(DATA_DIR, \"test_series\")\n\nIMG_SIZE = 224\nN_SLICES_PER_SERIES = 3          # -> 3-channel \"2.5D\" image\nMAX_SERIES_PER_STUDY = 3         # try 1 sagittal + 1 coronal + 1 axial\nBATCH_SIZE = 8\nEPOCHS = 5\nLR = 1e-4\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nLABEL_COLS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\",\n]\n\nprint(\"Device:\", DEVICE)\n\n# ---------------------------------------------------------------------\n# 1. LOAD METADATA\n# ---------------------------------------------------------------------\ntrain = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\"))\ntrain_series = pd.read_csv(os.path.join(DATA_DIR, \"train_series.csv\"))\ntest = pd.read_csv(os.path.join(DATA_DIR, \"test.csv\"))\ntest_series = pd.read_csv(os.path.join(DATA_DIR, \"test_series.csv\"))\nsample_sub = pd.read_csv(os.path.join(DATA_DIR, \"sample_submission.csv\"))\n\npresent_labels = [c for c in LABEL_COLS if c in train.columns]\nhas_gold = train[present_labels].notna().any(axis=1)\nprint(f\"Gold-labeled studies: {has_gold.sum()} / {len(train)}\")\n\n# ---------------------------------------------------------------------\n# 2. WEAK LABELS FROM REPORTS (reuse the v1 extractor - condensed here)\n#    NOTE: for a real gold run, import your refined extractor instead\n#    of this condensed copy once you've improved coverage.\n# ---------------------------------------------------------------------\nNEGATION = [\n    r\"\\bno\\b\", r\"\\bnot\\b\", r\"\\bwithout\\b\", r\"\\bintact\\b\", r\"\\bnormal\\b\",\n    r\"\\bpreserved\\b\", r\"\\bunremarkable\\b\", r\"\\bnegative for\\b\",\n    r\"\\bno evidence of\\b\", r\"\\bsin\\b\", r\"\\bno hay\\b\", r\"\\bconservad[ao]s?\\b\",\n    r\"\\bíntegr[ao]s?\\b\", r\"\\bintact[ao]s?\\b\",\n]\nLEXICON = {\n    \"ACL\": [r\"\\bacl\\b.{0,40}\\b(tear|rupture)\\b\", r\"anterior cruciate ligament.{0,40}(tear|rupture|disrupt)\"],\n    \"MCL\": [r\"\\bmcl\\b.{0,40}\\b(tear|sprain|injury)\\b\", r\"medial collateral ligament.{0,40}(tear|sprain|injury)\"],\n    \"Medial Meniscus\": [r\"medial meniscus.{0,60}(tear|torn|extrusion)\"],\n    \"Lateral Meniscus\": [r\"lateral meniscus.{0,60}(tear|torn|extrusion)\"],\n    \"Medial OA\": [r\"medial compartment.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\"],\n    \"Lateral OA\": [r\"lateral compartment.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\"],\n    \"PF OA\": [r\"patellofemoral.{0,60}(cartilage loss|chondral|osteoarthrit|degenerat)\"],\n    \"Effusion\": [r\"\\beffusion\\b\", r\"\\bderrame articular\\b\"],\n    \"Synovitis\": [r\"\\bsynovitis\\b\", r\"\\bsinovitis\\b\"],\n    \"Baker's\": [r\"baker'?s? cyst\", r\"popliteal cyst\"],\n    \"Contusion\": [r\"bone (bruise|contusion)\", r\"marrow edema\"],\n    \"Fracture\": [r\"\\bfracture\\b\", r\"\\bfractura\\b\"],\n}\n\n\ndef split_sentences(text):\n    if not isinstance(text, str):\n        return []\n    return [c.strip() for c in re.split(r\"[\\n\\r]+|(?<=[.;])\\s+(?=[A-ZÁÉÍÓÚÑ])\", text) if c.strip()]\n\n\ndef extract_weak_labels(text):\n    sentences = split_sentences(text)\n    out = {f: np.nan for f in LABEL_COLS}\n    for finding, patterns in LEXICON.items():\n        mentioned, positive = False, False\n        for sent in sentences:\n            for p in patterns:\n                m = re.search(p, sent, re.IGNORECASE)\n                if m:\n                    mentioned = True\n                    negated = any(re.search(n, sent[: m.start()], re.IGNORECASE) for n in NEGATION)\n                    if not negated:\n                        positive = True\n        if mentioned:\n            out[finding] = 1.0 if positive else 0.0\n    return out\n\n\nprint(\"Extracting weak labels from reports (this can take a minute)...\")\nweak = train[\"Report\"].apply(extract_weak_labels).apply(pd.Series)\n\n# Final label table: gold overrides weak; mask = 1 wherever we have\n# either a gold or a weak (extractor-confident) value.\nlabel_table = weak.copy()\nmask_table = weak.notna().astype(float)\nfor c in present_labels:\n    gold_vals = train[c]\n    override = gold_vals.notna()\n    label_table.loc[override, c] = gold_vals[override]\n    mask_table.loc[override, c] = 1.0\n\nlabel_table = label_table.fillna(0.0)  # unknowns filled but MASKED OUT in loss\nprint(\"Label table ready:\", label_table.shape)\n\n# ---------------------------------------------------------------------\n# 3. SERIES SELECTION\n# ---------------------------------------------------------------------\ndef pick_series_for_study(study_uid, series_df, series_root):\n    rows = series_df[series_df[\"StudyInstanceUID\"] == study_uid]\n    if rows.empty:\n        return []\n    chosen = []\n    for plane in [\"Sagittal\", \"Coronal\", \"Axial\"]:\n        plane_rows = rows[rows[\"Anatomical_Plane\"] == plane]\n        if plane_rows.empty:\n            continue\n        # prefer fluid sensitive series\n        plane_rows = plane_rows.sort_values(\"Fluid_Sensitive\", ascending=False)\n        series_uid = plane_rows.iloc[0][\"SeriesInstanceUID\"]\n        series_path = os.path.join(series_root, study_uid, series_uid)\n        if os.path.isdir(series_path):\n            chosen.append(series_path)\n        if len(chosen) >= MAX_SERIES_PER_STUDY:\n            break\n    if not chosen:\n        # fallback: just take the first available series\n        series_uid = rows.iloc[0][\"SeriesInstanceUID\"]\n        series_path = os.path.join(series_root, study_uid, series_uid)\n        if os.path.isdir(series_path):\n            chosen.append(series_path)\n    return chosen\n\n\n# ---------------------------------------------------------------------\n# 4. DICOM LOADING -> 2.5D image\n# ---------------------------------------------------------------------\ndef load_series_as_image(series_path, n_slices=N_SLICES_PER_SERIES, size=IMG_SIZE):\n    files = sorted(glob.glob(os.path.join(series_path, \"*.dcm\")))\n    if not files:\n        return np.zeros((size, size, n_slices), dtype=np.float32)\n\n    idxs = np.linspace(0, len(files) - 1, num=min(n_slices, len(files))).astype(int)\n    idxs = list(idxs)\n    while len(idxs) < n_slices:  # pad if series has fewer files than n_slices\n        idxs.append(idxs[-1])\n\n    slices = []\n    for i in idxs:\n        try:\n            ds = pydicom.dcmread(files[i])\n            arr = ds.pixel_array.astype(np.float32)\n            # basic normalization\n            arr -= arr.min()\n            if arr.max() > 0:\n                arr /= arr.max()\n            slices.append(arr)\n        except Exception:\n            slices.append(np.zeros((size, size), dtype=np.float32))\n\n    # resize each slice to (size, size)\n    resized = []\n    for arr in slices:\n        t = torch.from_numpy(arr).unsqueeze(0).unsqueeze(0)\n        t = torch.nn.functional.interpolate(t, size=(size, size), mode=\"bilinear\", align_corners=False)\n        resized.append(t.squeeze().numpy())\n\n    img = np.stack(resized, axis=-1)  # (H, W, n_slices)\n    return img\n\n\n# ---------------------------------------------------------------------\n# 5. DATASET\n# ---------------------------------------------------------------------\nIMAGENET_MEAN = [0.485, 0.456, 0.406]\nIMAGENET_STD = [0.229, 0.224, 0.225]\nnormalize = T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)\n\n\nclass KneeStudyDataset(Dataset):\n    \"\"\"One (study, series) pair per item; study-level label is shared\n    across its series during training. At inference we average\n    predictions across a study's series.\"\"\"\n\n    def __init__(self, study_uids, series_df, series_root, labels_df=None, mask_df=None):\n        self.items = []  # list of (study_uid, series_path)\n        for uid in study_uids:\n            for sp in pick_series_for_study(uid, series_df, series_root):\n                self.items.append((uid, sp))\n        self.labels_df = labels_df\n        self.mask_df = mask_df\n        self.study_index = {uid: i for i, uid in enumerate(study_uids)}\n\n    def __len__(self):\n        return len(self.items)\n\n    def __getitem__(self, idx):\n        study_uid, series_path = self.items[idx]\n        img = load_series_as_image(series_path)  # (H, W, 3)\n        img = torch.from_numpy(img).permute(2, 0, 1).float()  # (3, H, W)\n        img = normalize(img)\n\n        if self.labels_df is not None:\n            y = torch.tensor(self.labels_df.loc[self.study_index[study_uid]].values, dtype=torch.float32)\n            m = torch.tensor(self.mask_df.loc[self.study_index[study_uid]].values, dtype=torch.float32)\n            return img, y, m, study_uid\n        return img, study_uid\n\n\n# ---------------------------------------------------------------------\n# 6. TRAIN / VAL SPLIT\n#    Put ALL gold-labeled studies in val (our only trustworthy signal),\n#    train on everything else (weak-labeled, masked loss).\n# ---------------------------------------------------------------------\nall_study_uids = train[\"StudyInstanceUID\"].tolist()\ngold_uids = train.loc[has_gold, \"StudyInstanceUID\"].tolist()\ntrain_uids = train.loc[~has_gold, \"StudyInstanceUID\"].tolist()\n\nlabel_table = label_table.reset_index(drop=True)\nmask_table = mask_table.reset_index(drop=True)\n\ntrain_ds = KneeStudyDataset(train_uids, train_series, TRAIN_SERIES_DIR, label_table, mask_table)\nval_ds = KneeStudyDataset(gold_uids, train_series, TRAIN_SERIES_DIR, label_table, mask_table)\n\nprint(f\"Train items (study-series pairs): {len(train_ds)}\")\nprint(f\"Val items (study-series pairs): {len(val_ds)}\")\n\ntrain_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\n# ---------------------------------------------------------------------\n# 7. MODEL\n# ---------------------------------------------------------------------\nmodel = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1)\nmodel.fc = nn.Linear(model.fc.in_features, len(LABEL_COLS))\nmodel = model.to(DEVICE)\n\ncriterion = nn.BCEWithLogitsLoss(reduction=\"none\")\noptimizer = torch.optim.AdamW(model.parameters(), lr=LR)\n\n\ndef masked_bce(logits, targets, mask):\n    loss = criterion(logits, targets)\n    loss = (loss * mask).sum() / mask.sum().clamp(min=1.0)\n    return loss\n\n\n# ---------------------------------------------------------------------\n# 8. TRAIN LOOP\n# ---------------------------------------------------------------------\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0.0\n    for imgs, ys, ms, _ in train_loader:\n        imgs, ys, ms = imgs.to(DEVICE), ys.to(DEVICE), ms.to(DEVICE)\n        optimizer.zero_grad()\n        logits = model(imgs)\n        loss = masked_bce(logits, ys, ms)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item() * imgs.size(0)\n    avg_loss = total_loss / len(train_ds)\n\n    # ---- validation on gold studies only ----\n    model.eval()\n    study_logits = {}\n    study_targets = {}\n    with torch.no_grad():\n        for imgs, ys, ms, uids in val_loader:\n            imgs = imgs.to(DEVICE)\n            logits = torch.sigmoid(model(imgs)).cpu().numpy()\n            for i, uid in enumerate(uids):\n                study_logits.setdefault(uid, []).append(logits[i])\n                study_targets[uid] = ys[i].numpy()\n\n    aucs = []\n    for j, finding in enumerate(LABEL_COLS):\n        y_true, y_pred = [], []\n        for uid in study_logits:\n            y_true.append(study_targets[uid][j])\n            y_pred.append(np.mean([p[j] for p in study_logits[uid]]))\n        y_true = np.array(y_true)\n        y_pred = np.array(y_pred)\n        if len(np.unique(y_true)) > 1:\n            aucs.append(roc_auc_score(y_true, y_pred))\n    mean_auc = np.mean(aucs) if aucs else float(\"nan\")\n\n    print(f\"Epoch {epoch+1}/{EPOCHS} - train_loss={avg_loss:.4f} - val_mean_AUC(gold)={mean_auc:.4f}\")\n\n# ---------------------------------------------------------------------\n# 9. INFERENCE ON TEST SET -> submission.csv\n# ---------------------------------------------------------------------\ntest_uids = test[\"StudyInstanceUID\"].tolist()\ntest_ds = KneeStudyDataset(test_uids, test_series, TEST_SERIES_DIR)\ntest_loader = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\nmodel.eval()\ntest_preds = {}\nwith torch.no_grad():\n    for imgs, uids in test_loader:\n        imgs = imgs.to(DEVICE)\n        probs = torch.sigmoid(model(imgs)).cpu().numpy()\n        for i, uid in enumerate(uids):\n            test_preds.setdefault(uid, []).append(probs[i])\n\nrows = []\nfor uid in test_uids:\n    if uid in test_preds:\n        avg_probs = np.mean(test_preds[uid], axis=0)\n    else:\n        avg_probs = np.full(len(LABEL_COLS), 0.5)  # fallback if no series found\n    rows.append([uid] + list(avg_probs))\n\nsubmission = pd.DataFrame(rows, columns=[\"StudyInstanceUID\"] + LABEL_COLS)\n\n# make sure column order / names exactly match sample_submission.csv\nsubmission = submission[sample_sub.columns]\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"\\nsubmission.csv written:\")\nprint(submission.head())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T13:00:25.346504Z","iopub.execute_input":"2026-08-07T13:00:25.346958Z"}},"outputs":[],"execution_count":null}]}