{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport os, re, json, random, warnings\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\")\npd.set_option(\"display.max_columns\", 100)\nrandom.seed(0)\nnp.random.seed(0)\n\nBASE = Path(\"/kaggle/input\")\nCOMP = None\nfor p in BASE.rglob(\"sample_submission.csv\"):\n    COMP = p.parent\n    break\n\nif COMP is None:\n    raise FileNotFoundError(\"Не нашёл sample_submission.csv. Проверь Add Data.\")\n\nprint(\"COMP:\", COMP)\n\ntrain = pd.read_csv(COMP / \"train.csv\")\ntest = pd.read_csv(COMP / \"test.csv\")\ntrain_series = pd.read_csv(COMP / \"train_series.csv\")\ntest_series = pd.read_csv(COMP / \"test_series.csv\")\nsample_sub = pd.read_csv(COMP / \"sample_submission.csv\")\n\nLABELS = [\n    \"ACL\", \"MCL\", \"Medial Meniscus\", \"Lateral Meniscus\",\n    \"Medial OA\", \"Lateral OA\", \"PF OA\", \"Effusion\",\n    \"Synovitis\", \"Baker's\", \"Contusion\", \"Fracture\",\n]\n\nfor name, df in [(\"train\", train), (\"test\", test), (\"train_series\", train_series), (\"test_series\", test_series), (\"sample_sub\", sample_sub)]:\n    print(\"\\n\", name, df.shape)\n    print(df.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:31:06.491886Z","iopub.execute_input":"2026-08-14T19:31:06.492702Z","iopub.status.idle":"2026-08-14T19:31:06.653925Z","shell.execute_reply.started":"2026-08-14T19:31:06.49266Z","shell.execute_reply":"2026-08-14T19:31:06.653075Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def series_stats(sdf, name):\n    print(\"\\n\" + \"=\" * 70)\n    print(name, sdf.shape)\n    print(\"unique studies:\", sdf[\"StudyInstanceUID\"].nunique())\n    print(\"unique series:\", sdf[\"SeriesInstanceUID\"].nunique())\n\n    per_study = sdf.groupby(\"StudyInstanceUID\").size()\n    print(\"\\nseries per study:\")\n    print(per_study.describe())\n\n    for c in [\"Fluid_Sensitive\", \"Fat_Suppression\", \"Anatomical_Plane\"]:\n        print(\"\\n\", c)\n        print(sdf[c].value_counts(dropna=False).head(20))\n\n    print(\"\\nTop combos plane/fluid/fat:\")\n    print(\n        sdf.groupby([\"Anatomical_Plane\", \"Fluid_Sensitive\", \"Fat_Suppression\"])\n        .size()\n        .sort_values(ascending=False)\n        .head(25)\n    )\n\nseries_stats(train_series, \"train_series\")\nseries_stats(test_series, \"test_series\")\n\ndef availability(sdf):\n    rows = []\n    for study, x in sdf.groupby(\"StudyInstanceUID\"):\n        rows.append({\n            \"n_series\": len(x),\n            \"has_axial\": bool((x[\"Anatomical_Plane\"] == \"Axial\").any()),\n            \"has_sagittal\": bool((x[\"Anatomical_Plane\"] == \"Sagittal\").any()),\n            \"has_coronal\": bool((x[\"Anatomical_Plane\"] == \"Coronal\").any()),\n            \"has_fluid_sensitive\": bool((x[\"Fluid_Sensitive\"] == 1).any()),\n            \"has_fat_suppression\": bool((x[\"Fat_Suppression\"] == 1).any()),\n            \"has_all_3_planes\": bool(\n                (x[\"Anatomical_Plane\"] == \"Axial\").any()\n                and (x[\"Anatomical_Plane\"] == \"Sagittal\").any()\n                and (x[\"Anatomical_Plane\"] == \"Coronal\").any()\n            ),\n        })\n    return pd.DataFrame(rows)\n\navail = availability(train_series)\nprint(\"\\nTrain study availability:\")\nprint(avail[[\"has_axial\", \"has_sagittal\", \"has_coronal\", \"has_fluid_sensitive\", \"has_fat_suppression\", \"has_all_3_planes\"]].mean().round(4))\nprint(avail[\"n_series\"].describe())\n\navail.to_csv(\"/kaggle/working/train_series_availability.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:31:06.655479Z","iopub.execute_input":"2026-08-14T19:31:06.655794Z","iopub.status.idle":"2026-08-14T19:31:09.015511Z","shell.execute_reply.started":"2026-08-14T19:31:06.655765Z","shell.execute_reply":"2026-08-14T19:31:09.014725Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nprint(\"pydicom:\", pydicom.__version__)\n\ndef series_dir(split, study, series):\n    return COMP / f\"{split}_series\" / str(study) / str(series)\n\n# Берём случайные train-серии + все публичные test-серии.\ntrain_s = train_series.sample(min(500, len(train_series)), random_state=0)\ntest_s = test_series.copy()\n\nrows = []\nfor split, sdf in [(\"train\", train_s), (\"test\", test_s)]:\n    for _, r in sdf.iterrows():\n        d = series_dir(split, r[\"StudyInstanceUID\"], r[\"SeriesInstanceUID\"])\n        files = list(d.glob(\"*.dcm\"))\n        if not files:\n            continue\n        mid = files[len(files) // 2]\n        try:\n            ds = pydicom.dcmread(str(mid), stop_before_pixels=True, force=True)\n            rows.append({\n                \"split\": split,\n                \"study\": r[\"StudyInstanceUID\"],\n                \"series\": r[\"SeriesInstanceUID\"],\n                \"n_files\": len(files),\n                \"rows\": getattr(ds, \"Rows\", np.nan),\n                \"cols\": getattr(ds, \"Columns\", np.nan),\n                \"series_desc\": getattr(ds, \"SeriesDescription\", None),\n                \"plane\": r[\"Anatomical_Plane\"],\n                \"fluid\": r[\"Fluid_Sensitive\"],\n                \"fat\": r[\"Fat_Suppression\"],\n            })\n        except Exception:\n            pass\n\nmeta_sample = pd.DataFrame(rows)\nprint(\"meta_sample:\", meta_sample.shape)\ndisplay(meta_sample.head())\n\nprint(\"\\nTop random series_desc:\")\nprint(meta_sample[\"series_desc\"].astype(str).value_counts().head(40))\n\nprint(\"\\nShape counts:\")\nprint(meta_sample.groupby([\"rows\", \"cols\"]).size().sort_values(ascending=False).head(20))\n\nmeta_sample.to_csv(\"/kaggle/working/dicom_meta_random_sample.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:31:09.016632Z","iopub.execute_input":"2026-08-14T19:31:09.016963Z","iopub.status.idle":"2026-08-14T19:31:10.921351Z","shell.execute_reply.started":"2026-08-14T19:31:09.016939Z","shell.execute_reply":"2026-08-14T19:31:10.920562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\n\ntry:\n    from pydicom.pixel_data_handlers.util import apply_voi_lut\nexcept Exception:\n    apply_voi_lut = None\n\n# На случай если ячейку запускают отдельно после перезапуска kernel\nif \"COMP\" not in globals():\n    BASE = Path(\"/kaggle/input\")\n    COMP = None\n    for p in BASE.rglob(\"sample_submission.csv\"):\n        COMP = p.parent\n        break\n    if COMP is None:\n        raise FileNotFoundError(\"Не нашёл sample_submission.csv. Проверь Add Data.\")\n\nif \"train_series\" not in globals():\n    train_series = pd.read_csv(COMP / \"train_series.csv\")\n\ndef series_dir(split, study, series):\n    return COMP / f\"{split}_series\" / str(study) / str(series)\n\ndef load_slice(path):\n    ds = pydicom.dcmread(str(path))\n    arr = ds.pixel_array\n\n    if apply_voi_lut is not None:\n        try:\n            arr = apply_voi_lut(arr, ds)\n        except Exception:\n            pass\n\n    arr = arr.astype(np.float32)\n\n    if arr.ndim == 3:\n        if arr.shape[-1] in [3, 4]:\n            arr = arr[..., 0]\n        else:\n            arr = arr[0]\n\n    if str(getattr(ds, \"PhotometricInterpretation\", \"\")).upper() == \"MONOCHROME1\":\n        arr = arr.max() - arr\n\n    arr = (arr - arr.min()) / (arr.max() - arr.min() + 1e-6)\n    return arr\n\ndef pick_series(sdf, study):\n    x = sdf[sdf[\"StudyInstanceUID\"] == study].copy()\n    chosen = {}\n\n    for plane in [\"Axial\", \"Sagittal\", \"Coronal\"]:\n        xp = x[x[\"Anatomical_Plane\"] == plane].copy()\n        if len(xp) == 0:\n            continue\n        xp[\"score\"] = xp[\"Fluid_Sensitive\"].fillna(0) * 2 + xp[\"Fat_Suppression\"].fillna(0)\n        xp = xp.sort_values(\"score\", ascending=False)\n        chosen[plane] = xp.iloc[0]\n\n    return chosen\n\n# Собираем нормальный good_df с StudyInstanceUID, без merge по индексу\ngood_rows = []\nfor study, x in train_series.groupby(\"StudyInstanceUID\"):\n    has_ax = bool((x[\"Anatomical_Plane\"] == \"Axial\").any())\n    has_sag = bool((x[\"Anatomical_Plane\"] == \"Sagittal\").any())\n    has_cor = bool((x[\"Anatomical_Plane\"] == \"Coronal\").any())\n\n    good_rows.append({\n        \"StudyInstanceUID\": study,\n        \"has_axial\": has_ax,\n        \"has_sagittal\": has_sag,\n        \"has_coronal\": has_cor,\n        \"has_fluid_sensitive\": bool((x[\"Fluid_Sensitive\"] == 1).any()),\n        \"has_all_3_planes\": bool(has_ax and has_sag and has_cor),\n    })\n\ngood_df = pd.DataFrame(good_rows)\n\nmask = good_df[\"has_all_3_planes\"] & good_df[\"has_fluid_sensitive\"]\ngood_studies = good_df.loc[mask, \"StudyInstanceUID\"]\n\n# fallback, если строгий фильтр вдруг пустой\nif len(good_studies) == 0:\n    good_studies = good_df.loc[good_df[\"has_all_3_planes\"], \"StudyInstanceUID\"]\nif len(good_studies) == 0:\n    good_studies = train_series[\"StudyInstanceUID\"].drop_duplicates()\n\nshow_studies = list(good_studies.sample(min(4, len(good_studies)), random_state=1))\nprint(\"show studies:\", show_studies)\n\nplt.figure(figsize=(15, 4 * len(show_studies)))\nplot_i = 1\n\nfor study in show_studies:\n    chosen = pick_series(train_series, study)\n\n    for plane in [\"Axial\", \"Sagittal\", \"Coronal\"]:\n        if plane not in chosen:\n            continue\n\n        r = chosen[plane]\n        d = series_dir(\"train\", r[\"StudyInstanceUID\"], r[\"SeriesInstanceUID\"])\n        files = list(d.glob(\"*.dcm\"))\n        if not files:\n            continue\n\n        mid = files[len(files) // 2]\n        ax = plt.subplot(len(show_studies), 3, plot_i)\n\n        try:\n            img = load_slice(mid)\n            ax.imshow(img, cmap=\"gray\")\n            ax.set_title(\n                f\"{plane} | fluid={r['Fluid_Sensitive']} fat={r['Fat_Suppression']}\\n{len(files)} files\",\n                fontsize=8\n            )\n        except Exception as e:\n            ax.set_title(\"ERR \" + repr(e)[:60], fontsize=8)\n\n        ax.axis(\"off\")\n        plot_i += 1\n\nplt.tight_layout()\nplt.savefig(\"/kaggle/working/sample_by_plane.png\", dpi=150)\nplt.show()\nprint(\"saved: /kaggle/working/sample_by_plane.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:38:18.223608Z","iopub.execute_input":"2026-08-14T19:38:18.224017Z","iopub.status.idle":"2026-08-14T19:38:24.212272Z","shell.execute_reply.started":"2026-08-14T19:38:18.223989Z","shell.execute_reply":"2026-08-14T19:38:24.211244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"report_len\"] = train[\"Report\"].fillna(\"\").astype(str).str.len()\nprint(\"Report length:\")\nprint(train[\"report_len\"].describe())\n\nLANG_HINTS = {\n    \"es\": [\" el \", \" la \", \" rodilla\", \"menisco\", \"derrame\", \"hallazgos\", \"impresión\"],\n    \"en\": [\" the \", \" knee\", \"meniscus\", \"effusion\", \"findings\", \"impression\"],\n    \"de\": [\" der \", \" die \", \" knie\", \"meniskus\", \"erguss\", \"befund\"],\n    \"nl\": [\" de \", \" het \", \" knie\", \"meniscus\", \"bevindingen\", \"vraagstelling\"],\n    \"fr\": [\" le \", \" la \", \" genou\", \"ménisque\", \"épanchement\", \"conclusion\"],\n    \"pt\": [\" o \", \" a \", \" joelho\", \"menisco\", \"derrame\", \"impressão\"],\n    \"it\": [\" il \", \" la \", \" ginocchio\", \"menisco\", \"versamento\", \"conclusione\"],\n    \"ru\": [\" и \", \" в \", \"колен\", \"мениск\", \"выпот\", \"заключение\"],\n}\n\ndef guess_lang(text):\n    t = \" \" + str(text).lower() + \" \"\n    scores = {lang: sum(t.count(\" \" + w.strip() + \" \") if \" \" in w else t.count(w) for w in words) for lang, words in LANG_HINTS.items()}\n    return max(scores, key=scores.get) if max(scores.values()) > 0 else \"unknown\"\n\ntrain[\"lang_guess\"] = train[\"Report\"].fillna(\"\").map(guess_lang)\nprint(\"\\nLanguage guess:\")\nprint(train[\"lang_guess\"].value_counts())\n\n# ВАЖНО: это только грубый extractor для оценки покрытия, без отрицаний.\nKEYWORDS = {\n    \"ACL\": [r\"\\bacl\\b\", r\"\\blca\\b\", r\"cruzado anterior\", r\"croisé antérieur\", r\"vorderes kreuzband\", r\"крестообраз\"],\n    \"MCL\": [r\"\\bmcl\\b\", r\"colateral medial\", r\"collatéral médial\", r\"mediales kollateral\", r\"медиальн.*коллатерал\"],\n    \"Medial Meniscus\": [r\"menisco interno\", r\"medial meniscus\", r\"meniscus medial\", r\"menisco medial\", r\"innenmeniskus\", r\"медиальн.*мениск\"],\n    \"Lateral Meniscus\": [r\"menisco externo\", r\"lateral meniscus\", r\"meniscus lateral\", r\"menisco lateral\", r\"außenmeniskus\", r\"латеральн.*мениск\"],\n    \"Effusion\": [r\"effusion\", r\"derrame\", r\"erguss\", r\"épanchement\", r\"versamento\", r\"выпот\", r\"joint fluid\"],\n    \"Fracture\": [r\"fracture\", r\"fractura\", r\"fraktur\", r\"перелом\"],\n    \"Baker's\": [r\"baker\", r\"quiste popl\", r\"popliteal cyst\", r\"poplitea\", r\"беккер\", r\"бейкер\"],\n    \"Contusion\": [r\"contusion\", r\"contusión\", r\"bone bruise\", r\"edema óseo\", r\"bone marrow edema\", r\"костномозгов\"],\n    \"Synovitis\": [r\"synovitis\", r\"sinovitis\", r\"синовит\"],\n    \"Medial OA\": [r\"artrosis femorotibial medial\", r\"medial.*osteoarth\", r\"medial.*arthrose\", r\"медиальн.*остеоартр\"],\n    \"Lateral OA\": [r\"artrosis femorotibial lateral\", r\"lateral.*osteoarth\", r\"lateral.*arthrose\", r\"латеральн.*остеоартр\"],\n    \"PF OA\": [r\"femoropatelar\", r\"patellofemoral\", r\"patellofemor\", r\"пателлофеморал\"],\n}\n\ndef extract_keywords(text):\n    t = str(text).lower()\n    return {lab: int(any(re.search(p, t, flags=re.I) for p in pats)) for lab, pats in KEYWORDS.items()}\n\nkw = pd.DataFrame([extract_keywords(x) for x in train[\"Report\"].fillna(\"\")])\ncoverage = kw.mean().sort_values(ascending=False)\nprint(\"\\nKeyword coverage, доля отчётов где слово встретилось хотя бы раз:\")\nprint(coverage.round(4))\n\nkw.to_csv(\"/kaggle/working/report_keyword_extractor_v0.csv\", index=False)\ncoverage.to_csv(\"/kaggle/working/report_keyword_coverage_v0.csv\")\n\nprint(\"\\nПримеры совпадений:\")\nfor lab in coverage.index[:6]:\n    idx = kw[kw[lab] == 1].index[:2]\n    print(\"\\n###\", lab)\n    for i in idx:\n        print(train.loc[i, \"lang_guess\"], \"|\", str(train.loc[i, \"Report\"])[:500].replace(\"\\n\", \" \"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:38:39.770441Z","iopub.execute_input":"2026-08-14T19:38:39.77085Z","iopub.status.idle":"2026-08-14T19:38:42.703372Z","shell.execute_reply.started":"2026-08-14T19:38:39.770822Z","shell.execute_reply":"2026-08-14T19:38:42.702633Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = sample_sub.copy()\nmissing = [c for c in LABELS if c not in sub.columns]\nextra = [c for c in sub.columns if c not in [\"StudyInstanceUID\"] + LABELS]\nprint(\"missing label cols:\", missing)\nprint(\"extra cols:\", extra)\n\nsub[LABELS] = 0.5\nsub = sub[[\"StudyInstanceUID\"] + LABELS]\nout = \"/kaggle/working/submission_constant_0_5.csv\"\nsub.to_csv(out, index=False)\nprint(\"saved:\", out, sub.shape)\ndisplay(sub.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-14T19:38:47.874365Z","iopub.execute_input":"2026-08-14T19:38:47.874755Z","iopub.status.idle":"2026-08-14T19:38:47.895874Z","shell.execute_reply.started":"2026-08-14T19:38:47.874728Z","shell.execute_reply":"2026-08-14T19:38:47.894956Z"}},"outputs":[],"execution_count":null}]}