{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nimport warnings\nimport re\nimport unicodedata\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport pydicom\nfrom pydicom.errors import InvalidDicomError\n\nfrom IPython.display import display\n\nwarnings.filterwarnings(\"ignore\")\n\nSEED = 42\nrng = np.random.default_rng(SEED)\n\npd.set_option(\"display.max_columns\", 100)\npd.set_option(\"display.max_colwidth\", 120)\n\nLABELS = [\n    \"ACL\",\n    \"MCL\",\n    \"Medial Meniscus\",\n    \"Lateral Meniscus\",\n    \"Medial OA\",\n    \"Lateral OA\",\n    \"PF OA\",\n    \"Effusion\",\n    \"Synovitis\",\n    \"Baker's\",\n    \"Contusion\",\n    \"Fracture\",\n]\n\n# DICOMを大量に読まないための上限\nN_META_SERIES = 200\nN_IMAGE_SLICES = 7","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.458563Z","iopub.execute_input":"2026-08-22T04:57:17.459392Z","iopub.status.idle":"2026-08-22T04:57:17.4665Z","shell.execute_reply.started":"2026-08-22T04:57:17.459357Z","shell.execute_reply":"2026-08-22T04:57:17.465502Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"/kaggle/input/competitions/rsna-knee-abnormality-detection","metadata":{}},{"cell_type":"code","source":"KAGGLE_INPUT = Path(\"/kaggle/input/competitions\")\n\ndefault_root = KAGGLE_INPUT / \"rsna-knee-abnormality-detection\"\n\nif (\n    (default_root / \"train.csv\").exists()\n    and (default_root / \"train_series.csv\").exists()\n):\n    DATA_ROOT = default_root\nelse:\n    candidates = [\n        p for p in KAGGLE_INPUT.iterdir()\n        if p.is_dir()\n        and (p / \"train.csv\").exists()\n        and (p / \"train_series.csv\").exists()\n    ]\n\n    if not candidates:\n        raise FileNotFoundError(\n            \"train.csv と train_series.csv を含むKaggle inputが見つかりません。\"\n        )\n\n    DATA_ROOT = candidates[0]\n\nprint(\"DATA_ROOT =\", DATA_ROOT)\n\nTRAIN_CSV = DATA_ROOT / \"train.csv\"\nTRAIN_SERIES_CSV = DATA_ROOT / \"train_series.csv\"\nTRAIN_SERIES_DIR = DATA_ROOT / \"train_series\"\n\nTEST_CSV = DATA_ROOT / \"test.csv\"\nTEST_SERIES_CSV = DATA_ROOT / \"test_series.csv\"\nSAMPLE_SUBMISSION_CSV = DATA_ROOT / \"sample_submission.csv\"\n\nprint(\"train.csv       :\", TRAIN_CSV.exists())\nprint(\"train_series.csv:\", TRAIN_SERIES_CSV.exists())\nprint(\"train_series/   :\", TRAIN_SERIES_DIR.exists())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.468141Z","iopub.execute_input":"2026-08-22T04:57:17.468556Z","iopub.status.idle":"2026-08-22T04:57:17.50055Z","shell.execute_reply.started":"2026-08-22T04:57:17.468531Z","shell.execute_reply":"2026-08-22T04:57:17.499662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(TRAIN_CSV)\ntrain_series = pd.read_csv(TRAIN_SERIES_CSV)\n\nprint(\"train.shape       =\", train.shape)\nprint(\"train_series.shape=\", train_series.shape)\n\nprint(\"\\ntrain columns:\")\nprint(train.columns.tolist())\n\nprint(\"\\ntrain_series columns:\")\nprint(train_series.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.502483Z","iopub.execute_input":"2026-08-22T04:57:17.502851Z","iopub.status.idle":"2026-08-22T04:57:17.665515Z","shell.execute_reply.started":"2026-08-22T04:57:17.502814Z","shell.execute_reply":"2026-08-22T04:57:17.664602Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.666537Z","iopub.execute_input":"2026-08-22T04:57:17.666951Z","iopub.status.idle":"2026-08-22T04:57:17.682978Z","shell.execute_reply.started":"2026-08-22T04:57:17.666911Z","shell.execute_reply":"2026-08-22T04:57:17.682106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train_series.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.685339Z","iopub.execute_input":"2026-08-22T04:57:17.685763Z","iopub.status.idle":"2026-08-22T04:57:17.707126Z","shell.execute_reply.started":"2026-08-22T04:57:17.685736Z","shell.execute_reply":"2026-08-22T04:57:17.70615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"===== Study =====\")\nprint(\"train studies:\", train[\"StudyInstanceUID\"].nunique())\nprint(\"StudyInstanceUID unique:\", train[\"StudyInstanceUID\"].is_unique)\n\nprint(\"\\n===== Series =====\")\nprint(\"series rows:\", len(train_series))\nprint(\"unique SeriesInstanceUID:\",\n      train_series[\"SeriesInstanceUID\"].nunique())\n\nprint(\n    \"SeriesInstanceUID unique:\",\n    train_series[\"SeriesInstanceUID\"].is_unique\n)\n\nseries_studies = set(train_series[\"StudyInstanceUID\"])\ntrain_studies = set(train[\"StudyInstanceUID\"])\n\nprint(\n    \"\\ntrain_seriesに存在するstudyがすべてtrain.csvに存在:\",\n    series_studies.issubset(train_studies)\n)\n\nprint(\n    \"seriesを1つ以上持つstudy:\",\n    len(series_studies)\n)\n\nprint(\n    \"train.csvにあるがtrain_seriesにないstudy:\",\n    len(train_studies - series_studies)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.708579Z","iopub.execute_input":"2026-08-22T04:57:17.708937Z","iopub.status.idle":"2026-08-22T04:57:17.748914Z","shell.execute_reply.started":"2026-08-22T04:57:17.7089Z","shell.execute_reply":"2026-08-22T04:57:17.748006Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_per_study = (\n    train_series\n    .groupby(\"StudyInstanceUID\")\n    .size()\n    .rename(\"n_series\")\n)\n\ndisplay(series_per_study.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.750124Z","iopub.execute_input":"2026-08-22T04:57:17.750449Z","iopub.status.idle":"2026-08-22T04:57:17.767498Z","shell.execute_reply.started":"2026-08-22T04:57:17.750418Z","shell.execute_reply":"2026-08-22T04:57:17.766483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 4))\n\nax.hist(\n    series_per_study,\n    bins=np.arange(\n        series_per_study.min(),\n        series_per_study.max() + 2\n    ) - 0.5\n)\n\nax.set_xlabel(\"Number of series per study\")\nax.set_ylabel(\"Number of studies\")\nax.set_title(\"Series count per study\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.768777Z","iopub.execute_input":"2026-08-22T04:57:17.769153Z","iopub.status.idle":"2026-08-22T04:57:17.918108Z","shell.execute_reply.started":"2026-08-22T04:57:17.76912Z","shell.execute_reply":"2026-08-22T04:57:17.917147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_summary = (\n    train[[\"StudyInstanceUID\"]]\n    .merge(\n        series_per_study,\n        left_on=\"StudyInstanceUID\",\n        right_index=True,\n        how=\"left\"\n    )\n)\n\ndisplay(study_summary.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.919323Z","iopub.execute_input":"2026-08-22T04:57:17.919655Z","iopub.status.idle":"2026-08-22T04:57:17.93447Z","shell.execute_reply.started":"2026-08-22T04:57:17.919629Z","shell.execute_reply":"2026-08-22T04:57:17.933557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_stats = []\n\nfor label in LABELS:\n    known = train[label].notna()\n\n    n_labeled = int(known.sum())\n    n_positive = int((train.loc[known, label] == 1).sum())\n    n_negative = int((train.loc[known, label] == 0).sum())\n\n    prevalence = (\n        n_positive / n_labeled\n        if n_labeled > 0 else np.nan\n    )\n\n    label_stats.append({\n        \"Label\": label,\n        \"Labeled\": n_labeled,\n        \"Positive\": n_positive,\n        \"Negative\": n_negative,\n        \"Positive prevalence\": prevalence,\n    })\n\nlabel_stats = pd.DataFrame(label_stats)\n\ndisplay(label_stats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.935684Z","iopub.execute_input":"2026-08-22T04:57:17.93603Z","iopub.status.idle":"2026-08-22T04:57:17.96837Z","shell.execute_reply.started":"2026-08-22T04:57:17.935999Z","shell.execute_reply":"2026-08-22T04:57:17.967309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train[\"n_known_labels\"] = train[LABELS].notna().sum(axis=1)\n\nprint(train[\"n_known_labels\"].value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.971719Z","iopub.execute_input":"2026-08-22T04:57:17.971982Z","iopub.status.idle":"2026-08-22T04:57:17.991866Z","shell.execute_reply.started":"2026-08-22T04:57:17.971959Z","shell.execute_reply":"2026-08-22T04:57:17.990739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"counts = train[\"n_known_labels\"].value_counts().sort_index()\n\nfig, ax = plt.subplots(figsize=(8, 4))\n\nax.bar(\n    counts.index.astype(str),\n    counts.values\n)\n\nax.set_xlabel(\"Number of known labels in a study\")\nax.set_ylabel(\"Number of studies\")\nax.set_title(\"Label availability per study\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:17.993423Z","iopub.execute_input":"2026-08-22T04:57:17.993745Z","iopub.status.idle":"2026-08-22T04:57:18.108744Z","shell.execute_reply.started":"2026-08-22T04:57:17.99372Z","shell.execute_reply":"2026-08-22T04:57:18.107929Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gold_mask = train[LABELS].notna().all(axis=1)\ngold = train.loc[gold_mask].copy()\n\nprint(\"Fully labeled studies:\", len(gold))\nprint(\"Fraction:\", len(gold) / len(train))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.109962Z","iopub.execute_input":"2026-08-22T04:57:18.110435Z","iopub.status.idle":"2026-08-22T04:57:18.120249Z","shell.execute_reply.started":"2026-08-22T04:57:18.110408Z","shell.execute_reply":"2026-08-22T04:57:18.119146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"no_label_mask = train[LABELS].isna().all(axis=1)\n\nprint(\"Studies with no expert label:\", no_label_mask.sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.121528Z","iopub.execute_input":"2026-08-22T04:57:18.121829Z","iopub.status.idle":"2026-08-22T04:57:18.142521Z","shell.execute_reply.started":"2026-08-22T04:57:18.121804Z","shell.execute_reply":"2026-08-22T04:57:18.141389Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_df = (\n    label_stats\n    .sort_values(\"Positive prevalence\")\n)\n\nfig, ax = plt.subplots(figsize=(9, 6))\n\nax.barh(\n    plot_df[\"Label\"],\n    plot_df[\"Positive prevalence\"]\n)\n\nax.set_xlabel(\"Positive prevalence\")\nax.set_title(\"Abnormality prevalence among labeled studies\")\nax.set_xlim(0, 1)\n\nfor i, x in enumerate(plot_df[\"Positive prevalence\"]):\n    ax.text(\n        x + 0.01,\n        i,\n        f\"{x:.1%}\",\n        va=\"center\"\n    )\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.143681Z","iopub.execute_input":"2026-08-22T04:57:18.14437Z","iopub.status.idle":"2026-08-22T04:57:18.370151Z","shell.execute_reply.started":"2026-08-22T04:57:18.144327Z","shell.execute_reply":"2026-08-22T04:57:18.368934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(\n    label_stats[\n        [\"Label\", \"Labeled\", \"Positive\", \"Negative\", \"Positive prevalence\"]\n    ].sort_values(\"Positive prevalence\", ascending=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.371751Z","iopub.execute_input":"2026-08-22T04:57:18.372122Z","iopub.status.idle":"2026-08-22T04:57:18.384362Z","shell.execute_reply.started":"2026-08-22T04:57:18.372086Z","shell.execute_reply":"2026-08-22T04:57:18.383467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if len(gold) >= 2:\n    corr = gold[LABELS].astype(float).corr()\n\n    fig, ax = plt.subplots(figsize=(10, 9))\n\n    im = ax.imshow(\n        corr,\n        vmin=-1,\n        vmax=1,\n        cmap=\"coolwarm\"\n    )\n\n    ax.set_xticks(range(len(LABELS)))\n    ax.set_yticks(range(len(LABELS)))\n\n    ax.set_xticklabels(LABELS, rotation=90)\n    ax.set_yticklabels(LABELS)\n\n    fig.colorbar(im, ax=ax, label=\"Correlation\")\n\n    ax.set_title(\"Label correlation\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.385615Z","iopub.execute_input":"2026-08-22T04:57:18.386812Z","iopub.status.idle":"2026-08-22T04:57:18.711428Z","shell.execute_reply.started":"2026-08-22T04:57:18.386781Z","shell.execute_reply":"2026-08-22T04:57:18.710419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plane_counts = (\n    train_series[\"Anatomical_Plane\"]\n    .value_counts(dropna=False)\n)\n\ndisplay(plane_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.712839Z","iopub.execute_input":"2026-08-22T04:57:18.713297Z","iopub.status.idle":"2026-08-22T04:57:18.722629Z","shell.execute_reply.started":"2026-08-22T04:57:18.713259Z","shell.execute_reply":"2026-08-22T04:57:18.72159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(6, 4))\n\nax.bar(\n    plane_counts.index.astype(str),\n    plane_counts.values\n)\n\nax.set_xlabel(\"Anatomical plane\")\nax.set_ylabel(\"Number of series\")\nax.set_title(\"Series by anatomical plane\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.723764Z","iopub.execute_input":"2026-08-22T04:57:18.724121Z","iopub.status.idle":"2026-08-22T04:57:18.8403Z","shell.execute_reply.started":"2026-08-22T04:57:18.724087Z","shell.execute_reply":"2026-08-22T04:57:18.839355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_table = pd.crosstab(\n    train_series[\"Fluid_Sensitive\"],\n    train_series[\"Fat_Suppression\"],\n    margins=True\n)\n\ndisplay(sequence_table)\n\ndisplay(\n    pd.crosstab(\n        train_series[\"Fluid_Sensitive\"],\n        train_series[\"Fat_Suppression\"],\n        normalize=\"all\"\n    ).round(3)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.841474Z","iopub.execute_input":"2026-08-22T04:57:18.841731Z","iopub.status.idle":"2026-08-22T04:57:18.890611Z","shell.execute_reply.started":"2026-08-22T04:57:18.841709Z","shell.execute_reply":"2026-08-22T04:57:18.8898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plane_sequence = (\n    train_series\n    .groupby(\n        [\n            \"Anatomical_Plane\",\n            \"Fluid_Sensitive\",\n            \"Fat_Suppression\"\n        ],\n        dropna=False\n    )\n    .size()\n    .rename(\"count\")\n    .reset_index()\n)\n\ndisplay(plane_sequence)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.891686Z","iopub.execute_input":"2026-08-22T04:57:18.892088Z","iopub.status.idle":"2026-08-22T04:57:18.910289Z","shell.execute_reply.started":"2026-08-22T04:57:18.89202Z","shell.execute_reply":"2026-08-22T04:57:18.909131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pivot = pd.crosstab(\n    train_series[\"Anatomical_Plane\"],\n    [\n        train_series[\"Fluid_Sensitive\"],\n        train_series[\"Fat_Suppression\"]\n    ]\n)\n\ndisplay(pivot)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.911605Z","iopub.execute_input":"2026-08-22T04:57:18.911873Z","iopub.status.idle":"2026-08-22T04:57:18.93539Z","shell.execute_reply.started":"2026-08-22T04:57:18.91185Z","shell.execute_reply":"2026-08-22T04:57:18.934404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ax = pivot.plot(\n    kind=\"bar\",\n    figsize=(9, 5)\n)\n\nax.set_ylabel(\"Number of series\")\nax.set_title(\n    \"Sequence characteristics by anatomical plane\"\n)\n\nplt.xticks(rotation=0)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:18.936565Z","iopub.execute_input":"2026-08-22T04:57:18.936925Z","iopub.status.idle":"2026-08-22T04:57:19.117275Z","shell.execute_reply.started":"2026-08-22T04:57:18.936893Z","shell.execute_reply":"2026-08-22T04:57:19.116364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"planes_per_study = (\n    train_series\n    .groupby(\"StudyInstanceUID\")[\"Anatomical_Plane\"]\n    .nunique()\n)\n\ndisplay(\n    planes_per_study\n    .value_counts()\n    .sort_index()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.1184Z","iopub.execute_input":"2026-08-22T04:57:19.118759Z","iopub.status.idle":"2026-08-22T04:57:19.13606Z","shell.execute_reply.started":"2026-08-22T04:57:19.118724Z","shell.execute_reply":"2026-08-22T04:57:19.135079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plane_presence = (\n    train_series\n    .assign(value=1)\n    .pivot_table(\n        index=\"StudyInstanceUID\",\n        columns=\"Anatomical_Plane\",\n        values=\"value\",\n        aggfunc=\"max\",\n        fill_value=0\n    )\n)\n\ndisplay(plane_presence.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.137181Z","iopub.execute_input":"2026-08-22T04:57:19.137445Z","iopub.status.idle":"2026-08-22T04:57:19.167943Z","shell.execute_reply.started":"2026-08-22T04:57:19.137422Z","shell.execute_reply":"2026-08-22T04:57:19.167115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n    \"All three planes:\",\n    (\n        plane_presence.reindex(\n            columns=[\"Sagittal\", \"Coronal\", \"Axial\"],\n            fill_value=0\n        ).sum(axis=1) == 3\n    ).mean()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.169198Z","iopub.execute_input":"2026-08-22T04:57:19.169542Z","iopub.status.idle":"2026-08-22T04:57:19.177598Z","shell.execute_reply.started":"2026-08-22T04:57:19.169509Z","shell.execute_reply":"2026-08-22T04:57:19.176581Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Report missing:\", train[\"Report\"].isna().sum())\n\ntrain[\"report_chars\"] = (\n    train[\"Report\"]\n    .fillna(\"\")\n    .str.len()\n)\n\ntrain[\"report_words\"] = (\n    train[\"Report\"]\n    .fillna(\"\")\n    .str.split()\n    .str.len()\n)\n\ndisplay(\n    train[\n        [\"report_chars\", \"report_words\"]\n    ].describe()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.178946Z","iopub.execute_input":"2026-08-22T04:57:19.179412Z","iopub.status.idle":"2026-08-22T04:57:19.309304Z","shell.execute_reply.started":"2026-08-22T04:57:19.179385Z","shell.execute_reply":"2026-08-22T04:57:19.308017Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 4))\n\nx = train.loc[\n    train[\"report_chars\"] > 0,\n    \"report_chars\"\n]\n\nax.hist(\n    x,\n    bins=50\n)\n\nax.set_xlabel(\"Report length [characters]\")\nax.set_ylabel(\"Number of studies\")\nax.set_title(\"Radiology report length\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.31042Z","iopub.execute_input":"2026-08-22T04:57:19.310875Z","iopub.status.idle":"2026-08-22T04:57:19.509099Z","shell.execute_reply.started":"2026-08-22T04:57:19.310848Z","shell.execute_reply":"2026-08-22T04:57:19.508133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_reports = (\n    train.loc[\n        train[\"Report\"].notna(),\n        [\"StudyInstanceUID\", \"Report\"]\n    ]\n    .sample(\n        min(5, train[\"Report\"].notna().sum()),\n        random_state=SEED\n    )\n)\n\nfor _, row in sample_reports.iterrows():\n    print(\"=\" * 80)\n    print(\"Study:\", row[\"StudyInstanceUID\"])\n    print(str(row[\"Report\"])[:1000])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.514366Z","iopub.execute_input":"2026-08-22T04:57:19.514659Z","iopub.status.idle":"2026-08-22T04:57:19.527885Z","shell.execute_reply.started":"2026-08-22T04:57:19.514635Z","shell.execute_reply":"2026-08-22T04:57:19.526856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dominant_script(text):\n    if not isinstance(text, str) or not text.strip():\n        return \"Empty\"\n\n    counts = {\n        \"Latin\": 0,\n        \"CJK\": 0,\n        \"Cyrillic\": 0,\n        \"Arabic\": 0,\n        \"Other\": 0,\n    }\n\n    for ch in text:\n        if not ch.isalpha():\n            continue\n\n        code = ord(ch)\n\n        if (\n            0x0041 <= code <= 0x024F\n            or 0x1E00 <= code <= 0x1EFF\n        ):\n            counts[\"Latin\"] += 1\n\n        elif (\n            0x3040 <= code <= 0x30FF\n            or 0x3400 <= code <= 0x9FFF\n            or 0xAC00 <= code <= 0xD7AF\n        ):\n            counts[\"CJK\"] += 1\n\n        elif 0x0400 <= code <= 0x052F:\n            counts[\"Cyrillic\"] += 1\n\n        elif 0x0600 <= code <= 0x06FF:\n            counts[\"Arabic\"] += 1\n\n        else:\n            counts[\"Other\"] += 1\n\n    total = sum(counts.values())\n\n    if total == 0:\n        return \"Other\"\n\n    script = max(counts, key=counts.get)\n\n    if counts[script] / total < 0.7:\n        return \"Mixed\"\n\n    return script\n\n\ntrain[\"report_script\"] = (\n    train[\"Report\"]\n    .fillna(\"\")\n    .map(dominant_script)\n)\n\ndisplay(\n    train[\"report_script\"]\n    .value_counts()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:19.52931Z","iopub.execute_input":"2026-08-22T04:57:19.529699Z","iopub.status.idle":"2026-08-22T04:57:20.33065Z","shell.execute_reply.started":"2026-08-22T04:57:19.52966Z","shell.execute_reply":"2026-08-22T04:57:20.329747Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"script_counts = train[\"report_script\"].value_counts()\n\nfig, ax = plt.subplots(figsize=(7, 4))\n\nax.bar(\n    script_counts.index,\n    script_counts.values\n)\n\nax.set_ylabel(\"Number of reports\")\nax.set_title(\"Dominant writing system in reports\")\n\nplt.xticks(rotation=30)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:20.332427Z","iopub.execute_input":"2026-08-22T04:57:20.332761Z","iopub.status.idle":"2026-08-22T04:57:20.461417Z","shell.execute_reply.started":"2026-08-22T04:57:20.332735Z","shell.execute_reply":"2026-08-22T04:57:20.460247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_sample = min(\n    N_META_SERIES,\n    len(train_series)\n)\n\nsampled_series = train_series.sample(\n    n=n_sample,\n    random_state=SEED\n).copy()\n\nprint(\"Sampled series:\", len(sampled_series))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:20.462693Z","iopub.execute_input":"2026-08-22T04:57:20.463101Z","iopub.status.idle":"2026-08-22T04:57:20.470875Z","shell.execute_reply.started":"2026-08-22T04:57:20.463065Z","shell.execute_reply":"2026-08-22T04:57:20.469775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def safe_value(ds, name, default=np.nan):\n    try:\n        value = getattr(ds, name)\n        return value\n    except Exception:\n        return default\n\n\ndef first_dicom_in_series(study_uid, series_uid):\n    series_dir = (\n        TRAIN_SERIES_DIR\n        / str(study_uid)\n        / str(series_uid)\n    )\n\n    files = list(series_dir.glob(\"*.dcm\"))\n\n    if len(files) == 0:\n        return None\n\n    return files[0]\n\n\ndef read_series_metadata(row):\n    study_uid = row[\"StudyInstanceUID\"]\n    series_uid = row[\"SeriesInstanceUID\"]\n\n    series_dir = (\n        TRAIN_SERIES_DIR\n        / str(study_uid)\n        / str(series_uid)\n    )\n\n    dcm_files = list(series_dir.glob(\"*.dcm\"))\n\n    result = {\n        \"StudyInstanceUID\": study_uid,\n        \"SeriesInstanceUID\": series_uid,\n        \"Anatomical_Plane\": row[\"Anatomical_Plane\"],\n        \"Fluid_Sensitive\": row[\"Fluid_Sensitive\"],\n        \"Fat_Suppression\": row[\"Fat_Suppression\"],\n        \"n_slices\": len(dcm_files),\n    }\n\n    if not dcm_files:\n        return result\n\n    try:\n        ds = pydicom.dcmread(\n            str(dcm_files[0]),\n            stop_before_pixels=True\n        )\n\n        result.update({\n            \"Rows\": safe_value(ds, \"Rows\"),\n            \"Columns\": safe_value(ds, \"Columns\"),\n            \"SliceThickness\": safe_value(ds, \"SliceThickness\"),\n            \"SpacingBetweenSlices\":\n                safe_value(ds, \"SpacingBetweenSlices\"),\n            \"PixelSpacing\":\n                str(safe_value(ds, \"PixelSpacing\", \"\")),\n            \"MagneticFieldStrength\":\n                safe_value(ds, \"MagneticFieldStrength\"),\n            \"Manufacturer\":\n                safe_value(ds, \"Manufacturer\", \"\"),\n            \"ManufacturerModelName\":\n                safe_value(ds, \"ManufacturerModelName\", \"\"),\n            \"RepetitionTime\":\n                safe_value(ds, \"RepetitionTime\"),\n            \"EchoTime\":\n                safe_value(ds, \"EchoTime\"),\n            \"BitsStored\":\n                safe_value(ds, \"BitsStored\"),\n            \"PhotometricInterpretation\":\n                safe_value(ds, \"PhotometricInterpretation\", \"\"),\n            \"TransferSyntaxUID\":\n                str(\n                    getattr(\n                        ds.file_meta,\n                        \"TransferSyntaxUID\",\n                        \"\"\n                    )\n                ),\n        })\n\n    except Exception as e:\n        result[\"error\"] = str(e)\n\n    return result","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:20.472242Z","iopub.execute_input":"2026-08-22T04:57:20.472591Z","iopub.status.idle":"2026-08-22T04:57:20.490677Z","shell.execute_reply.started":"2026-08-22T04:57:20.472552Z","shell.execute_reply":"2026-08-22T04:57:20.489321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"metadata_rows = []\n\nfor _, row in sampled_series.iterrows():\n    metadata_rows.append(\n        read_series_metadata(row)\n    )\n\ndicom_meta = pd.DataFrame(metadata_rows)\n\ndisplay(dicom_meta.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:20.492095Z","iopub.execute_input":"2026-08-22T04:57:20.492449Z","iopub.status.idle":"2026-08-22T04:57:21.345412Z","shell.execute_reply.started":"2026-08-22T04:57:20.492413Z","shell.execute_reply":"2026-08-22T04:57:21.344566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(\n    dicom_meta[\"n_slices\"].describe()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.346442Z","iopub.execute_input":"2026-08-22T04:57:21.346762Z","iopub.status.idle":"2026-08-22T04:57:21.357134Z","shell.execute_reply.started":"2026-08-22T04:57:21.346737Z","shell.execute_reply":"2026-08-22T04:57:21.356184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 4))\n\nax.hist(\n    dicom_meta[\"n_slices\"],\n    bins=40\n)\n\nax.set_xlabel(\"Slices per series\")\nax.set_ylabel(\"Number of sampled series\")\nax.set_title(\"Number of DICOM slices per series\")\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.358649Z","iopub.execute_input":"2026-08-22T04:57:21.359069Z","iopub.status.idle":"2026-08-22T04:57:21.544607Z","shell.execute_reply.started":"2026-08-22T04:57:21.359011Z","shell.execute_reply":"2026-08-22T04:57:21.543678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(\n    dicom_meta.sort_values(\n        \"n_slices\",\n        ascending=False\n    )[\n        [\n            \"StudyInstanceUID\",\n            \"SeriesInstanceUID\",\n            \"Anatomical_Plane\",\n            \"n_slices\"\n        ]\n    ].head(20)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.545625Z","iopub.execute_input":"2026-08-22T04:57:21.545978Z","iopub.status.idle":"2026-08-22T04:57:21.558996Z","shell.execute_reply.started":"2026-08-22T04:57:21.545951Z","shell.execute_reply":"2026-08-22T04:57:21.558113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"matrix_counts = (\n    dicom_meta\n    .dropna(subset=[\"Rows\", \"Columns\"])\n    .groupby([\"Rows\", \"Columns\"])\n    .size()\n    .sort_values(ascending=False)\n)\n\ndisplay(matrix_counts.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.56027Z","iopub.execute_input":"2026-08-22T04:57:21.560692Z","iopub.status.idle":"2026-08-22T04:57:21.578852Z","shell.execute_reply.started":"2026-08-22T04:57:21.560658Z","shell.execute_reply":"2026-08-22T04:57:21.578101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top = matrix_counts.head(15)\n\nlabels = [\n    f\"{int(r)}×{int(c)}\"\n    for r, c in top.index\n]\n\nfig, ax = plt.subplots(figsize=(10, 5))\n\nax.bar(\n    labels,\n    top.values\n)\n\nax.set_ylabel(\"Number of sampled series\")\nax.set_xlabel(\"Matrix size\")\nax.set_title(\"Common MRI matrix sizes\")\n\nplt.xticks(rotation=45)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.579823Z","iopub.execute_input":"2026-08-22T04:57:21.580137Z","iopub.status.idle":"2026-08-22T04:57:21.812205Z","shell.execute_reply.started":"2026-08-22T04:57:21.580113Z","shell.execute_reply":"2026-08-22T04:57:21.811106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thickness = pd.to_numeric(\n    dicom_meta[\"SliceThickness\"],\n    errors=\"coerce\"\n).dropna()\n\nif len(thickness):\n    display(thickness.describe())\n\n    fig, ax = plt.subplots(figsize=(8, 4))\n\n    ax.hist(\n        thickness,\n        bins=30\n    )\n\n    ax.set_xlabel(\"Slice thickness [mm]\")\n    ax.set_ylabel(\"Number of sampled series\")\n    ax.set_title(\"Slice thickness distribution\")\n\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.813525Z","iopub.execute_input":"2026-08-22T04:57:21.813823Z","iopub.status.idle":"2026-08-22T04:57:21.977927Z","shell.execute_reply.started":"2026-08-22T04:57:21.813799Z","shell.execute_reply":"2026-08-22T04:57:21.977137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if \"Manufacturer\" in dicom_meta:\n    manufacturer_counts = (\n        dicom_meta[\"Manufacturer\"]\n        .replace(\"\", np.nan)\n        .dropna()\n        .value_counts()\n    )\n\n    display(manufacturer_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.979162Z","iopub.execute_input":"2026-08-22T04:57:21.97951Z","iopub.status.idle":"2026-08-22T04:57:21.989307Z","shell.execute_reply.started":"2026-08-22T04:57:21.979476Z","shell.execute_reply":"2026-08-22T04:57:21.988399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if len(manufacturer_counts):\n    fig, ax = plt.subplots(figsize=(9, 4))\n\n    ax.bar(\n        manufacturer_counts.index.astype(str),\n        manufacturer_counts.values\n    )\n\n    ax.set_ylabel(\"Number of sampled series\")\n    ax.set_title(\"Scanner manufacturer\")\n\n    plt.xticks(rotation=45, ha=\"right\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:21.990762Z","iopub.execute_input":"2026-08-22T04:57:21.991543Z","iopub.status.idle":"2026-08-22T04:57:22.189441Z","shell.execute_reply.started":"2026-08-22T04:57:21.991514Z","shell.execute_reply":"2026-08-22T04:57:22.188454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transfer_counts = (\n    dicom_meta[\"TransferSyntaxUID\"]\n    .replace(\"\", np.nan)\n    .dropna()\n    .value_counts()\n)\n\ndisplay(transfer_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:22.190701Z","iopub.execute_input":"2026-08-22T04:57:22.191078Z","iopub.status.idle":"2026-08-22T04:57:22.199936Z","shell.execute_reply.started":"2026-08-22T04:57:22.191019Z","shell.execute_reply":"2026-08-22T04:57:22.199198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_sorted_dicom_paths(series_dir):\n    paths = list(Path(series_dir).glob(\"*.dcm\"))\n\n    records = []\n\n    tags = [\n        \"InstanceNumber\",\n        \"ImagePositionPatient\",\n        \"ImageOrientationPatient\",\n    ]\n\n    for path in paths:\n        try:\n            ds = pydicom.dcmread(\n                str(path),\n                stop_before_pixels=True,\n                specific_tags=tags\n            )\n\n            instance = np.nan\n            position = np.nan\n\n            try:\n                instance = float(ds.InstanceNumber)\n            except Exception:\n                pass\n\n            try:\n                ipp = np.asarray(\n                    ds.ImagePositionPatient,\n                    dtype=float\n                )\n\n                iop = np.asarray(\n                    ds.ImageOrientationPatient,\n                    dtype=float\n                )\n\n                row = iop[:3]\n                col = iop[3:]\n\n                normal = np.cross(row, col)\n\n                position = float(\n                    np.dot(ipp, normal)\n                )\n\n            except Exception:\n                pass\n\n            records.append(\n                {\n                    \"path\": path,\n                    \"instance\": instance,\n                    \"position\": position,\n                }\n            )\n\n        except Exception:\n            records.append(\n                {\n                    \"path\": path,\n                    \"instance\": np.nan,\n                    \"position\": np.nan,\n                }\n            )\n\n    df = pd.DataFrame(records)\n\n    if len(df) == 0:\n        return []\n\n    if df[\"position\"].notna().sum() >= max(2, len(df) // 2):\n        df = df.sort_values(\n            [\"position\", \"instance\"]\n        )\n\n    elif df[\"instance\"].notna().sum() >= max(2, len(df) // 2):\n        df = df.sort_values(\"instance\")\n\n    else:\n        df = df.sort_values(\n            \"path\",\n            key=lambda s: s.astype(str)\n        )\n\n    return df[\"path\"].tolist()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:22.201325Z","iopub.execute_input":"2026-08-22T04:57:22.201705Z","iopub.status.idle":"2026-08-22T04:57:22.216865Z","shell.execute_reply.started":"2026-08-22T04:57:22.201668Z","shell.execute_reply":"2026-08-22T04:57:22.215802Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def dicom_to_image(ds):\n    \"\"\"\n    DICOM pixel arrayをEDA表示用の0-1画像へ変換する。\n    学習時のpreprocessingとは別物。\n    \"\"\"\n\n    arr = ds.pixel_array.astype(np.float32)\n\n    slope = float(\n        getattr(ds, \"RescaleSlope\", 1.0)\n    )\n\n    intercept = float(\n        getattr(ds, \"RescaleIntercept\", 0.0)\n    )\n\n    arr = arr * slope + intercept\n\n    valid = arr[np.isfinite(arr)]\n\n    if len(valid) == 0:\n        return np.zeros_like(arr)\n\n    lo, hi = np.percentile(\n        valid,\n        [1, 99]\n    )\n\n    if hi <= lo:\n        lo = valid.min()\n        hi = valid.max()\n\n    if hi > lo:\n        arr = np.clip(arr, lo, hi)\n        arr = (arr - lo) / (hi - lo)\n    else:\n        arr = np.zeros_like(arr)\n\n    if (\n        getattr(\n            ds,\n            \"PhotometricInterpretation\",\n            \"\"\n        ) == \"MONOCHROME1\"\n    ):\n        arr = 1.0 - arr\n\n    return arr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:22.218173Z","iopub.execute_input":"2026-08-22T04:57:22.218937Z","iopub.status.idle":"2026-08-22T04:57:22.238276Z","shell.execute_reply.started":"2026-08-22T04:57:22.218881Z","shell.execute_reply":"2026-08-22T04:57:22.237226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_series(\n    study_uid,\n    series_uid,\n    n_slices=7,\n    figsize=(15, 4)\n):\n    series_dir = (\n        TRAIN_SERIES_DIR\n        / str(study_uid)\n        / str(series_uid)\n    )\n\n    paths = get_sorted_dicom_paths(\n        series_dir\n    )\n\n    if len(paths) == 0:\n        print(\"No DICOM files found\")\n        return\n\n    indices = np.linspace(\n        0,\n        len(paths) - 1,\n        min(n_slices, len(paths))\n    ).round().astype(int)\n\n    fig, axes = plt.subplots(\n        1,\n        len(indices),\n        figsize=figsize\n    )\n\n    axes = np.atleast_1d(axes)\n\n    for ax, idx in zip(axes, indices):\n        path = paths[idx]\n\n        try:\n            ds = pydicom.dcmread(str(path))\n            img = dicom_to_image(ds)\n\n            ax.imshow(\n                img,\n                cmap=\"gray\"\n            )\n\n            instance = getattr(\n                ds,\n                \"InstanceNumber\",\n                \"?\"\n            )\n\n            ax.set_title(\n                f\"{idx + 1}/{len(paths)}\\n\"\n                f\"Inst={instance}\"\n            )\n\n        except Exception as e:\n            ax.text(\n                0.5,\n                0.5,\n                f\"Decode error\\n{type(e).__name__}\",\n                ha=\"center\",\n                va=\"center\"\n            )\n\n        ax.axis(\"off\")\n\n    plt.suptitle(\n        f\"Study: {study_uid}\\n\"\n        f\"Series: {series_uid}\"\n    )\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:22.239656Z","iopub.execute_input":"2026-08-22T04:57:22.240016Z","iopub.status.idle":"2026-08-22T04:57:22.256737Z","shell.execute_reply.started":"2026-08-22T04:57:22.239971Z","shell.execute_reply":"2026-08-22T04:57:22.255654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example = train_series.sample(\n    1,\n    random_state=SEED\n).iloc[0]\n\nprint(example)\n\nshow_series(\n    example[\"StudyInstanceUID\"],\n    example[\"SeriesInstanceUID\"],\n    n_slices=N_IMAGE_SLICES\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:22.257945Z","iopub.execute_input":"2026-08-22T04:57:22.258396Z","iopub.status.idle":"2026-08-22T04:57:24.236272Z","shell.execute_reply.started":"2026-08-22T04:57:22.258356Z","shell.execute_reply":"2026-08-22T04:57:24.235107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_plane_count = (\n    train_series\n    .groupby(\"StudyInstanceUID\")[\"Anatomical_Plane\"]\n    .nunique()\n)\n\nthree_plane_studies = study_plane_count[\n    study_plane_count >= 3\n].index\n\nprint(\n    \"Studies with >=3 planes:\",\n    len(three_plane_studies)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:24.23742Z","iopub.execute_input":"2026-08-22T04:57:24.237739Z","iopub.status.idle":"2026-08-22T04:57:24.255574Z","shell.execute_reply.started":"2026-08-22T04:57:24.237712Z","shell.execute_reply":"2026-08-22T04:57:24.254491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def middle_slice_of_series(study_uid, series_uid):\n    series_dir = (\n        TRAIN_SERIES_DIR\n        / str(study_uid)\n        / str(series_uid)\n    )\n\n    paths = get_sorted_dicom_paths(\n        series_dir\n    )\n\n    if not paths:\n        return None, None\n\n    path = paths[len(paths) // 2]\n\n    ds = pydicom.dcmread(str(path))\n    img = dicom_to_image(ds)\n\n    return img, ds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:24.256732Z","iopub.execute_input":"2026-08-22T04:57:24.257422Z","iopub.status.idle":"2026-08-22T04:57:24.263274Z","shell.execute_reply.started":"2026-08-22T04:57:24.257384Z","shell.execute_reply":"2026-08-22T04:57:24.262353Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_study_planes(study_uid):\n    rows = train_series.loc[\n        train_series[\"StudyInstanceUID\"]\n        == study_uid\n    ].copy()\n\n    planes = [\n        \"Sagittal\",\n        \"Coronal\",\n        \"Axial\"\n    ]\n\n    fig, axes = plt.subplots(\n        1,\n        3,\n        figsize=(12, 4)\n    )\n\n    for ax, plane in zip(axes, planes):\n        candidates = rows.loc[\n            rows[\"Anatomical_Plane\"] == plane\n        ]\n\n        if len(candidates) == 0:\n            ax.text(\n                0.5,\n                0.5,\n                \"No series\",\n                ha=\"center\",\n                va=\"center\"\n            )\n            ax.set_title(plane)\n            ax.axis(\"off\")\n            continue\n\n        # Fluid-sensitiveを優先して代表seriesを選ぶ\n        candidates = candidates.sort_values(\n            [\n                \"Fluid_Sensitive\",\n                \"Fat_Suppression\"\n            ],\n            ascending=False\n        )\n\n        row = candidates.iloc[0]\n\n        try:\n            img, ds = middle_slice_of_series(\n                study_uid,\n                row[\"SeriesInstanceUID\"]\n            )\n\n            ax.imshow(\n                img,\n                cmap=\"gray\"\n            )\n\n            ax.set_title(\n                f\"{plane}\\n\"\n                f\"Fluid={row['Fluid_Sensitive']}, \"\n                f\"FS={row['Fat_Suppression']}\"\n            )\n\n        except Exception as e:\n            ax.text(\n                0.5,\n                0.5,\n                str(e)[:80],\n                ha=\"center\",\n                va=\"center\"\n            )\n            ax.set_title(plane)\n\n        ax.axis(\"off\")\n\n    plt.suptitle(\n        f\"Study {study_uid}\"\n    )\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:24.264614Z","iopub.execute_input":"2026-08-22T04:57:24.264961Z","iopub.status.idle":"2026-08-22T04:57:24.286795Z","shell.execute_reply.started":"2026-08-22T04:57:24.264928Z","shell.execute_reply":"2026-08-22T04:57:24.28591Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if len(three_plane_studies):\n\n    demo_study = pd.Series(\n        three_plane_studies\n    ).sample(\n        1,\n        random_state=SEED\n    ).iloc[0]\n\n    print(\"Demo study:\", demo_study)\n\n    show_study_planes(\n        demo_study\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:24.288111Z","iopub.execute_input":"2026-08-22T04:57:24.28848Z","iopub.status.idle":"2026-08-22T04:57:25.406593Z","shell.execute_reply.started":"2026-08-22T04:57:24.288455Z","shell.execute_reply":"2026-08-22T04:57:25.405258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def show_positive_example(\n    label,\n    random_state=42\n):\n    if label not in LABELS:\n        raise ValueError(label)\n\n    candidates = train.loc[\n        train[label] == 1,\n        \"StudyInstanceUID\"\n    ]\n\n    if len(candidates) == 0:\n        print(\n            f\"No positive labeled study for {label}\"\n        )\n        return\n\n    study_uid = candidates.sample(\n        1,\n        random_state=random_state\n    ).iloc[0]\n\n    print(\n        f\"{label}=1\"\n    )\n\n    print(\n        \"Study:\",\n        study_uid\n    )\n\n    row = train.loc[\n        train[\"StudyInstanceUID\"]\n        == study_uid,\n        [\"StudyInstanceUID\"] + LABELS\n    ]\n\n    display(row)\n\n    show_study_planes(\n        study_uid\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:25.407841Z","iopub.execute_input":"2026-08-22T04:57:25.408205Z","iopub.status.idle":"2026-08-22T04:57:25.415172Z","shell.execute_reply.started":"2026-08-22T04:57:25.40817Z","shell.execute_reply":"2026-08-22T04:57:25.414175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_positive_example(\"ACL\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:25.416486Z","iopub.execute_input":"2026-08-22T04:57:25.417219Z","iopub.status.idle":"2026-08-22T04:57:26.029129Z","shell.execute_reply.started":"2026-08-22T04:57:25.417193Z","shell.execute_reply":"2026-08-22T04:57:26.028092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_positive_example(\"Medial Meniscus\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:26.030389Z","iopub.execute_input":"2026-08-22T04:57:26.030737Z","iopub.status.idle":"2026-08-22T04:57:26.741849Z","shell.execute_reply.started":"2026-08-22T04:57:26.030701Z","shell.execute_reply":"2026-08-22T04:57:26.740774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_positive_example(\"Effusion\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:26.743261Z","iopub.execute_input":"2026-08-22T04:57:26.743622Z","iopub.status.idle":"2026-08-22T04:57:27.324334Z","shell.execute_reply.started":"2026-08-22T04:57:26.743586Z","shell.execute_reply":"2026-08-22T04:57:27.322785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"show_positive_example(\"Fracture\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.325566Z","iopub.execute_input":"2026-08-22T04:57:27.325923Z","iopub.status.idle":"2026-08-22T04:57:27.875253Z","shell.execute_reply.started":"2026-08-22T04:57:27.325883Z","shell.execute_reply":"2026-08-22T04:57:27.874344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def inspect_study(study_uid):\n    print(\"=\" * 80)\n    print(\"Study:\", study_uid)\n\n    train_row = train.loc[\n        train[\"StudyInstanceUID\"]\n        == study_uid\n    ]\n\n    print(\"\\nStudy-level information:\")\n    display(\n        train_row[\n            [\n                \"StudyInstanceUID\",\n                \"Report\"\n            ] + LABELS\n        ]\n    )\n\n    print(\"\\nSeries:\")\n    series_rows = train_series.loc[\n        train_series[\"StudyInstanceUID\"]\n        == study_uid\n    ]\n\n    display(\n        series_rows[\n            [\n                \"SeriesInstanceUID\",\n                \"Anatomical_Plane\",\n                \"Fluid_Sensitive\",\n                \"Fat_Suppression\"\n            ]\n        ]\n    )\n\n    print(\n        \"Number of series:\",\n        len(series_rows)\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.876608Z","iopub.execute_input":"2026-08-22T04:57:27.876943Z","iopub.status.idle":"2026-08-22T04:57:27.884756Z","shell.execute_reply.started":"2026-08-22T04:57:27.87689Z","shell.execute_reply":"2026-08-22T04:57:27.883822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inspect_study(demo_study)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.885871Z","iopub.execute_input":"2026-08-22T04:57:27.886627Z","iopub.status.idle":"2026-08-22T04:57:27.923394Z","shell.execute_reply.started":"2026-08-22T04:57:27.886589Z","shell.execute_reply":"2026-08-22T04:57:27.922416Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TEST_CSV.exists():\n    test = pd.read_csv(TEST_CSV)\n\n    print(\n        \"test.shape =\",\n        test.shape\n    )\n\n    display(test.head())\n\nif TEST_SERIES_CSV.exists():\n    test_series = pd.read_csv(\n        TEST_SERIES_CSV\n    )\n\n    print(\n        \"test_series.shape =\",\n        test_series.shape\n    )\n\n    display(test_series.head())\n\nif SAMPLE_SUBMISSION_CSV.exists():\n    sample_submission = pd.read_csv(\n        SAMPLE_SUBMISSION_CSV\n    )\n\n    print(\n        \"sample_submission.shape =\",\n        sample_submission.shape\n    )\n\n    display(sample_submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.924486Z","iopub.execute_input":"2026-08-22T04:57:27.925103Z","iopub.status.idle":"2026-08-22T04:57:27.964294Z","shell.execute_reply.started":"2026-08-22T04:57:27.925028Z","shell.execute_reply":"2026-08-22T04:57:27.963443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if TEST_CSV.exists():\n\n    print(\"Train columns:\")\n    print(train.columns.tolist())\n\n    print(\"\\nTest columns:\")\n    print(test.columns.tolist())\n\n    print(\"\\nColumns only available during training:\")\n    print(\n        sorted(\n            set(train.columns)\n            - set(test.columns)\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.965353Z","iopub.execute_input":"2026-08-22T04:57:27.965736Z","iopub.status.idle":"2026-08-22T04:57:27.97239Z","shell.execute_reply.started":"2026-08-22T04:57:27.965697Z","shell.execute_reply":"2026-08-22T04:57:27.971504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"summary = {\n    \"Studies\": train[\"StudyInstanceUID\"].nunique(),\n    \"Series\": train_series[\"SeriesInstanceUID\"].nunique(),\n    \"Mean series / study\": series_per_study.mean(),\n    \"Median series / study\": series_per_study.median(),\n\n    \"Fully labeled studies\":\n        train[LABELS].notna().all(axis=1).sum(),\n\n    \"Completely unlabeled studies\":\n        train[LABELS].isna().all(axis=1).sum(),\n\n    \"Studies with Sagittal\":\n        (\n            train_series[\"Anatomical_Plane\"]\n            == \"Sagittal\"\n        ).groupby(\n            train_series[\"StudyInstanceUID\"]\n        ).any().sum(),\n\n    \"Studies with Coronal\":\n        (\n            train_series[\"Anatomical_Plane\"]\n            == \"Coronal\"\n        ).groupby(\n            train_series[\"StudyInstanceUID\"]\n        ).any().sum(),\n\n    \"Studies with Axial\":\n        (\n            train_series[\"Anatomical_Plane\"]\n            == \"Axial\"\n        ).groupby(\n            train_series[\"StudyInstanceUID\"]\n        ).any().sum(),\n}\n\nsummary_df = pd.DataFrame(\n    summary.items(),\n    columns=[\n        \"Metric\",\n        \"Value\"\n    ]\n)\n\ndisplay(summary_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:27.973658Z","iopub.execute_input":"2026-08-22T04:57:27.97402Z","iopub.status.idle":"2026-08-22T04:57:28.03174Z","shell.execute_reply.started":"2026-08-22T04:57:27.973986Z","shell.execute_reply":"2026-08-22T04:57:28.03072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(\n    label_stats\n    .sort_values(\n        \"Positive prevalence\",\n        ascending=False\n    )\n    .reset_index(drop=True)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:28.032973Z","iopub.execute_input":"2026-08-22T04:57:28.033372Z","iopub.status.idle":"2026-08-22T04:57:28.046394Z","shell.execute_reply.started":"2026-08-22T04:57:28.033336Z","shell.execute_reply":"2026-08-22T04:57:28.045399Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"このEDAで把握すべきこと\nこのNotebookを一周すると、少なくとも次の構造が明確になります。\n```\nStudyInstanceUID\n│\n├── study-level Report\n├── 12 labels（一部studyのみ）\n│\n└── 複数 SeriesInstanceUID\n      │\n      ├── Anatomical plane\n      │     ├── Sagittal\n      │     ├── Coronal\n      │     └── Axial\n      │\n      ├── Fluid sensitive\n      ├── Fat suppression\n      │\n      └── 20〜数十枚程度のDICOM slice\n```\nそしてML上は、単純な「MRI画像1枚 → 12分類」ではなく、\n```\n\\[\n\\text{multiple slices}\n\\rightarrow\n\\text{series representation}\n\\rightarrow\n\\text{multiple series / planes}\n\\rightarrow\n\\text{study representation}\n\\rightarrow\n12\\text{ labels}\n\\]\n```\nという階層型multi-label classificationであることが分かります。\nさらにこのコンペ特有の最大の問題は、画像数ではなくexpert labelが非常に少ないことです。公開されている解析でも完全な12ラベルを持つstudyは58例程度と報告されており、大部分ではReportしかありません。GitHub したがって、このEDAの次に検討すべきなのは、①Reportからのweak label生成、②study単位のCV、③plane/sequence selection、④2.5Dまたはseries-level aggregation、という順番になります。","metadata":{}},{"cell_type":"code","source":"import pandas as pd\n\nplanes = [\"Sagittal\", \"Coronal\", \"Axial\"]\nn_studies = train_series[\"StudyInstanceUID\"].nunique()\n\nrows = []\n\nfor plane in planes:\n    df = train_series[\n        train_series[\"Anatomical_Plane\"] == plane\n    ].copy()\n\n    # studyごとのseries数\n    n_series_per_study = (\n        df.groupby(\"StudyInstanceUID\")\n        .size()\n        .reindex(train_series[\"StudyInstanceUID\"].unique(), fill_value=0)\n    )\n\n    # 00: Fluid_Sensitive=0, Fat_Suppression=0\n    studies_00 = df.loc[\n        (df[\"Fluid_Sensitive\"] == 0) &\n        (df[\"Fat_Suppression\"] == 0),\n        \"StudyInstanceUID\"\n    ].nunique()\n\n    # 11: Fluid_Sensitive=1, Fat_Suppression=1\n    studies_11 = df.loc[\n        (df[\"Fluid_Sensitive\"] == 1) &\n        (df[\"Fat_Suppression\"] == 1),\n        \"StudyInstanceUID\"\n    ].nunique()\n\n    # 同じplaneに2 series以上あるstudy\n    multiple_series = (n_series_per_study >= 2).sum()\n\n    # 00 slot内に2 series以上あるstudy\n    n_00 = (\n        df[\n            (df[\"Fluid_Sensitive\"] == 0) &\n            (df[\"Fat_Suppression\"] == 0)\n        ]\n        .groupby(\"StudyInstanceUID\")\n        .size()\n    )\n    duplicate_00 = (n_00 >= 2).sum()\n\n    # 11 slot内に2 series以上あるstudy\n    n_11 = (\n        df[\n            (df[\"Fluid_Sensitive\"] == 1) &\n            (df[\"Fat_Suppression\"] == 1)\n        ]\n        .groupby(\"StudyInstanceUID\")\n        .size()\n    )\n    duplicate_11 = (n_11 >= 2).sum()\n\n    rows.append({\n        \"Plane\": plane,\n        \"00存在率\": studies_00 / n_studies,\n        \"11存在率\": studies_11 / n_studies,\n        \"plane内複数series率\": multiple_series / n_studies,\n        \"00内複数series率\": duplicate_00 / n_studies,\n        \"11内複数series率\": duplicate_11 / n_studies,\n    })\n\nslot_summary = pd.DataFrame(rows).set_index(\"Plane\")\n\ndisplay(\n    slot_summary.style.format(\"{:.1%}\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:28.047652Z","iopub.execute_input":"2026-08-22T04:57:28.048026Z","iopub.status.idle":"2026-08-22T04:57:28.145694Z","shell.execute_reply.started":"2026-08-22T04:57:28.047994Z","shell.execute_reply":"2026-08-22T04:57:28.144511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for plane in planes:\n    print(f\"\\n===== {plane} =====\")\n\n    tmp = (\n        train_series[\n            train_series[\"Anatomical_Plane\"] == plane\n        ]\n        .groupby(\n            [\n                \"StudyInstanceUID\",\n                \"Fluid_Sensitive\",\n                \"Fat_Suppression\",\n            ]\n        )\n        .size()\n        .rename(\"n_series\")\n    )\n\n    print(tmp.groupby(\n        [\"Fluid_Sensitive\", \"Fat_Suppression\"]\n    ).describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-22T04:57:28.146999Z","iopub.execute_input":"2026-08-22T04:57:28.147377Z","iopub.status.idle":"2026-08-22T04:57:28.220633Z","shell.execute_reply.started":"2026-08-22T04:57:28.147341Z","shell.execute_reply":"2026-08-22T04:57:28.219598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}