{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nprint(\"=\" * 70)\nprint(\"RSNA KNEE ABNORMALITY DETECTION\")\nprint(\"DATASET PATH CHECK\")\nprint(\"=\" * 70)\n\n# Kaggle competition input\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nprint(\"\\nChecking:\")\nprint(DATA_PATH)\n\nif os.path.exists(DATA_PATH):\n print(\"\\nāœ… DATASET FOUND!\")\n\n print(\"\\nDataset contents:\")\n\n for item in sorted(os.listdir(DATA_PATH)):\n full_path = os.path.join(DATA_PATH, item)\n\n if os.path.isdir(full_path):\n print(\"šŸ“\", item)\n else:\n print(\"šŸ“„\", item)\n\nelse:\n print(\"\\nāŒ Dataset path not found.\")\n\n print(\"\\nAvailable folders in /kaggle/input:\")\n for item in os.listdir(\"/kaggle/input\"):\n print(\"šŸ“\", item)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-08-07T11:50:46.06692Z","iopub.execute_input":"2026-08-07T11:50:46.067153Z","iopub.status.idle":"2026-08-07T11:50:46.084427Z","shell.execute_reply.started":"2026-08-07T11:50:46.06713Z","shell.execute_reply":"2026-08-07T11:50:46.083455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# RSNA KNEE ABNORMALITY DETECTION\n# STEP 3 - DATASET INSPECTION\n# ============================================================\n\nimport pandas as pd\nimport numpy as np\n\n# ------------------------------------------------------------\n# Dataset paths\n# ------------------------------------------------------------\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = f\"{DATA_PATH}/train.csv\"\nTRAIN_SERIES_CSV = f\"{DATA_PATH}/train_series.csv\"\nTEST_CSV = f\"{DATA_PATH}/test.csv\"\nTEST_SERIES_CSV = f\"{DATA_PATH}/test_series.csv\"\nSAMPLE_SUBMISSION = f\"{DATA_PATH}/sample_submission.csv\"\n\n\n# ------------------------------------------------------------\n# Load CSV files\n# ------------------------------------------------------------\n\nprint(\"=\" * 70)\nprint(\"LOADING DATA\")\nprint(\"=\" * 70)\n\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_series_df = pd.read_csv(TRAIN_SERIES_CSV)\n\ntest_df = pd.read_csv(TEST_CSV)\ntest_series_df = pd.read_csv(TEST_SERIES_CSV)\n\nsample_submission_df = pd.read_csv(SAMPLE_SUBMISSION)\n\nprint(\"āœ… All CSV files loaded successfully\")\n\n\n# ------------------------------------------------------------\n# Dataset shapes\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DATASET SHAPES\")\nprint(\"=\" * 70)\n\nprint(f\"train.csv : {train_df.shape}\")\nprint(f\"train_series.csv : {train_series_df.shape}\")\nprint(f\"test.csv : {test_df.shape}\")\nprint(f\"test_series.csv : {test_series_df.shape}\")\nprint(f\"sample_submission : {sample_submission_df.shape}\")\n\n\n# ------------------------------------------------------------\n# Train columns\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRAIN.CSV COLUMNS\")\nprint(\"=\" * 70)\n\nfor i, col in enumerate(train_df.columns, 1):\n print(f\"{i:2d}. {col}\")\n\n\n# ------------------------------------------------------------\n# Show train data\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FIRST 5 TRAINING STUDIES\")\nprint(\"=\" * 70)\n\ndisplay(train_df.head())\n\n\n# ------------------------------------------------------------\n# Train series columns\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRAIN_SERIES.CSV COLUMNS\")\nprint(\"=\" * 70)\n\nfor i, col in enumerate(train_series_df.columns, 1):\n print(f\"{i:2d}. {col}\")\n\n\nprint(\"\\nFirst 5 series:\")\ndisplay(train_series_df.head())\n\n\n# ------------------------------------------------------------\n# Test data\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TEST DATA\")\nprint(\"=\" * 70)\n\ndisplay(test_df.head())\n\n\n# ------------------------------------------------------------\n# Submission format\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SUBMISSION FORMAT\")\nprint(\"=\" * 70)\n\nprint(\"Submission columns:\")\n\nfor i, col in enumerate(sample_submission_df.columns, 1):\n print(f\"{i:2d}. {col}\")\n\ndisplay(sample_submission_df.head())\n\n\n# ------------------------------------------------------------\n# Target labels\n# ------------------------------------------------------------\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"12 TARGET ABNORMALITIES\")\nprint(\"=\" * 70)\n\nfor i, target in enumerate(TARGETS, 1):\n print(f\"{i:2d}. {target}\")\n\n\n# ------------------------------------------------------------\n# Verify target columns\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TARGET COLUMN VERIFICATION\")\nprint(\"=\" * 70)\n\nmissing_targets = [\n target for target in TARGETS\n if target not in train_df.columns\n]\n\nif not missing_targets:\n print(\"āœ… All 12 target columns found!\")\nelse:\n print(\"āŒ Missing columns:\")\n for target in missing_targets:\n print(\" -\", target)\n\n\n# ------------------------------------------------------------\n# Target distribution\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TARGET DISTRIBUTION\")\nprint(\"=\" * 70)\n\ntarget_stats = []\n\nfor target in TARGETS:\n\n positive = train_df[target].sum()\n total = train_df[target].notna().sum()\n negative = total - positive\n\n positive_percent = (positive / total) * 100\n\n target_stats.append({\n \"Target\": target,\n \"Positive\": int(positive),\n \"Negative\": int(negative),\n \"Positive %\": round(positive_percent, 2)\n })\n\ntarget_stats_df = pd.DataFrame(target_stats)\n\ndisplay(target_stats_df)\n\n\n# ------------------------------------------------------------\n# Missing values\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MISSING VALUES - TRAIN.CSV\")\nprint(\"=\" * 70)\n\nmissing_df = pd.DataFrame({\n \"Column\": train_df.columns,\n \"Missing\": train_df.isnull().sum().values,\n \"Missing %\": (\n train_df.isnull().mean().values * 100\n ).round(2)\n})\n\ndisplay(missing_df)\n\n\n# ------------------------------------------------------------\n# Basic information\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"BASIC INFORMATION\")\nprint(\"=\" * 70)\n\nprint(\"Training studies :\", len(train_df))\nprint(\"Training series :\", len(train_series_df))\nprint(\"Test studies :\", len(test_df))\nprint(\"Test series :\", len(test_series_df))\n\nprint(\"\\nPatient Sex distribution:\")\n\nif \"PatientSex\" in train_df.columns:\n display(train_df[\"PatientSex\"].value_counts(dropna=False))\n\n\n# ------------------------------------------------------------\n# Report availability\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"RADIOLOGY REPORT INFORMATION\")\nprint(\"=\" * 70)\n\nif \"Report\" in train_df.columns:\n\n report_missing = train_df[\"Report\"].isna().sum()\n report_available = train_df[\"Report\"].notna().sum()\n\n print(\"Reports available :\", report_available)\n print(\"Reports missing :\", report_missing)\n\n print(\"\\nExample report:\")\n \n example_report = train_df[\"Report\"].dropna().iloc[0]\n print(example_report)\n\n\n# ------------------------------------------------------------\n# Series information\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MRI SERIES INFORMATION\")\nprint(\"=\" * 70)\n\nif \"Anatomical_Plane\" in train_series_df.columns:\n\n print(\"\\nAnatomical planes:\")\n display(\n train_series_df[\"Anatomical_Plane\"]\n .value_counts(dropna=False)\n )\n\nif \"Fluid_Sensitive\" in train_series_df.columns:\n\n print(\"\\nFluid-sensitive sequences:\")\n display(\n train_series_df[\"Fluid_Sensitive\"]\n .value_counts(dropna=False)\n )\n\nif \"Fat_Suppression\" in train_series_df.columns:\n\n print(\"\\nFat-suppression:\")\n display(\n train_series_df[\"Fat_Suppression\"]\n .value_counts(dropna=False)\n )\n\n\n# ------------------------------------------------------------\n# Final\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 3 COMPLETED āœ…\")\nprint(\"=\" * 70)\n\nprint(\"\"\"\nWe have now inspected:\n\nāœ“ Training studies\nāœ“ MRI series information\nāœ“ Test studies\nāœ“ Submission format\nāœ“ 12 target abnormalities\nāœ“ Target distribution\nāœ“ Missing values\nāœ“ Patient sex\nāœ“ Radiology reports\nāœ“ MRI anatomical planes\nāœ“ Fluid-sensitive sequences\nāœ“ Fat suppression\n\"\"\")\n\nprint(\"šŸš€ Ready for STEP 4\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 4 ā€” DICOM / MRI IMAGE INSPECTION\n# RSNA Knee Abnormality Detection\n# ============================================================\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ntry:\n import pydicom\nexcept ImportError:\n !pip install -q pydicom\n import pydicom\n\n\nprint(\"=\" * 70)\nprint(\"STEP 4 ā€” DICOM / MRI IMAGE INSPECTION\")\nprint(\"=\" * 70)\n\n\n# ------------------------------------------------------------\n# 1. DATASET PATH\n# ------------------------------------------------------------\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_SERIES_DIR = os.path.join(DATA_PATH, \"train_series\")\nTEST_SERIES_DIR = os.path.join(DATA_PATH, \"test_series\")\n\nprint(\"\\nDataset path:\")\nprint(DATA_PATH)\n\nprint(\"\\nTrain series path:\")\nprint(TRAIN_SERIES_DIR)\n\nprint(\"\\nTest series path:\")\nprint(TEST_SERIES_DIR)\n\n\n# ------------------------------------------------------------\n# 2. FIND ONE TRAINING STUDY\n# ------------------------------------------------------------\n\ntrain_studies = sorted([\n x for x in os.listdir(TRAIN_SERIES_DIR)\n if os.path.isdir(os.path.join(TRAIN_SERIES_DIR, x))\n])\n\nprint(\"\\nNumber of training studies found:\", len(train_studies))\n\nif len(train_studies) == 0:\n raise FileNotFoundError(\"No training studies found!\")\n\nsample_study_uid = train_studies[0]\n\nprint(\"\\nSample StudyInstanceUID:\")\nprint(sample_study_uid)\n\n\n# ------------------------------------------------------------\n# 3. FIND SERIES INSIDE THE STUDY\n# ------------------------------------------------------------\n\nstudy_path = os.path.join(TRAIN_SERIES_DIR, sample_study_uid)\n\nseries_uids = sorted([\n x for x in os.listdir(study_path)\n if os.path.isdir(os.path.join(study_path, x))\n])\n\nprint(\"\\nNumber of series in sample study:\", len(series_uids))\n\nfor i, series_uid in enumerate(series_uids):\n print(f\"{i+1}. {series_uid}\")\n\n\n# ------------------------------------------------------------\n# 4. READ train_series.csv INFORMATION\n# ------------------------------------------------------------\n\ntrain_series_path = os.path.join(DATA_PATH, \"train_series.csv\")\n\ntrain_series_df = pd.read_csv(train_series_path)\n\nsample_series_info = train_series_df[\n train_series_df[\"StudyInstanceUID\"] == sample_study_uid\n].copy()\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SERIES INFORMATION\")\nprint(\"=\" * 70)\n\nprint(sample_series_info.to_string(index=False))\n\n\n# ------------------------------------------------------------\n# 5. FUNCTION TO FIND DICOM FILES\n# ------------------------------------------------------------\n\ndef find_dicom_files(series_path):\n \"\"\"\n Find all DICOM files inside one MRI series.\n \"\"\"\n \n files = glob.glob(\n os.path.join(series_path, \"*.dcm\")\n )\n \n # In case files are nested\n if len(files) == 0:\n files = glob.glob(\n os.path.join(series_path, \"**\", \"*.dcm\"),\n recursive=True\n )\n \n return sorted(files)\n\n\n# ------------------------------------------------------------\n# 6. INSPECT EACH SERIES\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DICOM FILE COUNTS\")\nprint(\"=\" * 70)\n\nseries_file_info = []\n\nfor series_uid in series_uids:\n\n series_path = os.path.join(\n study_path,\n series_uid\n )\n\n dcm_files = find_dicom_files(series_path)\n\n # Get metadata from first DICOM\n if len(dcm_files) > 0:\n\n try:\n ds = pydicom.dcmread(\n dcm_files[0],\n stop_before_pixels=True\n )\n\n rows = getattr(ds, \"Rows\", None)\n cols = getattr(ds, \"Columns\", None)\n modality = getattr(ds, \"Modality\", None)\n\n except Exception:\n rows = None\n cols = None\n modality = None\n\n else:\n rows = None\n cols = None\n modality = None\n\n series_file_info.append({\n \"SeriesInstanceUID\": series_uid,\n \"DICOM_Count\": len(dcm_files),\n \"Rows\": rows,\n \"Columns\": cols,\n \"Modality\": modality\n })\n\n\nseries_file_df = pd.DataFrame(series_file_info)\n\nprint(series_file_df.to_string(index=False))\n\n\n# ------------------------------------------------------------\n# 7. SELECT A SERIES WITH DICOM FILES\n# ------------------------------------------------------------\n\nvalid_series = series_file_df[\n series_file_df[\"DICOM_Count\"] > 0\n]\n\nif len(valid_series) == 0:\n raise FileNotFoundError(\n \"No DICOM files found in sample study.\"\n )\n\nselected_series_uid = valid_series.iloc[0][\"SeriesInstanceUID\"]\n\nselected_series_path = os.path.join(\n study_path,\n selected_series_uid\n)\n\ndcm_files = find_dicom_files(selected_series_path)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SELECTED SERIES\")\nprint(\"=\" * 70)\n\nprint(\"Series UID:\")\nprint(selected_series_uid)\n\nprint(\"\\nNumber of DICOM slices:\")\nprint(len(dcm_files))\n\n\n# ------------------------------------------------------------\n# 8. READ ONE DICOM IMAGE\n# ------------------------------------------------------------\n\nsample_dcm_path = dcm_files[len(dcm_files) // 2]\n\nprint(\"\\nReading middle slice:\")\nprint(sample_dcm_path)\n\nds = pydicom.dcmread(sample_dcm_path)\n\nprint(\"\\nDICOM basic information:\")\nprint(\"--------------------------------\")\n\nprint(\"Modality :\", getattr(ds, \"Modality\", \"N/A\"))\nprint(\"Rows :\", getattr(ds, \"Rows\", \"N/A\"))\nprint(\"Columns :\", getattr(ds, \"Columns\", \"N/A\"))\nprint(\"Pixel Spacing :\", getattr(ds, \"PixelSpacing\", \"N/A\"))\nprint(\"Slice Thickness:\", getattr(ds, \"SliceThickness\", \"N/A\"))\nprint(\"Instance Number:\", getattr(ds, \"InstanceNumber\", \"N/A\"))\n\n\n# ------------------------------------------------------------\n# 9. EXTRACT PIXEL ARRAY\n# ------------------------------------------------------------\n\nimage = ds.pixel_array.astype(np.float32)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"IMAGE ARRAY INFORMATION\")\nprint(\"=\" * 70)\n\nprint(\"Shape :\", image.shape)\nprint(\"Dtype :\", image.dtype)\nprint(\"Min :\", image.min())\nprint(\"Max :\", image.max())\nprint(\"Mean :\", image.mean())\nprint(\"Std :\", image.std())\n\n\n# ------------------------------------------------------------\n# 10. NORMALIZE IMAGE FOR DISPLAY\n# ------------------------------------------------------------\n\ndef normalize_image(img):\n \"\"\"\n Robust min-max normalization.\n \"\"\"\n \n img = img.astype(np.float32)\n\n low = np.percentile(img, 1)\n high = np.percentile(img, 99)\n\n img = np.clip(img, low, high)\n\n if high > low:\n img = (img - low) / (high - low)\n else:\n img = np.zeros_like(img)\n\n return img\n\n\ndisplay_image = normalize_image(image)\n\n\n# ------------------------------------------------------------\n# 11. DISPLAY SAMPLE MRI SLICE\n# ------------------------------------------------------------\n\nplt.figure(figsize=(7, 7))\n\nplt.imshow(\n display_image,\n cmap=\"gray\"\n)\n\nplt.title(\n f\"RSNA Knee MRI\\n\"\n f\"Study: {sample_study_uid[:25]}...\\n\"\n f\"Series: {selected_series_uid[:25]}...\"\n)\n\nplt.axis(\"off\")\n\nplt.show()\n\n\n# ------------------------------------------------------------\n# 12. DISPLAY MULTIPLE SLICES\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DISPLAYING MULTIPLE SLICES\")\nprint(\"=\" * 70)\n\n\nnum_slices = len(dcm_files)\n\n# Select 5 approximately equally spaced slices\nindices = np.linspace(\n 0,\n num_slices - 1,\n min(5, num_slices)\n).astype(int)\n\n\nplt.figure(figsize=(18, 4))\n\n\nfor plot_idx, slice_idx in enumerate(indices):\n\n try:\n\n slice_ds = pydicom.dcmread(\n dcm_files[slice_idx]\n )\n\n slice_img = slice_ds.pixel_array.astype(\n np.float32\n )\n\n slice_img = normalize_image(slice_img)\n\n ax = plt.subplot(\n 1,\n len(indices),\n plot_idx + 1\n )\n\n ax.imshow(\n slice_img,\n cmap=\"gray\"\n )\n\n ax.set_title(\n f\"Slice {slice_idx + 1}\\n\"\n f\"Instance {getattr(slice_ds, 'InstanceNumber', 'N/A')}\"\n )\n\n ax.axis(\"off\")\n\n except Exception as e:\n\n print(\n f\"Could not read slice {slice_idx}: {e}\"\n )\n\n\nplt.tight_layout()\nplt.show()\n\n\n# ------------------------------------------------------------\n# 13. SHOW SERIES TYPE FROM train_series.csv\n# ------------------------------------------------------------\n\nseries_metadata = train_series_df[\n train_series_df[\"SeriesInstanceUID\"] == selected_series_uid\n]\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SELECTED SERIES METADATA\")\nprint(\"=\" * 70)\n\nif len(series_metadata) > 0:\n\n print(\n series_metadata[\n [\n \"StudyInstanceUID\",\n \"SeriesInstanceUID\",\n \"Fluid_Sensitive\",\n \"Fat_Suppression\",\n \"Anatomical_Plane\"\n ]\n ].to_string(index=False)\n )\n\nelse:\n\n print(\"Series metadata not found.\")\n\n\n# ------------------------------------------------------------\n# 14. FINAL STEP 4 SUMMARY\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 4 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Training study found\")\nprint(\"āœ… MRI series found\")\nprint(\"āœ… DICOM files found\")\nprint(\"āœ… DICOM metadata read\")\nprint(\"āœ… Pixel array extracted\")\nprint(\"āœ… MRI image displayed\")\nprint(\"āœ… Multiple slices displayed\")\nprint(\"āœ… Series metadata checked\")\n\nprint(\"\\nšŸš€ Ready for STEP 5\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 5 ā€” LABELLED TRAINING DATA PREPARATION\n# RSNA Knee Abnormality Detection\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nprint(\"=\" * 70)\nprint(\"STEP 5 ā€” LABELLED TRAINING DATA PREPARATION\")\nprint(\"=\" * 70)\n\n\n# ------------------------------------------------------------\n# 1. LOAD TRAIN.CSV\n# ------------------------------------------------------------\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\ntrain_df = pd.read_csv(TRAIN_CSV)\n\nprint(\"\\nTrain shape:\")\nprint(train_df.shape)\n\n\n# ------------------------------------------------------------\n# 2. TARGET COLUMNS\n# ------------------------------------------------------------\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\"\\nTarget columns:\")\nfor i, target in enumerate(TARGETS, 1):\n print(f\"{i:2d}. {target}\")\n\n\n# ------------------------------------------------------------\n# 3. CHECK LABEL AVAILABILITY\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABEL AVAILABILITY\")\nprint(\"=\" * 70)\n\nlabel_counts = train_df[TARGETS].notna().sum()\n\nfor target in TARGETS:\n print(\n f\"{target:20s} : \"\n f\"{label_counts[target]} labelled\"\n )\n\n\n# ------------------------------------------------------------\n# 4. IDENTIFY STUDIES WITH LABELS\n# ------------------------------------------------------------\n\nlabelled_mask = train_df[TARGETS].notna().any(axis=1)\n\nlabelled_df = train_df[\n labelled_mask\n].copy()\n\nunlabelled_df = train_df[\n ~labelled_mask\n].copy()\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABELLED VS UNLABELLED STUDIES\")\nprint(\"=\" * 70)\n\nprint(\n \"Total studies :\",\n len(train_df)\n)\n\nprint(\n \"Labelled studies :\",\n len(labelled_df)\n)\n\nprint(\n \"Unlabelled studies :\",\n len(unlabelled_df)\n)\n\n\n# ------------------------------------------------------------\n# 5. CHECK COMPLETE LABEL ROWS\n# ------------------------------------------------------------\n\ncomplete_label_mask = train_df[\n TARGETS\n].notna().all(axis=1)\n\ncomplete_labelled_df = train_df[\n complete_label_mask\n].copy()\n\nprint(\n \"\\nStudies with ALL 12 labels:\",\n len(complete_labelled_df)\n)\n\n\n# ------------------------------------------------------------\n# 6. DISPLAY LABELLED DATA\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABELLED STUDY SAMPLE\")\nprint(\"=\" * 70)\n\ndisplay(\n labelled_df[\n [\"StudyInstanceUID\"] + TARGETS\n ].head(10)\n)\n\n\n# ------------------------------------------------------------\n# 7. CONVERT LABELS TO NUMERIC\n# ------------------------------------------------------------\n\nfor target in TARGETS:\n\n labelled_df[target] = pd.to_numeric(\n labelled_df[target],\n errors=\"coerce\"\n )\n\n\n# ------------------------------------------------------------\n# 8. LABEL DISTRIBUTION\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABEL DISTRIBUTION\")\nprint(\"=\" * 70)\n\ndistribution_rows = []\n\nfor target in TARGETS:\n\n positive = (\n labelled_df[target] == 1\n ).sum()\n\n negative = (\n labelled_df[target] == 0\n ).sum()\n\n missing = (\n labelled_df[target].isna()\n ).sum()\n\n total = positive + negative\n\n positive_pct = (\n positive / total * 100\n if total > 0 else 0\n )\n\n distribution_rows.append({\n \"Target\": target,\n \"Positive\": positive,\n \"Negative\": negative,\n \"Missing\": missing,\n \"Positive %\": round(\n positive_pct,\n 2\n )\n })\n\n\nlabel_distribution_df = pd.DataFrame(\n distribution_rows\n)\n\ndisplay(label_distribution_df)\n\n\n# ------------------------------------------------------------\n# 9. KEEP STUDIES HAVING AT LEAST ONE LABEL\n# ------------------------------------------------------------\n\nmodel_df = labelled_df[\n [\"StudyInstanceUID\", \"Report\"] + TARGETS\n].copy()\n\nprint(\"\\nModel dataframe shape:\")\nprint(model_df.shape)\n\n\n# ------------------------------------------------------------\n# 10. CHECK DUPLICATE STUDIES\n# ------------------------------------------------------------\n\nduplicate_count = (\n model_df[\"StudyInstanceUID\"]\n .duplicated()\n .sum()\n)\n\nprint(\n \"\\nDuplicate StudyInstanceUID:\",\n duplicate_count\n)\n\n\n# ------------------------------------------------------------\n# 11. CONNECT STUDIES WITH MRI SERIES\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"CONNECTING LABELS WITH MRI SERIES\")\nprint(\"=\" * 70)\n\nseries_counts = (\n train_series_df\n .groupby(\"StudyInstanceUID\")\n .size()\n .reset_index(\n name=\"Number_of_Series\"\n )\n)\n\nmodel_df = model_df.merge(\n series_counts,\n on=\"StudyInstanceUID\",\n how=\"left\"\n)\n\n\n# ------------------------------------------------------------\n# 12. CHECK SERIES AVAILABILITY\n# ------------------------------------------------------------\n\nprint(\n \"\\nStudies with MRI series:\",\n model_df[\"Number_of_Series\"].notna().sum()\n)\n\nprint(\n \"Studies without MRI series:\",\n model_df[\"Number_of_Series\"].isna().sum()\n)\n\n\n# ------------------------------------------------------------\n# 13. DISPLAY FINAL PREPARED DATA\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FINAL PREPARED TRAINING DATA\")\nprint(\"=\" * 70)\n\ndisplay(\n model_df.head(10)\n)\n\n\n# ------------------------------------------------------------\n# 14. SAVE PREPARED METADATA\n# ------------------------------------------------------------\n\nPREPARED_PATH = \"/kaggle/working/labelled_training_metadata.csv\"\n\nmodel_df.to_csv(\n PREPARED_PATH,\n index=False\n)\n\nprint(\n \"\\nSaved prepared metadata to:\"\n)\n\nprint(PREPARED_PATH)\n\n\n# ------------------------------------------------------------\n# 15. FINAL SUMMARY\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 5 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… Labels identified\"\n)\n\nprint(\n \"āœ… Labelled studies separated\"\n)\n\nprint(\n \"āœ… Target columns verified\"\n)\n\nprint(\n \"āœ… Label distribution calculated\"\n)\n\nprint(\n \"āœ… Duplicate studies checked\"\n)\n\nprint(\n \"āœ… MRI series connected\"\n)\n\nprint(\n \"āœ… Prepared metadata saved\"\n)\n\nprint(\"\\nšŸš€ Ready for STEP 6\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 6 ā€” CORRECTED RADIOLOGY REPORT ANALYSIS\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import roc_auc_score\n\nprint(\"=\" * 70)\nprint(\"STEP 6 ā€” CORRECTED RADIOLOGY REPORT ANALYSIS\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. FIND DATASET\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nTEST_CSV = os.path.join(\n DATA_PATH,\n \"test.csv\"\n)\n\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df = pd.read_csv(TEST_CSV)\n\nprint(\"\\nTrain shape:\", train_df.shape)\nprint(\"Test shape :\", test_df.shape)\n\nprint(\"\\nTrain columns:\")\nprint(train_df.columns.tolist())\n\nprint(\"\\nTest columns:\")\nprint(test_df.columns.tolist())\n\n\n# ============================================================\n# 2. TARGETS\n# ============================================================\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n\n# ============================================================\n# 3. REPORT CLEANING\n# ============================================================\n\ndef clean_report(text):\n\n if pd.isna(text):\n return \"\"\n\n text = str(text)\n\n text = text.lower()\n\n text = text.replace(\"\\n\", \" \")\n text = text.replace(\"\\r\", \" \")\n\n text = re.sub(\n r\"\\s+\",\n \" \",\n text\n )\n\n return text.strip()\n\n\ntrain_df[\"Report_Clean\"] = (\n train_df[\"Report\"]\n .apply(clean_report)\n)\n\n\nprint(\"\\nāœ… Training reports cleaned\")\n\nprint(\n \"Reports available:\",\n train_df[\"Report_Clean\"].ne(\"\").sum()\n)\n\n\n# ============================================================\n# 4. REPORT STATISTICS\n# ============================================================\n\ntrain_df[\"Report_Length\"] = (\n train_df[\"Report_Clean\"].str.len()\n)\n\ntrain_df[\"Report_Word_Count\"] = (\n train_df[\"Report_Clean\"]\n .str.split()\n .str.len()\n)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"REPORT STATISTICS\")\nprint(\"=\" * 70)\n\nprint(\n train_df[\n [\n \"Report_Length\",\n \"Report_Word_Count\"\n ]\n ].describe()\n)\n\n\n# ============================================================\n# 5. SAMPLE REPORT\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SAMPLE RADIOLOGY REPORT\")\nprint(\"=\" * 70)\n\nprint(\n train_df.iloc[0][\"Report\"]\n)\n\n\n# ============================================================\n# 6. CHECK LABELLED STUDIES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABEL AVAILABILITY\")\nprint(\"=\" * 70)\n\nlabel_info = []\n\nfor target in TARGETS:\n\n mask = train_df[target].notna()\n\n positive = (\n train_df.loc[mask, target] == 1\n ).sum()\n\n negative = (\n train_df.loc[mask, target] == 0\n ).sum()\n\n label_info.append({\n \"Target\": target,\n \"Available\": int(mask.sum()),\n \"Positive\": int(positive),\n \"Negative\": int(negative)\n })\n\n\nlabel_info_df = pd.DataFrame(\n label_info\n)\n\ndisplay(label_info_df)\n\n\n# ============================================================\n# 7. TEXT MODEL EXPERIMENT\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TF-IDF TEXT MODEL VALIDATION\")\nprint(\"=\" * 70)\n\ntext_auc_results = []\n\n\nfor target in TARGETS:\n\n print(\"\\n----------------------------------------\")\n print(\"Target:\", target)\n print(\"----------------------------------------\")\n\n\n # --------------------------------------------------------\n # Get available labels\n # --------------------------------------------------------\n\n mask = train_df[target].notna()\n\n data = train_df.loc[\n mask,\n [\n \"StudyInstanceUID\",\n \"Report_Clean\",\n target\n ]\n ].copy()\n\n\n # Convert labels\n data[target] = pd.to_numeric(\n data[target],\n errors=\"coerce\"\n )\n\n\n # Remove invalid labels\n data = data.dropna(\n subset=[target]\n )\n\n\n # Need both classes\n if data[target].nunique() < 2:\n\n print(\n \"āš ļø Skipped ā€” only one class available.\"\n )\n\n continue\n\n\n X = data[\"Report_Clean\"]\n\n y = data[target].astype(int)\n\n\n print(\n \"Samples:\",\n len(data)\n )\n\n print(\n \"Positive:\",\n int(y.sum())\n )\n\n print(\n \"Negative:\",\n int((y == 0).sum())\n )\n\n\n # --------------------------------------------------------\n # Train/validation split\n # --------------------------------------------------------\n\n X_train, X_valid, y_train, y_valid = train_test_split(\n\n X,\n y,\n\n test_size=0.25,\n\n random_state=42,\n\n stratify=y\n )\n\n\n # --------------------------------------------------------\n # TF-IDF + Logistic Regression\n # --------------------------------------------------------\n\n model = Pipeline([\n\n (\n \"tfidf\",\n\n TfidfVectorizer(\n\n lowercase=True,\n\n strip_accents=\"unicode\",\n\n ngram_range=(1, 2),\n\n min_df=1,\n\n max_features=30000,\n\n sublinear_tf=True\n )\n ),\n\n (\n \"classifier\",\n\n LogisticRegression(\n\n max_iter=2000,\n\n class_weight=\"balanced\",\n\n C=1.0,\n\n solver=\"liblinear\"\n )\n )\n ])\n\n\n # --------------------------------------------------------\n # Train\n # --------------------------------------------------------\n\n model.fit(\n X_train,\n y_train\n )\n\n\n # --------------------------------------------------------\n # Validation prediction\n # --------------------------------------------------------\n\n valid_probability = model.predict_proba(\n X_valid\n )[:, 1]\n\n\n # --------------------------------------------------------\n # ROC-AUC\n # --------------------------------------------------------\n\n auc = roc_auc_score(\n y_valid,\n valid_probability\n )\n\n\n print(\n \"Validation ROC-AUC:\",\n round(auc, 4)\n )\n\n\n text_auc_results.append({\n\n \"Target\": target,\n\n \"Samples\": len(data),\n\n \"Positive\": int(y.sum()),\n\n \"Negative\": int((y == 0).sum()),\n\n \"Validation_AUC\": round(\n auc,\n 4\n )\n })\n\n\n# ============================================================\n# 8. TEXT MODEL RESULTS\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TEXT MODEL RESULTS\")\nprint(\"=\" * 70)\n\ntext_results_df = pd.DataFrame(\n text_auc_results\n)\n\ndisplay(\n text_results_df\n)\n\n\n# ============================================================\n# 9. MACRO AUC\n# ============================================================\n\nif len(text_results_df) > 0:\n\n macro_auc = (\n text_results_df[\n \"Validation_AUC\"\n ].mean()\n )\n\n print(\n \"\\nMacro-average validation AUC:\",\n round(\n macro_auc,\n 4\n )\n )\n\nelse:\n\n print(\n \"\\nNo valid target models were trained.\"\n )\n\n\n# ============================================================\n# 10. SAVE RESULTS\n# ============================================================\n\nRESULT_PATH = (\n \"/kaggle/working/\"\n \"text_model_validation_results.csv\"\n)\n\ntext_results_df.to_csv(\n RESULT_PATH,\n index=False\n)\n\nprint(\n \"\\nāœ… Results saved:\"\n)\n\nprint(RESULT_PATH)\n\n\n# ============================================================\n# 11. IMPORTANT TEST DATA CHECK\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TEST DATA CHECK\")\nprint(\"=\" * 70)\n\nprint(\n \"Test columns:\",\n test_df.columns.tolist()\n)\n\nif \"Report\" not in test_df.columns:\n\n print(\n \"\\nāš ļø IMPORTANT:\"\n )\n\n print(\n \"Test.csv does NOT contain Report.\"\n )\n\n print(\n \"Therefore, the final test prediction\"\n )\n\n print(\n \"cannot use a text-only model.\"\n )\n\n print(\n \"\\nWe will use MRI/DICOM information\"\n )\n\n print(\n \"for the actual competition submission.\"\n )\n\n\n# ============================================================\n# 12. FINAL\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 6 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Training reports cleaned\")\nprint(\"āœ… Label availability checked\")\nprint(\"āœ… TF-IDF + Logistic Regression tested\")\nprint(\"āœ… Validation ROC-AUC calculated\")\nprint(\"āœ… Macro AUC calculated\")\nprint(\"āœ… Test data limitation identified\")\n\nprint(\"\\nšŸš€ READY FOR STEP 7\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 7 ā€” MRI / DICOM DATA PREPARATION\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport pydicom\n\nfrom PIL import Image\n\nprint(\"=\" * 70)\nprint(\"STEP 7 ā€” MRI / DICOM DATA PREPARATION\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. DATA PATH\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nTRAIN_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"train_series.csv\"\n)\n\nTRAIN_SERIES_DIR = os.path.join(\n DATA_PATH,\n \"train_series\"\n)\n\n\n# ============================================================\n# 2. LOAD METADATA\n# ============================================================\n\ntrain_df = pd.read_csv(TRAIN_CSV)\n\ntrain_series_df = pd.read_csv(\n TRAIN_SERIES_CSV\n)\n\nprint(\"\\nTrain studies:\", len(train_df))\nprint(\"Train series :\", len(train_series_df))\n\n\n# ============================================================\n# 3. TARGETS\n# ============================================================\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n\n# ============================================================\n# 4. FIND LABELLED STUDIES\n# ============================================================\n\nlabelled_mask = train_df[TARGETS].notna().any(axis=1)\n\nlabelled_df = train_df[\n labelled_mask\n].copy()\n\nprint(\n \"\\nLabelled studies:\",\n len(labelled_df)\n)\n\n\n# ============================================================\n# 5. CONNECT SERIES METADATA\n# ============================================================\n\nlabelled_uids = set(\n labelled_df[\"StudyInstanceUID\"]\n)\n\nlabelled_series_df = train_series_df[\n train_series_df[\"StudyInstanceUID\"].isin(\n labelled_uids\n )\n].copy()\n\nprint(\n \"Series belonging to labelled studies:\",\n len(labelled_series_df)\n)\n\n\n# ============================================================\n# 6. SERIES DISTRIBUTION\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SERIES DISTRIBUTION\")\nprint(\"=\" * 70)\n\nprint(\n labelled_series_df[\n \"Anatomical_Plane\"\n ].value_counts()\n)\n\nprint(\"\\nFluid-sensitive:\")\nprint(\n labelled_series_df[\n \"Fluid_Sensitive\"\n ].value_counts()\n)\n\nprint(\"\\nFat-suppression:\")\nprint(\n labelled_series_df[\n \"Fat_Suppression\"\n ].value_counts()\n)\n\n\n# ============================================================\n# 7. FIND DICOM FILES\n# ============================================================\n\ndef find_dicom_files(series_path):\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"*.dcm\"\n )\n )\n\n if len(files) == 0:\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"**\",\n \"*.dcm\"\n ),\n recursive=True\n )\n\n return sorted(files)\n\n\n# ============================================================\n# 8. SELECT ONE LABELLED STUDY\n# ============================================================\n\nsample_uid = labelled_df.iloc[0][\n \"StudyInstanceUID\"\n]\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SAMPLE LABELLED STUDY\")\nprint(\"=\" * 70)\n\nprint(\"StudyInstanceUID:\")\nprint(sample_uid)\n\n\n# ============================================================\n# 9. GET LABELS\n# ============================================================\n\nsample_labels = labelled_df[\n labelled_df[\"StudyInstanceUID\"] == sample_uid\n][TARGETS].iloc[0]\n\nprint(\"\\nLabels:\")\n\nfor target in TARGETS:\n\n print(\n f\"{target:20s}: {sample_labels[target]}\"\n )\n\n\n# ============================================================\n# 10. GET SERIES FOR STUDY\n# ============================================================\n\nstudy_series = labelled_series_df[\n labelled_series_df[\"StudyInstanceUID\"] == sample_uid\n].copy()\n\nprint(\"\\nNumber of series:\", len(study_series))\n\ndisplay(study_series)\n\n\n# ============================================================\n# 11. BUILD DICOM INDEX FOR SAMPLE STUDY\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"BUILDING DICOM INDEX\")\nprint(\"=\" * 70)\n\ndicom_records = []\n\nfor _, row in study_series.iterrows():\n\n series_uid = row[\"SeriesInstanceUID\"]\n\n series_path = os.path.join(\n TRAIN_SERIES_DIR,\n sample_uid,\n series_uid\n )\n\n files = find_dicom_files(\n series_path\n )\n\n for file_path in files:\n\n dicom_records.append({\n\n \"StudyInstanceUID\":\n sample_uid,\n\n \"SeriesInstanceUID\":\n series_uid,\n\n \"DICOM_Path\":\n file_path,\n\n \"Fluid_Sensitive\":\n row[\"Fluid_Sensitive\"],\n\n \"Fat_Suppression\":\n row[\"Fat_Suppression\"],\n\n \"Anatomical_Plane\":\n row[\"Anatomical_Plane\"]\n\n })\n\n\ndicom_index_df = pd.DataFrame(\n dicom_records\n)\n\nprint(\n \"Total DICOM slices found:\",\n len(dicom_index_df)\n)\n\ndisplay(\n dicom_index_df.head()\n)\n\n\n# ============================================================\n# 12. READ DICOM SAFELY\n# ============================================================\n\ndef read_dicom_image(path):\n\n ds = pydicom.dcmread(\n path\n )\n\n image = ds.pixel_array.astype(\n np.float32\n )\n\n # Handle MONOCHROME1\n if getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n ) == \"MONOCHROME1\":\n\n image = image.max() - image\n\n\n # Apply rescale slope/intercept\n slope = float(\n getattr(\n ds,\n \"RescaleSlope\",\n 1\n )\n )\n\n intercept = float(\n getattr(\n ds,\n \"RescaleIntercept\",\n 0\n )\n )\n\n image = (\n image * slope\n + intercept\n )\n\n return image\n\n\n# ============================================================\n# 13. NORMALIZATION\n# ============================================================\n\ndef normalize_mri(image):\n\n image = image.astype(\n np.float32\n )\n\n low = np.percentile(\n image,\n 1\n )\n\n high = np.percentile(\n image,\n 99\n )\n\n image = np.clip(\n image,\n low,\n high\n )\n\n if high > low:\n\n image = (\n image - low\n ) / (\n high - low\n )\n\n else:\n\n image = np.zeros_like(\n image\n )\n\n return image\n\n\n# ============================================================\n# 14. RESIZE\n# ============================================================\n\ndef resize_image(\n image,\n size=(224, 224)\n):\n\n image_uint8 = (\n image * 255\n ).clip(\n 0,\n 255\n ).astype(\n np.uint8\n )\n\n pil_image = Image.fromarray(\n image_uint8\n )\n\n pil_image = pil_image.resize(\n size,\n Image.Resampling.BILINEAR\n )\n\n return np.asarray(\n pil_image,\n dtype=np.float32\n ) / 255.0\n\n\n# ============================================================\n# 15. SELECT REPRESENTATIVE SLICES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SELECTING REPRESENTATIVE MRI SLICES\")\nprint(\"=\" * 70)\n\n\ndef select_representative_slices(\n file_list,\n num_slices=5\n):\n\n if len(file_list) == 0:\n return []\n\n if len(file_list) <= num_slices:\n return file_list\n\n indices = np.linspace(\n 0,\n len(file_list) - 1,\n num_slices\n ).astype(int)\n\n return [\n file_list[i]\n for i in indices\n ]\n\n\n# Pick first available series\nfirst_series = study_series.iloc[0]\n\nfirst_series_uid = first_series[\n \"SeriesInstanceUID\"\n]\n\nfirst_series_path = os.path.join(\n TRAIN_SERIES_DIR,\n sample_uid,\n first_series_uid\n)\n\nseries_files = find_dicom_files(\n first_series_path\n)\n\nselected_files = select_representative_slices(\n series_files,\n num_slices=5\n)\n\nprint(\n \"Original slices:\",\n len(series_files)\n)\n\nprint(\n \"Selected slices:\",\n len(selected_files)\n)\n\n\n# ============================================================\n# 16. READ + RESIZE SELECTED SLICES\n# ============================================================\n\nprocessed_images = []\n\nfor path in selected_files:\n\n try:\n\n image = read_dicom_image(\n path\n )\n\n image = normalize_mri(\n image\n )\n\n image = resize_image(\n image,\n size=(224, 224)\n )\n\n processed_images.append(\n image\n )\n\n except Exception as e:\n\n print(\n \"Error:\",\n path,\n e\n )\n\n\nprocessed_images = np.array(\n processed_images,\n dtype=np.float32\n)\n\nprint(\n \"\\nProcessed image array shape:\",\n processed_images.shape\n)\n\n\n# ============================================================\n# 17. DISPLAY PROCESSED SLICES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"PROCESSED MRI SLICES\")\nprint(\"=\" * 70)\n\nif len(processed_images) > 0:\n\n plt.figure(\n figsize=(18, 4)\n )\n\n for i in range(\n len(processed_images)\n ):\n\n ax = plt.subplot(\n 1,\n len(processed_images),\n i + 1\n )\n\n ax.imshow(\n processed_images[i],\n cmap=\"gray\"\n )\n\n ax.set_title(\n f\"Slice {i + 1}\"\n )\n\n ax.axis(\"off\")\n\n plt.tight_layout()\n plt.show()\n\n\n# ============================================================\n# 18. CONVERT GRAYSCALE ā†’ 3 CHANNEL\n# ============================================================\n\nif len(processed_images) > 0:\n\n images_rgb = np.repeat(\n processed_images[\n ..., np.newaxis\n ],\n 3,\n axis=-1\n )\n\nelse:\n\n images_rgb = np.empty(\n (0, 224, 224, 3),\n dtype=np.float32\n )\n\n\nprint(\n \"\\n3-channel image shape:\",\n images_rgb.shape\n)\n\n\n# ============================================================\n# 19. CHECK VALUES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"IMAGE VALUE CHECK\")\nprint(\"=\" * 70)\n\nif images_rgb.size > 0:\n\n print(\n \"Min:\",\n images_rgb.min()\n )\n\n print(\n \"Max:\",\n images_rgb.max()\n )\n\n print(\n \"Mean:\",\n images_rgb.mean()\n )\n\n print(\n \"Std:\",\n images_rgb.std()\n )\n\n\n# ============================================================\n# 20. FINAL SUMMARY\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 7 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Labelled MRI study selected\")\nprint(\"āœ… Series metadata connected\")\nprint(\"āœ… DICOM slices indexed\")\nprint(\"āœ… DICOM pixels read\")\nprint(\"āœ… MONOCHROME handling included\")\nprint(\"āœ… Rescale slope/intercept handled\")\nprint(\"āœ… MRI normalization completed\")\nprint(\"āœ… Images resized to 224 Ɨ 224\")\nprint(\"āœ… Representative slices selected\")\nprint(\"āœ… 3-channel tensor prepared\")\n\nprint(\"\\nšŸš€ READY FOR STEP 8\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 8 ā€” MRI DEEP LEARNING BASELINE\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport glob\nimport random\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\n\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nimport torchvision\nfrom torchvision import transforms, models\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n\nprint(\"=\" * 70)\nprint(\"STEP 8 ā€” MRI DEEP LEARNING BASELINE\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. CONFIGURATION\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nTRAIN_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"train_series.csv\"\n)\n\nTRAIN_SERIES_DIR = os.path.join(\n DATA_PATH,\n \"train_series\"\n)\n\nIMAGE_SIZE = 224\n\n# Keep this small initially so Kaggle runtime stays manageable\nSLICES_PER_STUDY = 3\n\nBATCH_SIZE = 8\n\nEPOCHS = 3\n\nLEARNING_RATE = 1e-4\n\nSEED = 42\n\n\n# ============================================================\n# 2. REPRODUCIBILITY\n# ============================================================\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\n\nif torch.cuda.is_available():\n torch.cuda.manual_seed_all(SEED)\n\n\n# ============================================================\n# 3. DEVICE\n# ============================================================\n\nDEVICE = torch.device(\n \"cuda\" if torch.cuda.is_available() else \"cpu\"\n)\n\nprint(\"\\nDevice:\", DEVICE)\n\nif torch.cuda.is_available():\n print(\n \"GPU:\",\n torch.cuda.get_device_name(0)\n )\n\n\n# ============================================================\n# 4. LOAD DATA\n# ============================================================\n\nprint(\"\\nLoading CSV files...\")\n\ntrain_df = pd.read_csv(\n TRAIN_CSV\n)\n\ntrain_series_df = pd.read_csv(\n TRAIN_SERIES_CSV\n)\n\nprint(\n \"Train studies:\",\n len(train_df)\n)\n\nprint(\n \"Train series:\",\n len(train_series_df)\n)\n\n\n# ============================================================\n# 5. TARGETS\n# ============================================================\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nNUM_TARGETS = len(TARGETS)\n\nprint(\n \"\\nNumber of targets:\",\n NUM_TARGETS\n)\n\n\n# ============================================================\n# 6. SELECT LABELLED STUDIES\n# ============================================================\n\nlabelled_mask = train_df[\n TARGETS\n].notna().any(axis=1)\n\nlabelled_df = train_df[\n labelled_mask\n].copy()\n\nprint(\n \"\\nLabelled studies:\",\n len(labelled_df)\n)\n\n\n# ============================================================\n# 7. IMPORTANT:\n# ONLY USE STUDIES WITH ALL 12 LABELS\n# FOR THIS FIRST BASELINE\n# ============================================================\n\ncomplete_mask = train_df[\n TARGETS\n].notna().all(axis=1)\n\ncomplete_df = train_df[\n complete_mask\n].copy()\n\nprint(\n \"Studies with all 12 labels:\",\n len(complete_df)\n)\n\n\nif len(complete_df) < 10:\n\n print(\n \"\\nāš ļø Very few complete labelled studies.\"\n )\n\n print(\n \"Using all studies with at least one label\"\n )\n\n # We cannot train ordinary multi-label BCE safely\n # if many target values are NaN.\n #\n # Therefore fill missing target values with -1.\n # -1 will be ignored during loss calculation.\n\n model_df = labelled_df.copy()\n\nelse:\n\n model_df = complete_df.copy()\n\n\n# ============================================================\n# 8. PREPARE TARGET MATRIX\n# ============================================================\n\nfor target in TARGETS:\n\n model_df[target] = pd.to_numeric(\n model_df[target],\n errors=\"coerce\"\n )\n\n\nY = model_df[\n TARGETS\n].values.astype(\n np.float32\n)\n\n# Missing labels become -1\nY = np.nan_to_num(\n Y,\n nan=-1.0\n)\n\nprint(\n \"\\nTarget matrix shape:\",\n Y.shape\n)\n\n\n# ============================================================\n# 9. FIND DICOM FILES\n# ============================================================\n\ndef find_dicom_files(series_path):\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"*.dcm\"\n )\n )\n\n if len(files) == 0:\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"**\",\n \"*.dcm\"\n ),\n recursive=True\n )\n\n return sorted(files)\n\n\n# ============================================================\n# 10. BUILD STUDY ā†’ DICOM INDEX\n# ============================================================\n\nprint(\"\\nBuilding study-level DICOM index...\")\n\nstudy_to_files = {}\n\nfor study_uid in model_df[\n \"StudyInstanceUID\"\n]:\n\n study_path = os.path.join(\n TRAIN_SERIES_DIR,\n study_uid\n )\n\n if not os.path.exists(\n study_path\n ):\n continue\n\n all_files = []\n\n # Get series belonging to this study\n study_series = train_series_df[\n train_series_df[\n \"StudyInstanceUID\"\n ] == study_uid\n ]\n\n # Prefer fluid-sensitive sequences\n study_series = study_series.sort_values(\n [\n \"Fluid_Sensitive\",\n \"Fat_Suppression\"\n ],\n ascending=False\n )\n\n for _, row in study_series.iterrows():\n\n series_uid = row[\n \"SeriesInstanceUID\"\n ]\n\n series_path = os.path.join(\n TRAIN_SERIES_DIR,\n study_uid,\n series_uid\n )\n\n files = find_dicom_files(\n series_path\n )\n\n if len(files) > 0:\n\n # Keep a limited number of files\n all_files.extend(files)\n\n if len(all_files) >= 60:\n break\n\n if len(all_files) > 0:\n\n study_to_files[\n study_uid\n ] = all_files\n\n\nprint(\n \"Studies with DICOM data:\",\n len(study_to_files)\n)\n\n\n# ============================================================\n# 11. KEEP ONLY STUDIES WITH IMAGES\n# ============================================================\n\nmodel_df = model_df[\n model_df[\n \"StudyInstanceUID\"\n ].isin(\n study_to_files.keys()\n )\n].reset_index(\n drop=True\n)\n\nY = model_df[\n TARGETS\n].values.astype(\n np.float32\n)\n\nY = np.nan_to_num(\n Y,\n nan=-1.0\n)\n\nprint(\n \"Final studies available:\",\n len(model_df)\n)\n\n\n# ============================================================\n# 12. TRAIN / VALIDATION SPLIT\n# ============================================================\n\nindices = np.arange(\n len(model_df)\n)\n\ntrain_indices, valid_indices = train_test_split(\n indices,\n test_size=0.20,\n random_state=SEED\n)\n\nprint(\n \"\\nTraining studies:\",\n len(train_indices)\n)\n\nprint(\n \"Validation studies:\",\n len(valid_indices)\n)\n\n\n# ============================================================\n# 13. DICOM IMAGE READER\n# ============================================================\n\ndef read_dicom_image(path):\n\n ds = pydicom.dcmread(\n path\n )\n\n image = ds.pixel_array.astype(\n np.float32\n )\n\n # MONOCHROME1\n if getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n ) == \"MONOCHROME1\":\n\n image = (\n image.max()\n - image\n )\n\n # Rescale\n slope = float(\n getattr(\n ds,\n \"RescaleSlope\",\n 1\n )\n )\n\n intercept = float(\n getattr(\n ds,\n \"RescaleIntercept\",\n 0\n )\n )\n\n image = (\n image * slope\n + intercept\n )\n\n # Robust normalization\n low = np.percentile(\n image,\n 1\n )\n\n high = np.percentile(\n image,\n 99\n )\n\n image = np.clip(\n image,\n low,\n high\n )\n\n if high > low:\n\n image = (\n image - low\n ) / (\n high - low\n )\n\n else:\n\n image = np.zeros_like(\n image\n )\n\n return image\n\n\n# ============================================================\n# 14. IMAGE TRANSFORMS\n# ============================================================\n\ntrain_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (IMAGE_SIZE, IMAGE_SIZE)\n ),\n\n transforms.RandomHorizontalFlip(\n p=0.5\n ),\n\n transforms.RandomRotation(\n degrees=5\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n )\n])\n\n\nvalid_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (IMAGE_SIZE, IMAGE_SIZE)\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n )\n])\n\n\n# ============================================================\n# 15. DATASET\n# ============================================================\n\nclass KneeMRIDataset(\n Dataset\n):\n\n def __init__(\n self,\n dataframe,\n targets,\n study_to_files,\n indices,\n transform=None,\n slices_per_study=3\n ):\n\n self.df = dataframe\n\n self.targets = targets\n\n self.study_to_files = (\n study_to_files\n )\n\n self.indices = indices\n\n self.transform = transform\n\n self.slices_per_study = (\n slices_per_study\n )\n\n\n def __len__(self):\n\n return len(\n self.indices\n )\n\n\n def __getitem__(\n self,\n idx\n ):\n\n real_idx = self.indices[\n idx\n ]\n\n row = self.df.iloc[\n real_idx\n ]\n\n study_uid = row[\n \"StudyInstanceUID\"\n ]\n\n files = self.study_to_files[\n study_uid\n ]\n\n\n # ----------------------------------------------------\n # Select evenly spaced slices\n # ----------------------------------------------------\n\n if len(files) <= self.slices_per_study:\n\n selected = files\n\n else:\n\n positions = np.linspace(\n 0,\n len(files) - 1,\n self.slices_per_study\n ).astype(int)\n\n selected = [\n files[p]\n for p in positions\n ]\n\n\n images = []\n\n\n # ----------------------------------------------------\n # Read selected slices\n # ----------------------------------------------------\n\n for path in selected:\n\n try:\n\n image = read_dicom_image(\n path\n )\n\n if self.transform:\n\n image = self.transform(\n image\n )\n\n else:\n\n image = torch.tensor(\n image,\n dtype=torch.float32\n )\n\n # grayscale ā†’ 3 channel\n if image.shape[0] == 1:\n\n image = image.repeat(\n 3,\n 1,\n 1\n )\n\n images.append(\n image\n )\n\n except Exception:\n\n continue\n\n\n # ----------------------------------------------------\n # Make sure number of slices is fixed\n # ----------------------------------------------------\n\n if len(images) == 0:\n\n image = torch.zeros(\n 3,\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n\n images = [\n image\n for _ in range(\n self.slices_per_study\n )\n ]\n\n\n while len(images) < self.slices_per_study:\n\n images.append(\n images[-1].clone()\n )\n\n\n images = images[\n :self.slices_per_study\n ]\n\n\n # Stack:\n # [S, C, H, W]\n\n images = torch.stack(\n images\n )\n\n\n # Targets\n target = torch.tensor(\n self.targets[\n real_idx\n ],\n dtype=torch.float32\n )\n\n\n return images, target\n\n\n# ============================================================\n# 16. CREATE DATASETS\n# ============================================================\n\ntrain_dataset = KneeMRIDataset(\n\n model_df,\n\n Y,\n\n study_to_files,\n\n train_indices,\n\n transform=train_transform,\n\n slices_per_study=SLICES_PER_STUDY\n)\n\n\nvalid_dataset = KneeMRIDataset(\n\n model_df,\n\n Y,\n\n study_to_files,\n\n valid_indices,\n\n transform=valid_transform,\n\n slices_per_study=SLICES_PER_STUDY\n)\n\n\n# ============================================================\n# 17. DATA LOADERS\n# ============================================================\n\ntrain_loader = DataLoader(\n\n train_dataset,\n\n batch_size=BATCH_SIZE,\n\n shuffle=True,\n\n num_workers=2,\n\n pin_memory=True\n)\n\n\nvalid_loader = DataLoader(\n\n valid_dataset,\n\n batch_size=BATCH_SIZE,\n\n shuffle=False,\n\n num_workers=2,\n\n pin_memory=True\n)\n\n\n# ============================================================\n# 18. CHECK ONE BATCH\n# ============================================================\n\nprint(\"\\nChecking DataLoader...\")\n\nimages, targets = next(\n iter(train_loader)\n)\n\nprint(\n \"Images shape:\",\n images.shape\n)\n\nprint(\n \"Targets shape:\",\n targets.shape\n)\n\n# Expected:\n# Images = [batch, slices, 3, 224, 224]\n# Targets = [batch, 12]\n\n\n# ============================================================\n# 19. CREATE PRETRAINED RESNET18\n# ============================================================\n\nprint(\"\\nLoading pretrained ResNet18...\")\n\ntry:\n\n weights = (\n models.ResNet18_Weights.DEFAULT\n )\n\n backbone = models.resnet18(\n weights=weights\n )\n\nexcept Exception as e:\n\n print(\n \"Could not load pretrained weights:\"\n )\n\n print(e)\n\n print(\n \"Using randomly initialized model.\"\n )\n\n backbone = models.resnet18(\n weights=None\n )\n\n\n# Remove original classifier\n\nnum_features = (\n backbone.fc.in_features\n)\n\nbackbone.fc = nn.Identity()\n\n\n# ============================================================\n# 20. MULTI-SLICE STUDY MODEL\n# ============================================================\n\nclass KneeResNet18(\n nn.Module\n):\n\n def __init__(\n self,\n backbone,\n num_features,\n num_targets=12\n ):\n\n super().__init__()\n\n self.backbone = backbone\n\n self.classifier = nn.Sequential(\n\n nn.Dropout(\n 0.3\n ),\n\n nn.Linear(\n num_features,\n num_targets\n )\n )\n\n\n def forward(\n self,\n x\n ):\n\n # x:\n # [B, S, C, H, W]\n\n batch_size = x.shape[0]\n\n num_slices = x.shape[1]\n\n channels = x.shape[2]\n\n height = x.shape[3]\n\n width = x.shape[4]\n\n\n # Merge batch and slices\n\n x = x.view(\n batch_size * num_slices,\n channels,\n height,\n width\n )\n\n\n # CNN features\n\n features = self.backbone(\n x\n )\n\n\n # Restore slice dimension\n\n features = features.view(\n batch_size,\n num_slices,\n -1\n )\n\n\n # Average slice features\n\n features = features.mean(\n dim=1\n )\n\n\n # 12 target logits\n\n output = self.classifier(\n features\n )\n\n return output\n\n\nmodel = KneeResNet18(\n\n backbone,\n\n num_features,\n\n NUM_TARGETS\n\n)\n\nmodel = model.to(\n DEVICE\n)\n\n\nprint(\"\\nModel created successfully.\")\n\n\n# ============================================================\n# 21. LOSS FUNCTION\n# ============================================================\n\n# We have missing labels (-1).\n# BCEWithLogitsLoss does not directly ignore them.\n# Therefore use reduction='none' and manually mask.\n\ncriterion = nn.BCEWithLogitsLoss(\n reduction=\"none\"\n)\n\n\n# ============================================================\n# 22. OPTIMIZER\n# ============================================================\n\noptimizer = torch.optim.AdamW(\n\n model.parameters(),\n\n lr=LEARNING_RATE,\n\n weight_decay=1e-4\n)\n\n\n# ============================================================\n# 23. TRAINING FUNCTION\n# ============================================================\n\ndef train_one_epoch():\n\n model.train()\n\n total_loss = 0.0\n\n batches = 0\n\n\n for images, targets in train_loader:\n\n images = images.to(\n DEVICE,\n non_blocking=True\n )\n\n targets = targets.to(\n DEVICE,\n non_blocking=True\n )\n\n\n optimizer.zero_grad()\n\n\n logits = model(\n images\n )\n\n\n # Mask missing labels\n valid_mask = (\n targets >= 0\n )\n\n\n safe_targets = torch.clamp(\n targets,\n 0,\n 1\n )\n\n\n losses = criterion(\n logits,\n safe_targets\n )\n\n\n # Ignore missing targets\n masked_loss = losses[\n valid_mask\n ]\n\n\n if masked_loss.numel() == 0:\n\n continue\n\n\n loss = masked_loss.mean()\n\n\n loss.backward()\n\n\n optimizer.step()\n\n\n total_loss += (\n loss.item()\n )\n\n batches += 1\n\n\n if batches == 0:\n\n return 0.0\n\n\n return (\n total_loss /\n batches\n )\n\n\n# ============================================================\n# 24. VALIDATION FUNCTION\n# ============================================================\n\ndef validate_model():\n\n model.eval()\n\n all_predictions = []\n\n all_targets = []\n\n\n with torch.no_grad():\n\n for images, targets in valid_loader:\n\n images = images.to(\n DEVICE,\n non_blocking=True\n )\n\n\n logits = model(\n images\n )\n\n\n probabilities = torch.sigmoid(\n logits\n )\n\n\n all_predictions.append(\n probabilities.cpu().numpy()\n )\n\n all_targets.append(\n targets.numpy()\n )\n\n\n predictions = np.concatenate(\n all_predictions,\n axis=0\n )\n\n targets = np.concatenate(\n all_targets,\n axis=0\n )\n\n\n aucs = []\n\n\n for i, target_name in enumerate(\n TARGETS\n ):\n\n y_true = targets[:, i]\n\n y_pred = predictions[:, i]\n\n\n # Only valid labels\n valid = y_true >= 0\n\n\n if (\n valid.sum() > 0\n and len(\n np.unique(\n y_true[valid]\n )\n ) == 2\n ):\n\n auc = roc_auc_score(\n y_true[valid],\n y_pred[valid]\n )\n\n aucs.append(\n auc\n )\n\n else:\n\n aucs.append(\n np.nan\n )\n\n\n macro_auc = np.nanmean(\n aucs\n )\n\n\n result_df = pd.DataFrame({\n\n \"Target\": TARGETS,\n\n \"AUC\": aucs\n\n })\n\n\n return (\n macro_auc,\n result_df,\n predictions,\n targets\n )\n\n\n# ============================================================\n# 25. TRAIN MODEL\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STARTING MRI MODEL TRAINING\")\nprint(\"=\" * 70)\n\nbest_auc = -1\n\nbest_model_path = (\n \"/kaggle/working/\"\n \"rsna_resnet18_best.pth\"\n)\n\n\nfor epoch in range(\n EPOCHS\n):\n\n print(\n f\"\\nEpoch {epoch + 1}/{EPOCHS}\"\n )\n\n train_loss = train_one_epoch()\n\n\n print(\n \"Training loss:\",\n round(\n train_loss,\n 5\n )\n )\n\n\n val_auc, auc_df, _, _ = (\n validate_model()\n )\n\n\n print(\n \"Validation Macro AUC:\",\n round(\n val_auc,\n 5\n )\n )\n\n\n if val_auc > best_auc:\n\n best_auc = val_auc\n\n torch.save(\n model.state_dict(),\n best_model_path\n )\n\n print(\n \"āœ… Best model saved\"\n )\n\n\n# ============================================================\n# 26. LOAD BEST MODEL\n# ============================================================\n\nif os.path.exists(\n best_model_path\n):\n\n model.load_state_dict(\n torch.load(\n best_model_path,\n map_location=DEVICE\n )\n )\n\n print(\n \"\\nāœ… Best model restored\"\n )\n\n\n# ============================================================\n# 27. FINAL VALIDATION\n# ============================================================\n\nfinal_auc, final_auc_df, _, _ = (\n validate_model()\n)\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FINAL MRI BASELINE RESULTS\")\nprint(\"=\" * 70)\n\ndisplay(\n final_auc_df\n)\n\nprint(\n \"\\nFinal MRI Macro ROC-AUC:\",\n round(\n final_auc,\n 5\n )\n)\n\n\n# ============================================================\n# 28. SAVE AUC RESULTS\n# ============================================================\n\nauc_path = (\n \"/kaggle/working/\"\n \"mri_baseline_auc_results.csv\"\n)\n\nfinal_auc_df.to_csv(\n auc_path,\n index=False\n)\n\n\n# ============================================================\n# 29. FINAL SUMMARY\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 8 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… MRI Dataset created\")\nprint(\"āœ… DICOM preprocessing implemented\")\nprint(\"āœ… Multi-slice input implemented\")\nprint(\"āœ… Pretrained ResNet18 loaded\")\nprint(\"āœ… 12-target prediction head created\")\nprint(\"āœ… Missing labels ignored during loss\")\nprint(\"āœ… Training completed\")\nprint(\"āœ… Validation ROC-AUC calculated\")\nprint(\"āœ… Best model saved\")\n\nprint(\n \"\\nBest MRI Macro AUC:\",\n round(\n best_auc,\n 5\n )\n)\n\nprint(\n \"\\nModel:\",\n best_model_path\n)\n\nprint(\n \"AUC results:\",\n auc_path\n)\n\nprint(\"\\nšŸš€ READY FOR STEP 9\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 9 ā€” IMPROVED MRI SERIES + MULTI-SLICE PIPELINE\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport glob\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport torch\n\nfrom PIL import Image\nfrom torchvision import transforms\n\nprint(\"=\" * 70)\nprint(\"STEP 9 ā€” IMPROVED MRI SERIES + MULTI-SLICE SAMPLING\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. CONFIGURATION\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nTRAIN_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"train_series.csv\"\n)\n\nTRAIN_SERIES_DIR = os.path.join(\n DATA_PATH,\n \"train_series\"\n)\n\nIMAGE_SIZE = 224\n\n# More information than Step 8\nSLICES_PER_SERIES = 5\n\n# Maximum series per study\nMAX_SERIES_PER_STUDY = 4\n\nSEED = 42\n\nnp.random.seed(SEED)\n\n\n# ============================================================\n# 2. LOAD DATA\n# ============================================================\n\ntrain_df = pd.read_csv(\n TRAIN_CSV\n)\n\ntrain_series_df = pd.read_csv(\n TRAIN_SERIES_CSV\n)\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\n \"Training studies:\",\n len(train_df)\n)\n\nprint(\n \"Training series:\",\n len(train_series_df)\n)\n\n\n# ============================================================\n# 3. LABELLED STUDIES\n# ============================================================\n\nlabelled_mask = train_df[\n TARGETS\n].notna().any(axis=1)\n\nlabelled_df = train_df[\n labelled_mask\n].copy()\n\nprint(\n \"Labelled studies:\",\n len(labelled_df)\n)\n\n\n# ============================================================\n# 4. DICOM FINDER\n# ============================================================\n\ndef find_dicom_files(series_path):\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"*.dcm\"\n )\n )\n\n if len(files) == 0:\n\n files = glob.glob(\n os.path.join(\n series_path,\n \"**\",\n \"*.dcm\"\n ),\n recursive=True\n )\n\n return sorted(files)\n\n\n# ============================================================\n# 5. SERIES SELECTION STRATEGY\n# ============================================================\n\ndef select_best_series(\n study_uid,\n max_series=4\n):\n\n series_df = train_series_df[\n train_series_df[\n \"StudyInstanceUID\"\n ] == study_uid\n ].copy()\n\n if len(series_df) == 0:\n return []\n\n # --------------------------------------------------------\n # Give priority to:\n # 1. Fluid-sensitive\n # 2. Fat suppression\n # 3. Anatomical diversity\n # --------------------------------------------------------\n\n plane_priority = {\n \"Sagittal\": 3,\n \"Coronal\": 2,\n \"Axial\": 1\n }\n\n series_df[\"PlanePriority\"] = (\n series_df[\n \"Anatomical_Plane\"\n ]\n .map(plane_priority)\n .fillna(0)\n )\n\n series_df[\"Score\"] = (\n series_df[\"Fluid_Sensitive\"] * 4\n +\n series_df[\"Fat_Suppression\"] * 3\n +\n series_df[\"PlanePriority\"] * 0.5\n )\n\n series_df = series_df.sort_values(\n \"Score\",\n ascending=False\n )\n\n\n selected = []\n\n used_planes = set()\n\n\n # --------------------------------------------------------\n # First try to get different anatomical planes\n # --------------------------------------------------------\n\n for _, row in series_df.iterrows():\n\n plane = row[\n \"Anatomical_Plane\"\n ]\n\n if plane not in used_planes:\n\n selected.append(\n row\n )\n\n used_planes.add(\n plane\n )\n\n if len(selected) >= max_series:\n break\n\n\n # --------------------------------------------------------\n # Fill remaining slots by score\n # --------------------------------------------------------\n\n if len(selected) < max_series:\n\n selected_uids = {\n row[\"SeriesInstanceUID\"]\n for row in selected\n }\n\n for _, row in series_df.iterrows():\n\n if (\n row[\"SeriesInstanceUID\"]\n not in selected_uids\n ):\n\n selected.append(\n row\n )\n\n if len(selected) >= max_series:\n break\n\n\n return selected\n\n\n# ============================================================\n# 6. SLICE SORTING\n# ============================================================\n\ndef get_sorted_dicom_files(\n files\n):\n\n records = []\n\n for path in files:\n\n try:\n\n ds = pydicom.dcmread(\n path,\n stop_before_pixels=True\n )\n\n # Prefer InstanceNumber\n instance_number = getattr(\n ds,\n \"InstanceNumber\",\n None\n )\n\n # Or ImagePositionPatient\n position = getattr(\n ds,\n \"ImagePositionPatient\",\n None\n )\n\n if instance_number is not None:\n\n sort_value = float(\n instance_number\n )\n\n elif position is not None:\n\n try:\n sort_value = float(\n position[2]\n )\n except:\n sort_value = 0.0\n\n else:\n\n sort_value = 0.0\n\n records.append(\n (\n sort_value,\n path\n )\n )\n\n except:\n\n continue\n\n\n records.sort(\n key=lambda x: x[0]\n )\n\n return [\n path\n for _, path in records\n ]\n\n\n# ============================================================\n# 7. BETTER SLICE SAMPLING\n# ============================================================\n\ndef select_center_slices(\n files,\n num_slices=5\n):\n\n if len(files) == 0:\n return []\n\n files = get_sorted_dicom_files(\n files\n )\n\n if len(files) <= num_slices:\n\n return files\n\n\n # Avoid extreme slices.\n # Use central 80% of the sequence.\n\n start = int(\n len(files) * 0.10\n )\n\n end = int(\n len(files) * 0.90\n )\n\n usable = files[\n start:end\n ]\n\n\n if len(usable) < num_slices:\n\n usable = files\n\n\n indices = np.linspace(\n 0,\n len(usable) - 1,\n num_slices\n ).astype(int)\n\n\n return [\n usable[i]\n for i in indices\n ]\n\n\n# ============================================================\n# 8. DICOM IMAGE PREPROCESSING\n# ============================================================\n\ndef read_and_normalize_dicom(\n path\n):\n\n ds = pydicom.dcmread(\n path\n )\n\n image = ds.pixel_array.astype(\n np.float32\n )\n\n\n # MONOCHROME1 inversion\n if getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n ) == \"MONOCHROME1\":\n\n image = (\n image.max()\n - image\n )\n\n\n # Rescale\n slope = float(\n getattr(\n ds,\n \"RescaleSlope\",\n 1\n )\n )\n\n intercept = float(\n getattr(\n ds,\n \"RescaleIntercept\",\n 0\n )\n )\n\n image = (\n image * slope\n + intercept\n )\n\n\n # Robust intensity normalization\n low = np.percentile(\n image,\n 1\n )\n\n high = np.percentile(\n image,\n 99\n )\n\n\n image = np.clip(\n image,\n low,\n high\n )\n\n\n if high > low:\n\n image = (\n image - low\n ) / (\n high - low\n )\n\n else:\n\n image = np.zeros_like(\n image\n )\n\n\n return image\n\n\n# ============================================================\n# 9. IMAGE TRANSFORM\n# ============================================================\n\nmri_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n )\n])\n\n\n# ============================================================\n# 10. BUILD IMPROVED STUDY INDEX\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"BUILDING IMPROVED STUDY INDEX\")\nprint(\"=\" * 70)\n\n\nimproved_study_index = {}\n\nsuccessful = 0\n\n\nfor study_uid in labelled_df[\n \"StudyInstanceUID\"\n]:\n\n selected_series = select_best_series(\n study_uid,\n max_series=MAX_SERIES_PER_STUDY\n )\n\n study_entries = []\n\n\n for series_row in selected_series:\n\n series_uid = series_row[\n \"SeriesInstanceUID\"\n ]\n\n plane = series_row[\n \"Anatomical_Plane\"\n ]\n\n fluid = series_row[\n \"Fluid_Sensitive\"\n ]\n\n fat = series_row[\n \"Fat_Suppression\"\n ]\n\n\n series_path = os.path.join(\n TRAIN_SERIES_DIR,\n study_uid,\n series_uid\n )\n\n\n files = find_dicom_files(\n series_path\n )\n\n\n if len(files) == 0:\n continue\n\n\n selected_slices = select_center_slices(\n files,\n num_slices=SLICES_PER_SERIES\n )\n\n\n if len(selected_slices) == 0:\n continue\n\n\n study_entries.append({\n\n \"SeriesInstanceUID\":\n series_uid,\n\n \"Anatomical_Plane\":\n plane,\n\n \"Fluid_Sensitive\":\n fluid,\n\n \"Fat_Suppression\":\n fat,\n\n \"files\":\n selected_slices\n\n })\n\n\n if len(study_entries) > 0:\n\n improved_study_index[\n study_uid\n ] = study_entries\n\n successful += 1\n\n\n if successful % 100 == 0 and successful > 0:\n\n print(\n \"Processed studies:\",\n successful\n )\n\n\nprint(\n \"\\nStudies successfully indexed:\",\n len(\n improved_study_index\n )\n)\n\n\n# ============================================================\n# 11. INSPECT ONE STUDY\n# ============================================================\n\nsample_uid = next(\n iter(\n improved_study_index\n )\n)\n\nsample_series = improved_study_index[\n sample_uid\n]\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SAMPLE IMPROVED STUDY\")\nprint(\"=\" * 70)\n\nprint(\n \"Study UID:\",\n sample_uid\n)\n\nprint(\n \"Selected series:\",\n len(sample_series)\n)\n\n\nfor i, series in enumerate(\n sample_series\n):\n\n print(\n f\"\\nSeries {i + 1}\"\n )\n\n print(\n \"Plane:\",\n series[\n \"Anatomical_Plane\"\n ]\n )\n\n print(\n \"Fluid-sensitive:\",\n series[\n \"Fluid_Sensitive\"\n ]\n )\n\n print(\n \"Fat suppression:\",\n series[\n \"Fat_Suppression\"\n ]\n )\n\n print(\n \"Slices:\",\n len(\n series[\"files\"]\n )\n )\n\n\n# ============================================================\n# 12. LOAD MULTI-SERIES MRI\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LOADING MULTI-SERIES MRI\")\nprint(\"=\" * 70)\n\n\nall_images = []\n\nimage_info = []\n\n\nfor series in sample_series:\n\n for path in series[\n \"files\"\n ]:\n\n try:\n\n image = read_and_normalize_dicom(\n path\n )\n\n image = mri_transform(\n image\n )\n\n # Grayscale ā†’ RGB\n image = image.repeat(\n 3,\n 1,\n 1\n )\n\n all_images.append(\n image\n )\n\n image_info.append({\n\n \"plane\":\n series[\n \"Anatomical_Plane\"\n ],\n\n \"fluid\":\n series[\n \"Fluid_Sensitive\"\n ],\n\n \"fat\":\n series[\n \"Fat_Suppression\"\n ],\n\n \"path\":\n path\n\n })\n\n except Exception as e:\n\n print(\n \"DICOM error:\",\n e\n )\n\n\nif len(all_images) > 0:\n\n multi_series_tensor = torch.stack(\n all_images\n )\n\nelse:\n\n multi_series_tensor = torch.empty(\n (\n 0,\n 3,\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n )\n\n\nprint(\n \"\\nMulti-series tensor shape:\",\n multi_series_tensor.shape\n)\n\n\n# ============================================================\n# 13. FEATURE GROUP SUMMARY\n# ============================================================\n\nif len(image_info) > 0:\n\n info_df = pd.DataFrame(\n image_info\n )\n\n print(\"\\nSelected image distribution:\")\n\n print(\n info_df[\n \"plane\"\n ].value_counts()\n )\n\n print(\n \"\\nFluid-sensitive:\",\n )\n\n print(\n info_df[\n \"fluid\"\n ].value_counts()\n )\n\n print(\n \"\\nFat suppression:\"\n )\n\n print(\n info_df[\n \"fat\"\n ].value_counts()\n )\n\n\n# ============================================================\n# 14. DISPLAY MULTI-SERIES SLICES\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\n\nif len(all_images) > 0:\n\n display_count = min(\n len(all_images),\n 12\n )\n\n plt.figure(\n figsize=(18, 9)\n )\n\n\n for i in range(\n display_count\n ):\n\n ax = plt.subplot(\n 3,\n 4,\n i + 1\n )\n\n\n # Undo normalization approximately\n image = all_images[i].clone()\n\n mean = torch.tensor(\n [\n 0.485,\n 0.456,\n 0.406\n ]\n ).view(\n 3,\n 1,\n 1\n )\n\n std = torch.tensor(\n [\n 0.229,\n 0.224,\n 0.225\n ]\n ).view(\n 3,\n 1,\n 1\n )\n\n\n image = (\n image * std\n + mean\n )\n\n\n image = image[\n 0\n ].numpy()\n\n\n image = np.clip(\n image,\n 0,\n 1\n )\n\n\n ax.imshow(\n image,\n cmap=\"gray\"\n )\n\n\n info = image_info[i]\n\n\n ax.set_title(\n f\"{info['plane']} | \"\n f\"Fluid={info['fluid']} | \"\n f\"Fat={info['fat']}\"\n )\n\n\n ax.axis(\n \"off\"\n )\n\n\n plt.tight_layout()\n\n plt.show()\n\n\n# ============================================================\n# 15. COMPARE WITH STEP 8\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 9 DESIGN\")\nprint(\"=\" * 70)\n\nprint(\"\"\"\nSTEP 8:\nResNet18\n+\n3 slices\n+\nsimple sampling\n+\nsingle combined study stream\n\nSTEP 9:\nResNet18\n+\nmultiple MRI series\n+\nSagittal / Coronal / Axial diversity\n+\nFluid-sensitive priority\n+\nFat-suppression priority\n+\n5 central slices per series\n+\nbetter study representation\n\"\"\")\n\n\n# ============================================================\n# 16. SAVE INDEX\n# ============================================================\n\nindex_records = []\n\n\nfor study_uid, series_list in (\n improved_study_index.items()\n):\n\n for series in series_list:\n\n for slice_path in series[\n \"files\"\n ]:\n\n index_records.append({\n\n \"StudyInstanceUID\":\n study_uid,\n\n \"SeriesInstanceUID\":\n series[\n \"SeriesInstanceUID\"\n ],\n\n \"Anatomical_Plane\":\n series[\n \"Anatomical_Plane\"\n ],\n\n \"Fluid_Sensitive\":\n series[\n \"Fluid_Sensitive\"\n ],\n\n \"Fat_Suppression\":\n series[\n \"Fat_Suppression\"\n ],\n\n \"DICOM_Path\":\n slice_path\n\n })\n\n\nimproved_index_df = pd.DataFrame(\n index_records\n)\n\n\nINDEX_PATH = (\n \"/kaggle/working/\"\n \"improved_mri_series_index.csv\"\n)\n\n\nimproved_index_df.to_csv(\n INDEX_PATH,\n index=False\n)\n\n\nprint(\n \"\\nSaved:\",\n INDEX_PATH\n)\n\n\n# ============================================================\n# 17. FINAL CHECK\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 9 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… Multi-series selection implemented\"\n)\n\nprint(\n \"āœ… Sagittal / Coronal / Axial diversity considered\"\n)\n\nprint(\n \"āœ… Fluid-sensitive sequences prioritized\"\n)\n\nprint(\n \"āœ… Fat-suppressed sequences prioritized\"\n)\n\nprint(\n \"āœ… DICOM slices sorted\"\n)\n\nprint(\n \"āœ… Central slices selected\"\n)\n\nprint(\n \"āœ… Multi-series tensor generated\"\n)\n\nprint(\n \"āœ… Improved MRI index saved\"\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 10\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10 ā€” IMPROVED MULTI-SERIES DEEP LEARNING MODEL\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport pydicom\nimport torch\nimport torch.nn as nn\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n\n# ============================================================\n# 1. CONFIGURATION\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nIMPROVED_INDEX = (\n \"/kaggle/working/\"\n \"improved_mri_series_index.csv\"\n)\n\nIMAGE_SIZE = 224\n\nSLICES_PER_STUDY = 12\n\nBATCH_SIZE = 4\n\nEPOCHS = 3\n\nLEARNING_RATE = 1e-4\n\nSEED = 42\n\nMODEL_PATH = (\n \"/kaggle/working/\"\n \"rsna_multiseries_resnet18_best.pth\"\n)\n\n\n# ============================================================\n# 2. SEED\n# ============================================================\n\nnp.random.seed(SEED)\n\ntorch.manual_seed(SEED)\n\nif torch.cuda.is_available():\n\n torch.cuda.manual_seed_all(\n SEED\n )\n\n\n# ============================================================\n# 3. DEVICE\n# ============================================================\n\nDEVICE = torch.device(\n \"cuda\"\n if torch.cuda.is_available()\n else \"cpu\"\n)\n\nprint(\"=\" * 70)\nprint(\"STEP 10 ā€” MULTI-SERIES MRI MODEL\")\nprint(\"=\" * 70)\n\nprint(\n \"\\nDevice:\",\n DEVICE\n)\n\nif torch.cuda.is_available():\n\n print(\n \"GPU:\",\n torch.cuda.get_device_name(0)\n )\n\n\n# ============================================================\n# 4. TARGETS\n# ============================================================\n\nTARGETS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nNUM_TARGETS = len(TARGETS)\n\n\n# ============================================================\n# 5. LOAD DATA\n# ============================================================\n\nprint(\"\\n[1] Loading training metadata...\")\n\ntrain_df = pd.read_csv(\n TRAIN_CSV\n)\n\nindex_df = pd.read_csv(\n IMPROVED_INDEX\n)\n\nprint(\n \"Train shape:\",\n train_df.shape\n)\n\nprint(\n \"Improved index shape:\",\n index_df.shape\n)\n\n\n# ============================================================\n# 6. KEEP STUDIES WITH DICOM INDEX\n# ============================================================\n\navailable_uids = set(\n index_df[\n \"StudyInstanceUID\"\n ].unique()\n)\n\nmodel_df = train_df[\n train_df[\n \"StudyInstanceUID\"\n ].isin(\n available_uids\n )\n].copy()\n\n\n# Only studies with at least one label\nmodel_df = model_df[\n model_df[\n TARGETS\n ].notna().any(axis=1)\n].copy()\n\n\nmodel_df = model_df.reset_index(\n drop=True\n)\n\n\nprint(\n \"\\nStudies available for training:\",\n len(model_df)\n)\n\n\n# ============================================================\n# 7. TARGET MATRIX\n# ============================================================\n\nY = model_df[\n TARGETS\n].values.astype(\n np.float32\n)\n\n# Missing targets = -1\nY = np.nan_to_num(\n Y,\n nan=-1.0\n)\n\n\n# ============================================================\n# 8. STUDY ā†’ IMAGE INDEX\n# ============================================================\n\nprint(\n \"\\n[2] Creating study image index...\"\n)\n\n\nstudy_to_images = {}\n\n\nfor study_uid in model_df[\n \"StudyInstanceUID\"\n]:\n\n temp = index_df[\n index_df[\n \"StudyInstanceUID\"\n ] == study_uid\n ].copy()\n\n\n # --------------------------------------------------------\n # Priority:\n # Fluid-sensitive > Fat-suppressed > plane diversity\n # --------------------------------------------------------\n\n plane_priority = {\n\n \"Sagittal\": 3,\n\n \"Coronal\": 2,\n\n \"Axial\": 1\n\n }\n\n\n temp[\"PlaneScore\"] = (\n temp[\n \"Anatomical_Plane\"\n ]\n .map(\n plane_priority\n )\n .fillna(0)\n )\n\n\n temp[\"Priority\"] = (\n\n temp[\n \"Fluid_Sensitive\"\n ].astype(float) * 4\n\n +\n\n temp[\n \"Fat_Suppression\"\n ].astype(float) * 3\n\n +\n\n temp[\n \"PlaneScore\"\n ].astype(float)\n\n )\n\n\n temp = temp.sort_values(\n \"Priority\",\n ascending=False\n )\n\n\n # --------------------------------------------------------\n # Try to preserve anatomical diversity\n # --------------------------------------------------------\n\n selected_rows = []\n\n used_planes = set()\n\n\n for _, row in temp.iterrows():\n\n plane = row[\n \"Anatomical_Plane\"\n ]\n\n if plane not in used_planes:\n\n selected_rows.append(\n row\n )\n\n used_planes.add(\n plane\n )\n\n if len(selected_rows) >= 3:\n\n break\n\n\n # Fill remaining images\n selected_paths = [\n row[\"DICOM_Path\"]\n for row in selected_rows\n ]\n\n\n if len(selected_paths) < SLICES_PER_STUDY:\n\n remaining = temp[\n ~temp[\n \"DICOM_Path\"\n ].isin(\n selected_paths\n )\n ]\n\n\n for _, row in remaining.iterrows():\n\n selected_paths.append(\n row[\n \"DICOM_Path\"\n ]\n )\n\n if len(\n selected_paths\n ) >= SLICES_PER_STUDY:\n\n break\n\n\n # --------------------------------------------------------\n # Fixed number of images\n # --------------------------------------------------------\n\n if len(selected_paths) > SLICES_PER_STUDY:\n\n selected_paths = (\n selected_paths[\n :SLICES_PER_STUDY\n ]\n )\n\n\n study_to_images[\n study_uid\n ] = selected_paths\n\n\nprint(\n \"Indexed studies:\",\n len(study_to_images)\n)\n\n\n# ============================================================\n# 9. REMOVE STUDIES WITH NO IMAGE\n# ============================================================\n\nmodel_df = model_df[\n model_df[\n \"StudyInstanceUID\"\n ].isin(\n study_to_images.keys()\n )\n].reset_index(\n drop=True\n)\n\n\nY = model_df[\n TARGETS\n].values.astype(\n np.float32\n)\n\nY = np.nan_to_num(\n Y,\n nan=-1.0\n)\n\n\nprint(\n \"Final usable studies:\",\n len(model_df)\n)\n\n\n# ============================================================\n# 10. TRAIN / VALIDATION SPLIT\n# ============================================================\n\nindices = np.arange(\n len(model_df)\n)\n\n\ntrain_indices, valid_indices = train_test_split(\n\n indices,\n\n test_size=0.20,\n\n random_state=SEED\n)\n\n\nprint(\n \"\\nTraining:\",\n len(train_indices)\n)\n\nprint(\n \"Validation:\",\n len(valid_indices)\n)\n\n\n# ============================================================\n# 11. DICOM READER\n# ============================================================\n\ndef read_dicom_image(\n path\n):\n\n ds = pydicom.dcmread(\n path\n )\n\n image = ds.pixel_array.astype(\n np.float32\n )\n\n\n # MONOCHROME1\n if getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n ) == \"MONOCHROME1\":\n\n image = (\n image.max()\n - image\n )\n\n\n # Rescale\n slope = float(\n getattr(\n ds,\n \"RescaleSlope\",\n 1\n )\n )\n\n intercept = float(\n getattr(\n ds,\n \"RescaleIntercept\",\n 0\n )\n )\n\n\n image = (\n image * slope\n + intercept\n )\n\n\n # Percentile normalization\n low = np.percentile(\n image,\n 1\n )\n\n high = np.percentile(\n image,\n 99\n )\n\n\n image = np.clip(\n image,\n low,\n high\n )\n\n\n if high > low:\n\n image = (\n image - low\n ) / (\n high - low\n )\n\n else:\n\n image = np.zeros_like(\n image\n )\n\n\n return image\n\n\n# ============================================================\n# 12. TRANSFORM\n# ============================================================\n\ntrain_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n ),\n\n transforms.RandomHorizontalFlip(\n p=0.5\n ),\n\n transforms.RandomRotation(\n 5\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n\n )\n\n])\n\n\nvalid_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n\n )\n\n])\n\n\n# ============================================================\n# 13. DATASET\n# ============================================================\n\nclass MultiSeriesKneeDataset(\n Dataset\n):\n\n def __init__(\n self,\n df,\n targets,\n study_to_images,\n indices,\n transform\n ):\n\n self.df = df\n\n self.targets = targets\n\n self.study_to_images = (\n study_to_images\n )\n\n self.indices = indices\n\n self.transform = transform\n\n\n def __len__(self):\n\n return len(\n self.indices\n )\n\n\n def __getitem__(\n self,\n idx\n ):\n\n real_idx = self.indices[\n idx\n ]\n\n\n row = self.df.iloc[\n real_idx\n ]\n\n\n study_uid = row[\n \"StudyInstanceUID\"\n ]\n\n\n files = self.study_to_images[\n study_uid\n ]\n\n\n images = []\n\n\n for path in files:\n\n try:\n\n image = read_dicom_image(\n path\n )\n\n image = self.transform(\n image\n )\n\n # grayscale ā†’ RGB\n image = image.repeat(\n 3,\n 1,\n 1\n )\n\n images.append(\n image\n )\n\n except Exception:\n\n continue\n\n\n # ----------------------------------------------------\n # Padding if necessary\n # ----------------------------------------------------\n\n if len(images) == 0:\n\n blank = torch.zeros(\n 3,\n IMAGE_SIZE,\n IMAGE_SIZE\n )\n\n images = [\n blank\n for _ in range(\n SLICES_PER_STUDY\n )\n ]\n\n\n while len(images) < SLICES_PER_STUDY:\n\n images.append(\n images[-1].clone()\n )\n\n\n images = images[\n :SLICES_PER_STUDY\n ]\n\n\n images = torch.stack(\n images\n )\n\n\n target = torch.tensor(\n self.targets[\n real_idx\n ],\n dtype=torch.float32\n )\n\n\n return images, target\n\n\n# ============================================================\n# 14. DATASETS\n# ============================================================\n\ntrain_dataset = MultiSeriesKneeDataset(\n\n model_df,\n\n Y,\n\n study_to_images,\n\n train_indices,\n\n train_transform\n\n)\n\n\nvalid_dataset = MultiSeriesKneeDataset(\n\n model_df,\n\n Y,\n\n study_to_images,\n\n valid_indices,\n\n valid_transform\n\n)\n\n\n# ============================================================\n# 15. DATALOADERS\n# ============================================================\n\ntrain_loader = DataLoader(\n\n train_dataset,\n\n batch_size=BATCH_SIZE,\n\n shuffle=True,\n\n num_workers=2,\n\n pin_memory=True\n\n)\n\n\nvalid_loader = DataLoader(\n\n valid_dataset,\n\n batch_size=BATCH_SIZE,\n\n shuffle=False,\n\n num_workers=2,\n\n pin_memory=True\n\n)\n\n\n# ============================================================\n# 16. CHECK BATCH\n# ============================================================\n\nprint(\n \"\\n[3] Checking DataLoader...\"\n)\n\n\nsample_images, sample_targets = next(\n iter(train_loader)\n)\n\n\nprint(\n \"Image batch shape:\",\n sample_images.shape\n)\n\nprint(\n \"Target batch shape:\",\n sample_targets.shape\n)\n\n\n# Expected:\n#\n# [BATCH, 12, 3, 224, 224]\n#\n# [BATCH, 12]\n\n\n# ============================================================\n# 17. PRETRAINED RESNET18\n# ============================================================\n\nprint(\n \"\\n[4] Loading pretrained ResNet18...\"\n)\n\n\ntry:\n\n weights = (\n models.ResNet18_Weights.DEFAULT\n )\n\n backbone = models.resnet18(\n weights=weights\n )\n\nexcept Exception as e:\n\n print(\n \"Pretrained weights unavailable:\"\n )\n\n print(e)\n\n backbone = models.resnet18(\n weights=None\n )\n\n\nnum_features = (\n backbone.fc.in_features\n)\n\n\nbackbone.fc = nn.Identity()\n\n\n# ============================================================\n# 18. ATTENTION-LIKE SLICE AGGREGATION\n# ============================================================\n\nclass MultiSeriesResNet18(\n nn.Module\n):\n\n def __init__(\n self,\n backbone,\n feature_dim,\n num_targets=12\n ):\n\n super().__init__()\n\n\n self.backbone = backbone\n\n\n # Learn which slices/features matter\n self.attention = nn.Sequential(\n\n nn.Linear(\n feature_dim,\n 128\n ),\n\n nn.ReLU(),\n\n nn.Linear(\n 128,\n 1\n )\n\n )\n\n\n self.classifier = nn.Sequential(\n\n nn.Dropout(\n 0.3\n ),\n\n nn.Linear(\n feature_dim,\n 256\n ),\n\n nn.ReLU(),\n\n nn.Dropout(\n 0.2\n ),\n\n nn.Linear(\n 256,\n num_targets\n )\n\n )\n\n\n def forward(\n self,\n x\n ):\n\n # x:\n # [B, S, C, H, W]\n\n\n B = x.shape[0]\n\n S = x.shape[1]\n\n C = x.shape[2]\n\n H = x.shape[3]\n\n W = x.shape[4]\n\n\n # Flatten slices into batch\n x = x.view(\n B * S,\n C,\n H,\n W\n )\n\n\n features = self.backbone(\n x\n )\n\n\n # Restore slices\n features = features.view(\n B,\n S,\n -1\n )\n\n\n # Attention score per slice\n attention_scores = self.attention(\n features\n )\n\n\n attention_weights = torch.softmax(\n attention_scores,\n dim=1\n )\n\n\n # Weighted aggregation\n study_features = (\n features\n * attention_weights\n ).sum(\n dim=1\n )\n\n\n output = self.classifier(\n study_features\n )\n\n\n return output\n\n\nmodel = MultiSeriesResNet18(\n\n backbone,\n\n num_features,\n\n NUM_TARGETS\n\n)\n\n\nmodel = model.to(\n DEVICE\n)\n\n\nprint(\n \"Model created successfully.\"\n)\n\n\n# ============================================================\n# 19. LOSS\n# ============================================================\n\ncriterion = nn.BCEWithLogitsLoss(\n reduction=\"none\"\n)\n\n\n# ============================================================\n# 20. OPTIMIZER\n# ============================================================\n\noptimizer = torch.optim.AdamW(\n\n model.parameters(),\n\n lr=LEARNING_RATE,\n\n weight_decay=1e-4\n\n)\n\n\n# ============================================================\n# 21. TRAIN FUNCTION\n# ============================================================\n\ndef train_one_epoch():\n\n model.train()\n\n total_loss = 0.0\n\n batches = 0\n\n\n for images, targets in train_loader:\n\n images = images.to(\n DEVICE,\n non_blocking=True\n )\n\n targets = targets.to(\n DEVICE,\n non_blocking=True\n )\n\n\n optimizer.zero_grad()\n\n\n logits = model(\n images\n )\n\n\n valid_mask = (\n targets >= 0\n )\n\n\n safe_targets = torch.clamp(\n targets,\n 0,\n 1\n )\n\n\n losses = criterion(\n logits,\n safe_targets\n )\n\n\n masked_loss = losses[\n valid_mask\n ]\n\n\n if masked_loss.numel() == 0:\n\n continue\n\n\n loss = masked_loss.mean()\n\n\n loss.backward()\n\n\n optimizer.step()\n\n\n total_loss += (\n loss.item()\n )\n\n batches += 1\n\n\n if batches == 0:\n\n return 0.0\n\n\n return (\n total_loss\n /\n batches\n )\n\n\n# ============================================================\n# 22. VALIDATION\n# ============================================================\n\ndef validate_model():\n\n model.eval()\n\n\n all_predictions = []\n\n all_targets = []\n\n\n with torch.no_grad():\n\n for images, targets in valid_loader:\n\n images = images.to(\n DEVICE,\n non_blocking=True\n )\n\n\n logits = model(\n images\n )\n\n\n probabilities = torch.sigmoid(\n logits\n )\n\n\n all_predictions.append(\n probabilities.cpu().numpy()\n )\n\n all_targets.append(\n targets.numpy()\n )\n\n\n predictions = np.concatenate(\n all_predictions,\n axis=0\n )\n\n\n targets = np.concatenate(\n all_targets,\n axis=0\n )\n\n\n aucs = []\n\n\n for i, target_name in enumerate(\n TARGETS\n ):\n\n y_true = targets[:, i]\n\n y_pred = predictions[:, i]\n\n\n valid = (\n y_true >= 0\n )\n\n\n if (\n\n valid.sum() > 0\n\n and\n\n len(\n np.unique(\n y_true[valid]\n )\n ) == 2\n\n ):\n\n auc = roc_auc_score(\n\n y_true[valid],\n\n y_pred[valid]\n\n )\n\n aucs.append(\n auc\n )\n\n else:\n\n aucs.append(\n np.nan\n )\n\n\n macro_auc = np.nanmean(\n aucs\n )\n\n\n result = pd.DataFrame({\n\n \"Target\":\n TARGETS,\n\n \"AUC\":\n aucs\n\n })\n\n\n return (\n macro_auc,\n result,\n predictions,\n targets\n )\n\n\n# ============================================================\n# 23. TRAIN\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STARTING MULTI-SERIES TRAINING\")\nprint(\"=\" * 70)\n\n\nbest_auc = -1\n\n\nfor epoch in range(\n EPOCHS\n):\n\n print(\n f\"\\nEpoch {epoch + 1}/{EPOCHS}\"\n )\n\n\n train_loss = train_one_epoch()\n\n\n print(\n \"Training Loss:\",\n round(\n train_loss,\n 5\n )\n )\n\n\n val_auc, auc_df, _, _ = (\n validate_model()\n )\n\n\n print(\n \"Validation Macro AUC:\",\n round(\n val_auc,\n 5\n )\n )\n\n\n if val_auc > best_auc:\n\n best_auc = val_auc\n\n\n torch.save(\n\n model.state_dict(),\n\n MODEL_PATH\n\n )\n\n\n print(\n \"āœ… New best model saved\"\n )\n\n\n# ============================================================\n# 24. RESTORE BEST MODEL\n# ============================================================\n\nif os.path.exists(\n MODEL_PATH\n):\n\n model.load_state_dict(\n\n torch.load(\n\n MODEL_PATH,\n\n map_location=DEVICE\n\n )\n\n )\n\n\n print(\n \"\\nāœ… Best model restored\"\n )\n\n\n# ============================================================\n# 25. FINAL VALIDATION\n# ============================================================\n\nfinal_auc, final_auc_df, predictions, targets = (\n validate_model()\n)\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 10 RESULTS\")\nprint(\"=\" * 70)\n\n\ndisplay(\n final_auc_df\n)\n\n\nprint(\n \"\\nImproved Multi-Series Macro ROC-AUC:\",\n round(\n final_auc,\n 5\n )\n)\n\n\n# ============================================================\n# 26. COMPARE BASELINES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MODEL COMPARISON\")\nprint(\"=\" * 70)\n\n\ncomparison_df = pd.DataFrame({\n\n \"Model\": [\n\n \"TF-IDF + Logistic Regression\",\n\n \"MRI ResNet18 Baseline\",\n\n \"Improved Multi-Series ResNet18\"\n\n ],\n\n \"Macro_AUC\": [\n\n 0.5459,\n\n 0.5474,\n\n final_auc\n\n ]\n\n})\n\n\ndisplay(\n comparison_df\n)\n\n\n# ============================================================\n# 27. SAVE RESULTS\n# ============================================================\n\nRESULT_PATH = (\n \"/kaggle/working/\"\n \"step10_multiseries_auc_results.csv\"\n)\n\n\ncomparison_df.to_csv(\n RESULT_PATH,\n index=False\n)\n\n\nPER_TARGET_PATH = (\n \"/kaggle/working/\"\n \"step10_per_target_auc.csv\"\n)\n\n\nfinal_auc_df.to_csv(\n PER_TARGET_PATH,\n index=False\n)\n\n\n# ============================================================\n# 28. FINAL\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 10 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… Improved multi-series dataset created\"\n)\n\nprint(\n \"āœ… Multiple MRI slices processed\"\n)\n\nprint(\n \"āœ… ResNet18 feature extractor used\"\n)\n\nprint(\n \"āœ… Learnable slice attention implemented\"\n)\n\nprint(\n \"āœ… Study-level feature aggregation implemented\"\n)\n\nprint(\n \"āœ… 12 abnormality predictions generated\"\n)\n\nprint(\n \"āœ… Macro ROC-AUC calculated\"\n)\n\nprint(\n \"āœ… Baseline comparison completed\"\n)\n\nprint(\n \"\\nBest Macro AUC:\",\n round(\n best_auc,\n 5\n )\n)\n\nprint(\n \"\\nModel saved:\",\n MODEL_PATH\n)\n\nprint(\n \"\\nComparison saved:\",\n RESULT_PATH\n)\n\nprint(\n \"\\nPer-target results saved:\",\n PER_TARGET_PATH\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 11\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 11 ā€” CLASS IMBALANCE + BALANCED TRAINING\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.metrics import roc_auc_score\nfrom torch.utils.data import DataLoader\n\n\nprint(\"=\" * 70)\nprint(\"STEP 11 ā€” CLASS IMBALANCE + BALANCED TRAINING\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. CHECK REQUIRED OBJECTS FROM STEP 10\n# ============================================================\n\nrequired_objects = [\n \"model_df\",\n \"Y\",\n \"train_indices\",\n \"valid_indices\",\n \"train_dataset\",\n \"valid_dataset\",\n \"train_loader\",\n \"valid_loader\",\n \"study_to_images\",\n \"TARGETS\",\n \"DEVICE\"\n]\n\nmissing = []\n\nfor obj in required_objects:\n\n if obj not in globals():\n\n missing.append(obj)\n\n\nif len(missing) > 0:\n\n raise RuntimeError(\n \"Missing objects from STEP 10: \"\n + str(missing)\n + \"\\nPlease run STEP 10 first.\"\n )\n\n\nprint(\"\\nāœ… STEP 10 objects found\")\n\n\n# ============================================================\n# 2. TARGET DISTRIBUTION\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TARGET DISTRIBUTION\")\nprint(\"=\" * 70)\n\n\ndistribution_rows = []\n\n\nfor i, target in enumerate(TARGETS):\n\n values = Y[:, i]\n\n labeled = values >= 0\n\n positives = (\n values[labeled] == 1\n ).sum()\n\n negatives = (\n values[labeled] == 0\n ).sum()\n\n missing_count = (\n values < 0\n ).sum()\n\n positive_rate = (\n\n positives / labeled.sum()\n\n if labeled.sum() > 0\n\n else np.nan\n\n )\n\n\n distribution_rows.append({\n\n \"Target\": target,\n\n \"Positive\": int(\n positives\n ),\n\n \"Negative\": int(\n negatives\n ),\n\n \"Missing\": int(\n missing_count\n ),\n\n \"Positive_%\": round(\n positive_rate * 100,\n 2\n )\n\n })\n\n\ndistribution_df = pd.DataFrame(\n distribution_rows\n)\n\n\ndisplay(\n distribution_df\n)\n\n\n# ============================================================\n# 3. CALCULATE POSITIVE WEIGHTS\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"CALCULATING CLASS WEIGHTS\")\nprint(\"=\" * 70)\n\n\npos_weights = []\n\n\nfor i, target in enumerate(TARGETS):\n\n values = Y[:, i]\n\n labeled = values >= 0\n\n positives = (\n values[labeled] == 1\n ).sum()\n\n negatives = (\n values[labeled] == 0\n ).sum()\n\n\n if positives > 0:\n\n weight = (\n negatives\n /\n positives\n )\n\n else:\n\n weight = 1.0\n\n\n # Avoid extreme weights\n weight = np.clip(\n weight,\n 1.0,\n 10.0\n )\n\n\n pos_weights.append(\n weight\n )\n\n\npos_weights = torch.tensor(\n pos_weights,\n dtype=torch.float32,\n device=DEVICE\n)\n\n\nweight_df = pd.DataFrame({\n\n \"Target\": TARGETS,\n\n \"Positive_Weight\":\n pos_weights.cpu().numpy()\n\n})\n\n\ndisplay(\n weight_df\n)\n\n\n# ============================================================\n# 4. CREATE FRESH DATA LOADERS\n# ============================================================\n\nprint(\"\\n[1] Creating balanced-training DataLoaders...\")\n\n\nBATCH_SIZE_STEP11 = 4\n\n\ntrain_loader_11 = DataLoader(\n\n train_dataset,\n\n batch_size=BATCH_SIZE_STEP11,\n\n shuffle=True,\n\n num_workers=2,\n\n pin_memory=True\n\n)\n\n\nvalid_loader_11 = DataLoader(\n\n valid_dataset,\n\n batch_size=BATCH_SIZE_STEP11,\n\n shuffle=False,\n\n num_workers=2,\n\n pin_memory=True\n\n)\n\n\nprint(\n \"Training batches:\",\n len(train_loader_11)\n)\n\nprint(\n \"Validation batches:\",\n len(valid_loader_11)\n)\n\n\n# ============================================================\n# 5. REINITIALIZE MODEL\n# ============================================================\n\nprint(\"\\n[2] Creating fresh model...\")\n\n\n# Re-create backbone\ntry:\n\n weights = (\n models.ResNet18_Weights.DEFAULT\n )\n\n backbone_11 = models.resnet18(\n weights=weights\n )\n\nexcept Exception:\n\n backbone_11 = models.resnet18(\n weights=None\n )\n\n\nnum_features_11 = (\n backbone_11.fc.in_features\n)\n\n\nbackbone_11.fc = nn.Identity()\n\n\nmodel_11 = MultiSeriesResNet18(\n\n backbone_11,\n\n num_features_11,\n\n NUM_TARGETS\n\n)\n\n\nmodel_11 = model_11.to(\n DEVICE\n)\n\n\nprint(\n \"āœ… Fresh model created\"\n)\n\n\n# ============================================================\n# 6. BALANCED LOSS\n# ============================================================\n\ncriterion_11 = nn.BCEWithLogitsLoss(\n\n pos_weight=pos_weights,\n\n reduction=\"none\"\n\n)\n\n\n# ============================================================\n# 7. OPTIMIZER\n# ============================================================\n\noptimizer_11 = torch.optim.AdamW(\n\n model_11.parameters(),\n\n lr=5e-5,\n\n weight_decay=1e-4\n\n)\n\n\n# ============================================================\n# 8. LEARNING RATE SCHEDULER\n# ============================================================\n\nscheduler_11 = torch.optim.lr_scheduler.CosineAnnealingLR(\n\n optimizer_11,\n\n T_max=5\n\n)\n\n\n# ============================================================\n# 9. TRAINING FUNCTION\n# ============================================================\n\ndef train_step11():\n\n model_11.train()\n\n total_loss = 0.0\n\n batches = 0\n\n\n for images, targets in train_loader_11:\n\n images = images.to(\n\n DEVICE,\n\n non_blocking=True\n\n )\n\n targets = targets.to(\n\n DEVICE,\n\n non_blocking=True\n\n )\n\n\n optimizer_11.zero_grad(\n set_to_none=True\n )\n\n\n logits = model_11(\n images\n )\n\n\n valid_mask = (\n targets >= 0\n )\n\n\n safe_targets = torch.clamp(\n\n targets,\n\n 0,\n\n 1\n\n )\n\n\n losses = criterion_11(\n\n logits,\n\n safe_targets\n\n )\n\n\n # Ignore missing labels\n masked_losses = losses[\n valid_mask\n ]\n\n\n if masked_losses.numel() == 0:\n\n continue\n\n\n loss = masked_losses.mean()\n\n\n loss.backward()\n\n\n torch.nn.utils.clip_grad_norm_(\n\n model_11.parameters(),\n\n max_norm=1.0\n\n )\n\n\n optimizer_11.step()\n\n\n total_loss += (\n loss.item()\n )\n\n batches += 1\n\n\n if batches == 0:\n\n return 0.0\n\n\n return (\n total_loss\n /\n batches\n )\n\n\n# ============================================================\n# 10. VALIDATION FUNCTION\n# ============================================================\n\ndef validate_step11():\n\n model_11.eval()\n\n\n predictions = []\n\n targets_list = []\n\n\n with torch.no_grad():\n\n for images, targets in valid_loader_11:\n\n images = images.to(\n\n DEVICE,\n\n non_blocking=True\n\n )\n\n\n logits = model_11(\n images\n )\n\n\n probs = torch.sigmoid(\n logits\n )\n\n\n predictions.append(\n\n probs.cpu().numpy()\n\n )\n\n\n targets_list.append(\n\n targets.numpy()\n\n )\n\n\n predictions = np.concatenate(\n\n predictions,\n\n axis=0\n\n )\n\n\n targets_array = np.concatenate(\n\n targets_list,\n\n axis=0\n\n )\n\n\n aucs = []\n\n\n for i, target_name in enumerate(\n TARGETS\n ):\n\n y_true = targets_array[:, i]\n\n y_pred = predictions[:, i]\n\n\n valid = (\n y_true >= 0\n )\n\n\n if (\n\n valid.sum() > 1\n\n and\n\n len(\n np.unique(\n y_true[valid]\n )\n ) == 2\n\n ):\n\n auc = roc_auc_score(\n\n y_true[valid],\n\n y_pred[valid]\n\n )\n\n else:\n\n auc = np.nan\n\n\n aucs.append(\n auc\n )\n\n\n macro_auc = np.nanmean(\n aucs\n )\n\n\n result_df = pd.DataFrame({\n\n \"Target\":\n TARGETS,\n\n \"AUC\":\n aucs\n\n })\n\n\n return (\n\n macro_auc,\n\n result_df,\n\n predictions,\n\n targets_array\n\n )\n\n\n# ============================================================\n# 11. TRAINING LOOP\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STARTING STEP 11 TRAINING\")\nprint(\"=\" * 70)\n\n\nEPOCHS_11 = 5\n\nbest_auc_11 = -1\n\nbest_epoch_11 = -1\n\n\nhistory_11 = []\n\n\nfor epoch in range(\n EPOCHS_11\n):\n\n print(\n f\"\\nEpoch {epoch + 1}/{EPOCHS_11}\"\n )\n\n\n train_loss = train_step11()\n\n\n val_auc, auc_df, _, _ = (\n validate_step11()\n )\n\n\n scheduler_11.step()\n\n\n current_lr = (\n optimizer_11\n .param_groups[0][\"lr\"]\n )\n\n\n print(\n \"Training Loss:\",\n round(\n train_loss,\n 5\n )\n )\n\n\n print(\n \"Validation Macro AUC:\",\n round(\n val_auc,\n 5\n )\n )\n\n\n print(\n \"Learning Rate:\",\n current_lr\n )\n\n\n history_11.append({\n\n \"Epoch\":\n epoch + 1,\n\n \"Train_Loss\":\n train_loss,\n\n \"Val_Macro_AUC\":\n val_auc,\n\n \"Learning_Rate\":\n current_lr\n\n })\n\n\n if val_auc > best_auc_11:\n\n best_auc_11 = val_auc\n\n best_epoch_11 = (\n epoch + 1\n )\n\n\n torch.save(\n\n model_11.state_dict(),\n\n \"/kaggle/working/\"\n \"rsna_step11_best.pth\"\n\n )\n\n\n print(\n \"āœ… Best Step 11 model saved\"\n )\n\n\n# ============================================================\n# 12. TRAINING HISTORY\n# ============================================================\n\nhistory_df = pd.DataFrame(\n history_11\n)\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRAINING HISTORY\")\nprint(\"=\" * 70)\n\n\ndisplay(\n history_df\n)\n\n\nhistory_df.to_csv(\n\n \"/kaggle/working/\"\n \"step11_training_history.csv\",\n\n index=False\n\n)\n\n\n# ============================================================\n# 13. RESTORE BEST MODEL\n# ============================================================\n\nbest_model_path = (\n\n \"/kaggle/working/\"\n \"rsna_step11_best.pth\"\n\n)\n\n\nif os.path.exists(\n best_model_path\n):\n\n model_11.load_state_dict(\n\n torch.load(\n\n best_model_path,\n\n map_location=DEVICE\n\n )\n\n )\n\n\n print(\n \"\\nāœ… Best Step 11 model restored\"\n )\n\n\n# ============================================================\n# 14. FINAL VALIDATION\n# ============================================================\n\nfinal_auc_11, final_auc_df_11, final_predictions_11, final_targets_11 = (\n\n validate_step11()\n\n)\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 11 FINAL RESULTS\")\nprint(\"=\" * 70)\n\n\nprint(\n\n \"\\nBest Macro ROC-AUC:\",\n\n round(\n best_auc_11,\n 5\n )\n\n)\n\n\nprint(\n\n \"Best Epoch:\",\n\n best_epoch_11\n\n)\n\n\ndisplay(\n final_auc_df_11\n)\n\n\n# ============================================================\n# 15. PREDICTION VARIANCE CHECK\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"PREDICTION SANITY CHECK\")\nprint(\"=\" * 70)\n\n\nprediction_stats = pd.DataFrame({\n\n \"Target\":\n TARGETS,\n\n \"Mean\":\n final_predictions_11.mean(\n axis=0\n ),\n\n \"Std\":\n final_predictions_11.std(\n axis=0\n ),\n\n \"Min\":\n final_predictions_11.min(\n axis=0\n ),\n\n \"Max\":\n final_predictions_11.max(\n axis=0\n )\n\n})\n\n\ndisplay(\n prediction_stats\n)\n\n\n# ============================================================\n# 16. COMPARE ALL MODELS\n# ============================================================\n\ncomparison_step11 = pd.DataFrame({\n\n \"Model\": [\n\n \"TF-IDF + Logistic Regression\",\n\n \"MRI ResNet18 Baseline\",\n\n \"Multi-Series ResNet18\",\n\n \"Balanced Multi-Series ResNet18\"\n\n ],\n\n \"Macro_AUC\": [\n\n 0.5459,\n\n 0.5474,\n\n 0.5000,\n\n final_auc_11\n\n ]\n\n})\n\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ALL MODEL COMPARISON\")\nprint(\"=\" * 70)\n\n\ndisplay(\n comparison_step11\n)\n\n\ncomparison_step11.to_csv(\n\n \"/kaggle/working/\"\n \"step11_model_comparison.csv\",\n\n index=False\n\n)\n\n\n# ============================================================\n# 17. SAVE PER-TARGET RESULTS\n# ============================================================\n\nfinal_auc_df_11.to_csv(\n\n \"/kaggle/working/\"\n \"step11_per_target_auc.csv\",\n\n index=False\n\n)\n\n\n# ============================================================\n# 18. FINAL\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 11 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… Target imbalance analyzed\"\n)\n\nprint(\n \"āœ… Positive class weights calculated\"\n)\n\nprint(\n \"āœ… Missing labels ignored\"\n)\n\nprint(\n \"āœ… Balanced BCE loss implemented\"\n)\n\nprint(\n \"āœ… Gradient clipping implemented\"\n)\n\nprint(\n \"āœ… Learning-rate scheduler implemented\"\n)\n\nprint(\n \"āœ… Best model saved\"\n)\n\nprint(\n \"\\nBest Macro AUC:\",\n round(\n best_auc_11,\n 5\n )\n)\n\nprint(\n \"\\nModel:\",\n best_model_path\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 12\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 12 ā€” MULTIMODAL TRAINING PREPARATION\n# MRI + RADIOLOGY REPORT\n# ============================================================\n\nimport os\nimport re\nimport numpy as np\nimport pandas as pd\nimport torch\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\n\n\nprint(\"=\" * 70)\nprint(\"STEP 12 ā€” MULTIMODAL DATA PREPARATION\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. FIND DATASET\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nTEST_CSV = os.path.join(\n DATA_PATH,\n \"test.csv\"\n)\n\nTRAIN_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"train_series.csv\"\n)\n\nTEST_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"test_series.csv\"\n)\n\n\nprint(\"\\n[1] Checking dataset...\")\n\n\nfor path in [\n TRAIN_CSV,\n TEST_CSV,\n TRAIN_SERIES_CSV,\n TEST_SERIES_CSV\n]:\n\n if not os.path.exists(path):\n\n raise FileNotFoundError(\n f\"Missing file: {path}\"\n )\n\n\nprint(\"āœ… Dataset files found\")\n\n\n# ============================================================\n# 2. LOAD DATA\n# ============================================================\n\ntrain_df_12 = pd.read_csv(\n TRAIN_CSV\n)\n\ntest_df_12 = pd.read_csv(\n TEST_CSV\n)\n\ntrain_series_df_12 = pd.read_csv(\n TRAIN_SERIES_CSV\n)\n\ntest_series_df_12 = pd.read_csv(\n TEST_SERIES_CSV\n)\n\n\nprint(\"\\nTrain:\", train_df_12.shape)\n\nprint(\n \"Train Series:\",\n train_series_df_12.shape\n)\n\nprint(\n \"Test:\",\n test_df_12.shape\n)\n\nprint(\n \"Test Series:\",\n test_series_df_12.shape\n)\n\n\n# ============================================================\n# 3. TARGETS\n# ============================================================\n\nTARGETS_12 = [\n\n \"ACL\",\n\n \"MCL\",\n\n \"Medial Meniscus\",\n\n \"Lateral Meniscus\",\n\n \"Medial OA\",\n\n \"Lateral OA\",\n\n \"PF OA\",\n\n \"Effusion\",\n\n \"Synovitis\",\n\n \"Baker's\",\n\n \"Contusion\",\n\n \"Fracture\"\n\n]\n\n\nmissing_targets = [\n\n col\n\n for col in TARGETS_12\n\n if col not in train_df_12.columns\n\n]\n\n\nif missing_targets:\n\n raise ValueError(\n \"Missing target columns: \"\n + str(missing_targets)\n )\n\n\nprint(\n \"\\nāœ… All 12 targets found\"\n)\n\n\n# ============================================================\n# 4. REPORT CLEANING\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"REPORT PREPROCESSING\")\n\nprint(\"=\" * 70)\n\n\ndef clean_report(text):\n\n if pd.isna(text):\n\n return \"\"\n\n text = str(text)\n\n # Lowercase\n text = text.lower()\n\n # Replace line breaks\n text = re.sub(\n r\"\\s+\",\n \" \",\n text\n )\n\n # Remove excessive symbols\n text = re.sub(\n r\"[^\\w\\s\\-]\",\n \" \",\n text\n )\n\n # Remove repeated spaces\n text = re.sub(\n r\"\\s+\",\n \" \",\n text\n )\n\n return text.strip()\n\n\ntrain_df_12[\"Report_Clean\"] = (\n\n train_df_12[\"Report\"]\n\n .fillna(\"\")\n\n .apply(clean_report)\n\n)\n\n\nprint(\n \"Reports:\",\n len(train_df_12)\n)\n\nprint(\n \"Empty reports:\",\n (\n train_df_12[\"Report_Clean\"]\n .str.len()\n .eq(0)\n .sum()\n )\n)\n\n\nprint(\n \"\\nExample cleaned report:\"\n)\n\nprint(\n train_df_12[\n \"Report_Clean\"\n ].iloc[0][:1000]\n)\n\n\n# ============================================================\n# 5. REPORT LENGTH FEATURES\n# ============================================================\n\ntrain_df_12[\"Report_Length\"] = (\n\n train_df_12[\n \"Report_Clean\"\n ]\n .str.len()\n\n)\n\n\ntrain_df_12[\"Report_Word_Count\"] = (\n\n train_df_12[\n \"Report_Clean\"\n ]\n .str.split()\n .str.len()\n\n)\n\n\nprint(\"\\nReport statistics:\")\n\ndisplay(\n\n train_df_12[\n [\n \"Report_Length\",\n \"Report_Word_Count\"\n ]\n ]\n .describe()\n\n)\n\n\n# ============================================================\n# 6. KEY CLINICAL TERM CHECK\n# ============================================================\n\nclinical_terms = {\n\n \"ACL\": [\n \"acl\",\n \"cruciate\",\n \"ligament\"\n ],\n\n \"MCL\": [\n \"mcl\",\n \"medial collateral\"\n ],\n\n \"Medial Meniscus\": [\n \"medial meniscus\",\n \"meniscus medialis\"\n ],\n\n \"Lateral Meniscus\": [\n \"lateral meniscus\",\n \"meniscus lateralis\"\n ],\n\n \"Medial OA\": [\n \"medial osteoarthritis\",\n \"medial arthrosis\",\n \"medial compartment\"\n ],\n\n \"Lateral OA\": [\n \"lateral osteoarthritis\",\n \"lateral arthrosis\",\n \"lateral compartment\"\n ],\n\n \"PF OA\": [\n \"patellofemoral\",\n \"patella\",\n \"pf oa\"\n ],\n\n \"Effusion\": [\n \"effusion\",\n \"joint fluid\",\n \"joint effusion\",\n \"derr ament\"\n ],\n\n \"Synovitis\": [\n \"synovitis\",\n \"synovial\"\n ],\n\n \"Baker's\": [\n \"baker\",\n \"popliteal cyst\",\n \"popliteal\"\n ],\n\n \"Contusion\": [\n \"contusion\",\n \"bone bruise\",\n \"bone marrow edema\"\n ],\n\n \"Fracture\": [\n \"fracture\",\n \"fractured\"\n ]\n\n}\n\n\nterm_stats = []\n\n\nfor target, terms in clinical_terms.items():\n\n count = 0\n\n for term in terms:\n\n count += (\n\n train_df_12[\n \"Report_Clean\"\n ]\n .str.contains(\n term,\n regex=False,\n na=False\n )\n .sum()\n\n )\n\n\n term_stats.append({\n\n \"Target\": target,\n\n \"Keyword_Matches\": int(count)\n\n })\n\n\nterm_stats_df = pd.DataFrame(\n term_stats\n)\n\n\ndisplay(\n term_stats_df\n)\n\n\n# ============================================================\n# 7. TF-IDF REPRESENTATION\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"BUILDING TF-IDF REPORT FEATURES\")\n\nprint(\"=\" * 70)\n\n\nvectorizer_12 = TfidfVectorizer(\n\n max_features=5000,\n\n ngram_range=(1, 2),\n\n min_df=2,\n\n max_df=0.98,\n\n sublinear_tf=True\n\n)\n\n\nX_text_12 = vectorizer_12.fit_transform(\n\n train_df_12[\n \"Report_Clean\"\n ]\n\n)\n\n\nprint(\n \"TF-IDF shape:\",\n X_text_12.shape\n)\n\n\nprint(\n \"Vocabulary size:\",\n len(\n vectorizer_12.vocabulary_\n )\n)\n\n\n# ============================================================\n# 8. NUMERIC REPORT FEATURES\n# ============================================================\n\nnumeric_text_features = (\n\n train_df_12[\n [\n \"Report_Length\",\n \"Report_Word_Count\"\n ]\n ]\n .values\n .astype(np.float32)\n\n)\n\n\nscaler_text_12 = StandardScaler()\n\n\nnumeric_text_features = (\n\n scaler_text_12\n .fit_transform(\n numeric_text_features\n )\n\n)\n\n\nprint(\n \"\\nNumeric text features:\",\n numeric_text_features.shape\n)\n\n\n# ============================================================\n# 9. CHECK LABELLED STUDIES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"LABEL AVAILABILITY\")\n\nprint(\"=\" * 70)\n\n\nlabel_matrix_12 = (\n\n train_df_12[\n TARGETS_12\n ]\n .values\n .astype(np.float32)\n\n)\n\n\nlabelled_mask_12 = (\n label_matrix_12 >= 0\n)\n\n\nlabelled_rows = (\n labelled_mask_12.any(\n axis=1\n )\n)\n\n\nprint(\n \"Total studies:\",\n len(train_df_12)\n)\n\nprint(\n \"Studies with labels:\",\n labelled_rows.sum()\n)\n\nprint(\n \"Studies without labels:\",\n (\n ~labelled_rows\n ).sum()\n)\n\n\n# ============================================================\n# 10. REPORT + LABEL OVERLAP\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"LABELLED REPORT DATA\")\n\nprint(\"=\" * 70)\n\n\nlabelled_report_df = train_df_12[\n labelled_rows\n].copy()\n\n\nprint(\n \"Labelled reports:\",\n len(labelled_report_df)\n)\n\n\nfor target in TARGETS_12:\n\n valid = (\n\n labelled_report_df[\n target\n ].notna()\n\n )\n\n\n positives = (\n\n labelled_report_df.loc[\n valid,\n target\n ] == 1\n\n ).sum()\n\n\n negatives = (\n\n labelled_report_df.loc[\n valid,\n target\n ] == 0\n\n ).sum()\n\n\n print(\n\n f\"{target:20s}\"\n f\" Positive={positives:3d}\"\n f\" Negative={negatives:3d}\"\n\n )\n\n\n# ============================================================\n# 11. SAVE TEXT FEATURES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"SAVING STEP 12 FEATURES\")\n\nprint(\"=\" * 70)\n\n\nTEXT_FEATURE_PATH = (\n\n \"/kaggle/working/\"\n \"rsna_step12_text_features.npz\"\n\n)\n\n\nfrom scipy.sparse import save_npz\n\n\nsave_npz(\n\n TEXT_FEATURE_PATH,\n\n X_text_12\n\n)\n\n\ntrain_df_12.to_csv(\n\n \"/kaggle/working/\"\n \"step12_train_prepared.csv\",\n\n index=False\n\n)\n\n\nterm_stats_df.to_csv(\n\n \"/kaggle/working/\"\n \"step12_clinical_term_stats.csv\",\n\n index=False\n\n)\n\n\n# ============================================================\n# 12. FINAL SUMMARY\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"STEP 12 COMPLETE\")\n\nprint(\"=\" * 70)\n\n\nprint(\n \"āœ… Radiology reports cleaned\"\n)\n\nprint(\n \"āœ… Report length features created\"\n)\n\nprint(\n \"āœ… Clinical keywords analyzed\"\n)\n\nprint(\n \"āœ… TF-IDF features generated\"\n)\n\nprint(\n \"āœ… Numeric text features generated\"\n)\n\nprint(\n \"āœ… Label availability checked\"\n)\n\nprint(\n \"āœ… Labelled report studies identified\"\n)\n\nprint(\n \"\\nTF-IDF features:\",\n X_text_12.shape\n)\n\nprint(\n \"Text feature file:\",\n TEXT_FEATURE_PATH\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 13\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 13 ā€” MULTIMODAL MRI + RADIOLOGY TEXT FUSION\n# RSNA KNEE ABNORMALITY DETECTION\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nprint(\"=\" * 70)\nprint(\"STEP 13 ā€” MRI + RADIOLOGY TEXT FUSION\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. CHECK STEP 12\n# ============================================================\n\nrequired_step12 = [\n \"train_df_12\",\n \"TARGETS_12\",\n \"X_text_12\",\n \"vectorizer_12\",\n \"label_matrix_12\"\n]\n\nmissing = [\n x for x in required_step12\n if x not in globals()\n]\n\nif missing:\n raise RuntimeError(\n \"STEP 12 objects missing: \"\n + str(missing)\n + \"\\nRun STEP 12 first.\"\n )\n\nprint(\"\\nāœ… STEP 12 objects found\")\n\n\n# ============================================================\n# 2. CHECK MRI MODEL / FEATURES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"CHECKING MRI PIPELINE\")\nprint(\"=\" * 70)\n\nfor name in [\n \"model_11\",\n \"train_dataset\",\n \"valid_dataset\",\n \"train_indices\",\n \"valid_indices\"\n]:\n\n if name in globals():\n print(\"āœ…\", name)\n else:\n print(\"āš ļø\", name, \"not found\")\n\n\n# ============================================================\n# 3. LABELLED DATA ONLY\n# ============================================================\n\nlabel_matrix = label_matrix_12.copy()\n\nlabelled_mask = (\n label_matrix >= 0\n).any(axis=1)\n\nlabelled_df = train_df_12[\n labelled_mask\n].copy()\n\nlabelled_indices = np.where(\n labelled_mask\n)[0]\n\n\nprint(\n \"\\nTotal training studies:\",\n len(train_df_12)\n)\n\nprint(\n \"Studies containing labels:\",\n len(labelled_df)\n)\n\n\n# ============================================================\n# 4. TEXT FEATURES FOR LABELLED STUDIES\n# ============================================================\n\nX_text_labelled = (\n X_text_12[\n labelled_indices\n ]\n)\n\n\nY_labelled = (\n label_matrix[\n labelled_indices\n ]\n)\n\n\nprint(\n \"\\nText feature matrix:\",\n X_text_labelled.shape\n)\n\nprint(\n \"Label matrix:\",\n Y_labelled.shape\n)\n\n\n# ============================================================\n# 5. CONVERT SPARSE TF-IDF TO DENSE\n# ============================================================\n\n# Limit dimensions for safe memory usage\n\nMAX_TEXT_FEATURES = 1500\n\n\nif X_text_labelled.shape[1] > MAX_TEXT_FEATURES:\n\n # Select highest-variance TF-IDF columns\n feature_variance = np.asarray(\n X_text_labelled.multiply(\n X_text_labelled\n ).mean(axis=0)\n ).ravel()\n\n selected_features = np.argsort(\n feature_variance\n )[\n -MAX_TEXT_FEATURES:\n ]\n\n selected_features = np.sort(\n selected_features\n )\n\n X_text_small = (\n X_text_labelled[\n :,\n selected_features\n ]\n )\n\nelse:\n\n X_text_small = X_text_labelled\n\n\nX_text_dense = (\n X_text_small\n .toarray()\n .astype(np.float32)\n)\n\n\nprint(\n \"\\nReduced text features:\",\n X_text_dense.shape\n)\n\n\n# ============================================================\n# 6. CREATE MULTIMODAL DATAFRAME\n# ============================================================\n\nmultimodal_df = labelled_df[\n [\n \"StudyInstanceUID\",\n \"Report\"\n ]\n].copy()\n\n\nmultimodal_df.reset_index(\n drop=True,\n inplace=True\n)\n\n\nprint(\n \"\\nMultimodal dataframe:\",\n multimodal_df.shape\n)\n\n\n# ============================================================\n# 7. TRAIN / VALIDATION SPLIT\n# ============================================================\n\nindices = np.arange(\n len(multimodal_df)\n)\n\n\ntrain_mm_idx, valid_mm_idx = (\n train_test_split(\n indices,\n test_size=0.20,\n random_state=42\n )\n)\n\n\nprint(\n \"\\nMultimodal training studies:\",\n len(train_mm_idx)\n)\n\nprint(\n \"Multimodal validation studies:\",\n len(valid_mm_idx)\n)\n\n\n# ============================================================\n# 8. PYTORCH TEXT DATASET\n# ============================================================\n\nclass TextOnlyDataset(\n torch.utils.data.Dataset\n):\n\n def __init__(\n self,\n features,\n labels\n ):\n\n self.features = torch.tensor(\n features,\n dtype=torch.float32\n )\n\n self.labels = torch.tensor(\n labels,\n dtype=torch.float32\n )\n\n\n def __len__(self):\n\n return len(\n self.features\n )\n\n\n def __getitem__(\n self,\n idx\n ):\n\n return (\n self.features[idx],\n self.labels[idx]\n )\n\n\ntext_train_dataset = TextOnlyDataset(\n\n X_text_dense[\n train_mm_idx\n ],\n\n Y_labelled[\n train_mm_idx\n ]\n\n)\n\n\ntext_valid_dataset = TextOnlyDataset(\n\n X_text_dense[\n valid_mm_idx\n ],\n\n Y_labelled[\n valid_mm_idx\n ]\n\n)\n\n\ntext_train_loader = (\n torch.utils.data.DataLoader(\n\n text_train_dataset,\n\n batch_size=16,\n\n shuffle=True\n\n )\n)\n\n\ntext_valid_loader = (\n torch.utils.data.DataLoader(\n\n text_valid_dataset,\n\n batch_size=16,\n\n shuffle=False\n\n )\n)\n\n\nprint(\n \"\\nāœ… Text datasets created\"\n)\n\n\n# ============================================================\n# 9. TEXT ENCODER\n# ============================================================\n\nTEXT_DIM = X_text_dense.shape[1]\n\n\nclass TextEncoder(\n nn.Module\n):\n\n def __init__(\n self,\n input_dim,\n hidden_dim=256,\n output_dim=128\n ):\n\n super().__init__()\n\n\n self.network = nn.Sequential(\n\n nn.Linear(\n input_dim,\n hidden_dim\n ),\n\n nn.LayerNorm(\n hidden_dim\n ),\n\n nn.GELU(),\n\n nn.Dropout(\n 0.25\n ),\n\n nn.Linear(\n hidden_dim,\n output_dim\n ),\n\n nn.LayerNorm(\n output_dim\n ),\n\n nn.GELU()\n\n )\n\n\n def forward(\n self,\n x\n ):\n\n return self.network(x)\n\n\ntext_encoder = TextEncoder(\n TEXT_DIM\n).to(DEVICE)\n\n\nprint(\n \"āœ… Text encoder created\"\n)\n\n\n# ============================================================\n# 10. TEXT PREDICTION HEAD\n# ============================================================\n\ntext_head = nn.Sequential(\n\n nn.Linear(\n 128,\n 64\n ),\n\n nn.GELU(),\n\n nn.Dropout(\n 0.2\n ),\n\n nn.Linear(\n 64,\n 12\n )\n\n).to(DEVICE)\n\n\n# ============================================================\n# 11. TEXT MODEL LOSS\n# ============================================================\n\n# Calculate positive weights\n\npos_weights = []\n\n\nfor i in range(12):\n\n y = Y_labelled[:, i]\n\n valid = y >= 0\n\n pos = (\n y[valid] == 1\n ).sum()\n\n neg = (\n y[valid] == 0\n ).sum()\n\n if pos > 0:\n\n weight = neg / pos\n\n else:\n\n weight = 1.0\n\n weight = np.clip(\n weight,\n 1.0,\n 10.0\n )\n\n pos_weights.append(\n weight\n )\n\n\npos_weights_tensor = torch.tensor(\n\n pos_weights,\n\n dtype=torch.float32,\n\n device=DEVICE\n\n)\n\n\ntext_criterion = nn.BCEWithLogitsLoss(\n\n pos_weight=pos_weights_tensor,\n\n reduction=\"none\"\n\n)\n\n\ntext_optimizer = torch.optim.AdamW(\n\n list(\n text_encoder.parameters()\n )\n +\n list(\n text_head.parameters()\n ),\n\n lr=1e-4,\n\n weight_decay=1e-4\n\n)\n\n\n# ============================================================\n# 12. TEXT VALIDATION\n# ============================================================\n\ndef evaluate_text_model():\n\n text_encoder.eval()\n\n text_head.eval()\n\n predictions = []\n\n targets = []\n\n\n with torch.no_grad():\n\n for x, y in text_valid_loader:\n\n x = x.to(\n DEVICE\n )\n\n logits = text_head(\n text_encoder(x)\n )\n\n probs = torch.sigmoid(\n logits\n )\n\n predictions.append(\n probs.cpu().numpy()\n )\n\n targets.append(\n y.numpy()\n )\n\n\n predictions = np.concatenate(\n predictions,\n axis=0\n )\n\n targets = np.concatenate(\n targets,\n axis=0\n )\n\n\n aucs = []\n\n\n for i in range(12):\n\n y_true = targets[:, i]\n\n y_pred = predictions[:, i]\n\n valid = (\n y_true >= 0\n )\n\n\n if (\n\n valid.sum() > 1\n and\n len(\n np.unique(\n y_true[valid]\n )\n ) == 2\n\n ):\n\n auc = roc_auc_score(\n\n y_true[valid],\n\n y_pred[valid]\n\n )\n\n else:\n\n auc = np.nan\n\n\n aucs.append(\n auc\n )\n\n\n macro_auc = np.nanmean(\n aucs\n )\n\n\n return (\n macro_auc,\n aucs\n )\n\n\n# ============================================================\n# 13. TRAIN TEXT ENCODER\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"TRAINING TEXT ENCODER\")\n\nprint(\"=\" * 70)\n\n\nTEXT_EPOCHS = 8\n\nbest_text_auc = -1\n\n\nfor epoch in range(\n TEXT_EPOCHS\n):\n\n text_encoder.train()\n\n text_head.train()\n\n\n running_loss = 0\n\n batches = 0\n\n\n for x, y in text_train_loader:\n\n x = x.to(\n DEVICE\n )\n\n y = y.to(\n DEVICE\n )\n\n\n text_optimizer.zero_grad()\n\n\n logits = text_head(\n\n text_encoder(x)\n\n )\n\n\n valid_mask = (\n y >= 0\n )\n\n\n safe_y = torch.clamp(\n y,\n 0,\n 1\n )\n\n\n loss_matrix = (\n text_criterion(\n logits,\n safe_y\n )\n )\n\n\n loss = (\n loss_matrix[\n valid_mask\n ].mean()\n )\n\n\n if not torch.isfinite(\n loss\n ):\n\n continue\n\n\n loss.backward()\n\n\n torch.nn.utils.clip_grad_norm_(\n\n list(\n text_encoder.parameters()\n )\n +\n list(\n text_head.parameters()\n ),\n\n 1.0\n\n )\n\n\n text_optimizer.step()\n\n\n running_loss += (\n loss.item()\n )\n\n batches += 1\n\n\n val_auc, val_aucs = (\n evaluate_text_model()\n )\n\n\n print(\n\n f\"Epoch {epoch + 1}/\"\n f\"{TEXT_EPOCHS} | \"\n f\"Loss={running_loss/max(batches,1):.5f} | \"\n f\"Macro AUC={val_auc:.5f}\"\n\n )\n\n\n if val_auc > best_text_auc:\n\n best_text_auc = val_auc\n\n\n torch.save(\n\n {\n\n \"encoder\":\n text_encoder.state_dict(),\n\n \"head\":\n text_head.state_dict()\n\n },\n\n \"/kaggle/working/\"\n \"rsna_step13_text_encoder.pth\"\n\n )\n\n\n# ============================================================\n# 14. FINAL TEXT RESULT\n# ============================================================\n\ntext_macro_auc, text_target_auc = (\n evaluate_text_model()\n)\n\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"STEP 13 TEXT BASELINE\")\n\nprint(\"=\" * 70)\n\n\nprint(\n \"Best text Macro AUC:\",\n round(\n best_text_auc,\n 5\n )\n)\n\n\ntext_auc_df = pd.DataFrame({\n\n \"Target\":\n TARGETS_12,\n\n \"AUC\":\n text_target_auc\n\n})\n\n\ndisplay(\n text_auc_df\n)\n\n\ntext_auc_df.to_csv(\n\n \"/kaggle/working/\"\n \"step13_text_auc.csv\",\n\n index=False\n\n)\n\n\n# ============================================================\n# 15. SAVE MULTIMODAL PREPARATION\n# ============================================================\n\nnp.save(\n\n \"/kaggle/working/\"\n \"step13_labelled_indices.npy\",\n\n labelled_indices\n\n)\n\n\nnp.save(\n\n \"/kaggle/working/\"\n \"step13_text_features.npy\",\n\n X_text_dense\n\n)\n\n\n# ============================================================\n# 16. FINAL CHECK\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\n\nprint(\"STEP 13 COMPLETE\")\n\nprint(\"=\" * 70)\n\n\nprint(\n \"āœ… Labelled studies isolated\"\n)\n\nprint(\n \"āœ… TF-IDF features reduced\"\n)\n\nprint(\n \"āœ… Text PyTorch dataset created\"\n)\n\nprint(\n \"āœ… Text encoder trained\"\n)\n\nprint(\n \"āœ… Class imbalance handled\"\n)\n\nprint(\n \"āœ… Per-target text AUC calculated\"\n)\n\nprint(\n \"āœ… Text encoder checkpoint saved\"\n)\n\nprint(\n \"\\nBest Text Macro AUC:\",\n round(\n best_text_auc,\n 5\n )\n)\n\nprint(\n \"\\nSaved:\"\n)\n\nprint(\n \"/kaggle/working/\"\n \"rsna_step13_text_encoder.pth\"\n)\n\nprint(\n \"/kaggle/working/\"\n \"step13_text_auc.csv\"\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 14\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 14 FIX ā€” ALIGN TEXT FEATURES WITH 58 LABELLED STUDIES\n# ============================================================\n\nprint(\"=\" * 70)\nprint(\"STEP 14 FIX ā€” TEXT / LABEL ALIGNMENT\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. CHECK CURRENT DATA\n# ------------------------------------------------------------\n\nprint(\"\\nOriginal text_features shape:\")\nprint(text_features.shape)\n\nprint(\"\\nLabelled studies:\")\nprint(len(labelled_indices))\n\n\n# ------------------------------------------------------------\n# 2. IMPORTANT:\n# text_features already contains ONLY the 58 labelled studies\n# DO NOT index it again with original 4407-row indices.\n# ------------------------------------------------------------\n\nif len(text_features) != len(labelled_indices):\n\n raise ValueError(\n f\"Mismatch: text_features={len(text_features)}, \"\n f\"labelled_indices={len(labelled_indices)}\"\n )\n\nprint(\n \"\\nāœ… Text features already aligned with labelled studies\"\n)\n\n\n# ------------------------------------------------------------\n# 3. CREATE CORRECT TEXT TENSOR\n# ------------------------------------------------------------\n\ntext_tensor_14 = torch.tensor(\n text_features,\n dtype=torch.float32\n)\n\n\n# ------------------------------------------------------------\n# 4. CREATE CORRECT LABEL TENSOR\n# ------------------------------------------------------------\n\nlabels_tensor_14 = torch.tensor(\n label_matrix[labelled_indices],\n dtype=torch.float32\n)\n\n\nprint(\n \"\\nText tensor shape:\",\n text_tensor_14.shape\n)\n\nprint(\n \"Labels tensor shape:\",\n labels_tensor_14.shape\n)\n\n\n# ------------------------------------------------------------\n# 5. CREATE TEXT EMBEDDINGS\n# ------------------------------------------------------------\n\ntext_encoder_14.eval()\n\ntext_embeddings_14 = []\n\nchunk_size = 32\n\n\nwith torch.no_grad():\n\n for start in range(\n 0,\n len(text_tensor_14),\n chunk_size\n ):\n\n batch = (\n text_tensor_14[\n start:start + chunk_size\n ]\n .to(DEVICE)\n )\n\n emb = text_encoder_14(\n batch\n )\n\n text_embeddings_14.append(\n emb.cpu()\n )\n\n\ntext_embeddings_14 = torch.cat(\n text_embeddings_14,\n dim=0\n)\n\n\nprint(\n \"\\nText embeddings shape:\",\n text_embeddings_14.shape\n)\n\n\n# ------------------------------------------------------------\n# 6. VERIFY\n# ------------------------------------------------------------\n\nassert (\n text_embeddings_14.shape[0]\n ==\n labels_tensor_14.shape[0]\n)\n\nassert (\n text_embeddings_14.shape[0]\n ==\n len(labelled_indices)\n)\n\n\nprint(\n \"\\nāœ… Text embeddings and labels perfectly aligned\"\n)\n\n\n# ------------------------------------------------------------\n# 7. SAVE\n# ------------------------------------------------------------\n\ntorch.save(\n\n text_embeddings_14,\n\n \"/kaggle/working/\"\n \"step14_text_embeddings.pt\"\n\n)\n\ntorch.save(\n\n labels_tensor_14,\n\n \"/kaggle/working/\"\n \"step14_labels.pt\"\n\n)\n\n\n# ------------------------------------------------------------\n# 8. FINAL\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 14 FIX COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… 58 labelled studies confirmed\"\n)\n\nprint(\n \"āœ… Text features aligned\"\n)\n\nprint(\n \"āœ… Text embeddings generated\"\n)\n\nprint(\n \"āœ… Labels aligned\"\n)\n\nprint(\n \"\\nText embeddings:\",\n text_embeddings_14.shape\n)\n\nprint(\n \"Labels:\",\n labels_tensor_14.shape\n)\n\nprint(\n \"\\nSaved:\"\n)\n\nprint(\n \"/kaggle/working/step14_text_embeddings.pt\"\n)\n\nprint(\n \"/kaggle/working/step14_labels.pt\"\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 15\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 15 FIX ā€” INSPECT ACTUAL MRI DATASET\n# ============================================================\n\nimport numpy as np\nimport pandas as pd\nimport torch\n\nprint(\"=\" * 70)\nprint(\"STEP 15 FIX ā€” MRI DATASET ALIGNMENT\")\nprint(\"=\" * 70)\n\n\n# ============================================================\n# 1. BASIC DATASET INFORMATION\n# ============================================================\n\nprint(\"\\nDataset type:\")\nprint(type(train_dataset))\n\nprint(\n \"\\nDataset length:\",\n len(train_dataset)\n)\n\n\n# ============================================================\n# 2. INSPECT FIRST SAMPLE\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FIRST DATASET SAMPLE\")\nprint(\"=\" * 70)\n\nsample = train_dataset[0]\n\nprint(\n \"\\nSample Python type:\",\n type(sample)\n)\n\n\nif isinstance(sample, dict):\n\n print(\n \"\\nSample is DICTIONARY\"\n )\n\n print(\n \"Keys:\",\n list(sample.keys())\n )\n\n for key, value in sample.items():\n\n try:\n\n if torch.is_tensor(value):\n\n print(\n f\"{key}: \"\n f\"Tensor {tuple(value.shape)}\"\n )\n\n elif isinstance(\n value,\n np.ndarray\n ):\n\n print(\n f\"{key}: \"\n f\"NumPy {value.shape}\"\n )\n\n else:\n\n print(\n f\"{key}: \"\n f\"{type(value)} -> \"\n f\"{str(value)[:200]}\"\n )\n\n except Exception:\n\n print(\n f\"{key}: \"\n f\"{type(value)}\"\n )\n\n\nelif isinstance(\n sample,\n (tuple, list)\n):\n\n print(\n \"\\nSample is LIST/TUPLE\"\n )\n\n print(\n \"Number of elements:\",\n len(sample)\n )\n\n for i, value in enumerate(\n sample\n ):\n\n try:\n\n if torch.is_tensor(value):\n\n print(\n f\"[{i}] Tensor \"\n f\"{tuple(value.shape)}\"\n )\n\n elif isinstance(\n value,\n np.ndarray\n ):\n\n print(\n f\"[{i}] NumPy \"\n f\"{value.shape}\"\n )\n\n else:\n\n print(\n f\"[{i}] \"\n f\"{type(value)} -> \"\n f\"{str(value)[:200]}\"\n )\n\n except Exception:\n\n print(\n f\"[{i}] \"\n f\"{type(value)}\"\n )\n\n\nelse:\n\n print(\n \"\\nSample value:\"\n )\n\n print(\n str(sample)[:1000]\n )\n\n\n# ============================================================\n# 3. INSPECT DATASET ATTRIBUTES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"DATASET ATTRIBUTES\")\nprint(\"=\" * 70)\n\n\nattributes = [\n \"df\",\n \"data\",\n \"samples\",\n \"study_ids\",\n \"study_uids\",\n \"uids\",\n \"ids\",\n \"indices\",\n \"paths\",\n \"image_paths\",\n \"records\",\n \"metadata\"\n]\n\n\nfor attr in attributes:\n\n if hasattr(\n train_dataset,\n attr\n ):\n\n try:\n\n value = getattr(\n train_dataset,\n attr\n )\n\n print(\n f\"\\nāœ… {attr}\"\n )\n\n print(\n \"Type:\",\n type(value)\n )\n\n if hasattr(\n value,\n \"shape\"\n ):\n\n print(\n \"Shape:\",\n value.shape\n )\n\n elif hasattr(\n value,\n \"__len__\"\n ):\n\n print(\n \"Length:\",\n len(value)\n )\n\n print(\n \"Preview:\",\n str(value)[:500]\n )\n\n except Exception as e:\n\n print(\n f\"āš ļø {attr}: {e}\"\n )\n\n\n# ============================================================\n# 4. INSPECT train_dataset.__dict__\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"INTERNAL DATASET VARIABLES\")\nprint(\"=\" * 70)\n\n\ntry:\n\n dataset_vars = (\n train_dataset.__dict__\n )\n\n for key, value in (\n dataset_vars.items()\n ):\n\n print(\n \"\\nVARIABLE:\",\n key\n )\n\n print(\n \"TYPE:\",\n type(value)\n )\n\n try:\n\n if hasattr(\n value,\n \"shape\"\n ):\n\n print(\n \"SHAPE:\",\n value.shape\n )\n\n elif hasattr(\n value,\n \"__len__\"\n ):\n\n print(\n \"LENGTH:\",\n len(value)\n )\n\n except Exception:\n\n pass\n\n print(\n \"PREVIEW:\",\n str(value)[:300]\n )\n\nexcept Exception as e:\n\n print(\n \"Could not inspect __dict__:\",\n e\n )\n\n\n# ============================================================\n# 5. CHECK IMPORTANT GLOBAL DATAFRAMES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"AVAILABLE DATAFRAMES\")\nprint(\"=\" * 70)\n\n\nfor name in [\n \"train_df\",\n \"train_df_12\",\n \"train_series_df\",\n \"improved_index_df\",\n \"mri_index_df\"\n]:\n\n if name in globals():\n\n obj = globals()[name]\n\n print(\n f\"\\nāœ… {name}\"\n )\n\n print(\n \"Shape:\",\n obj.shape\n )\n\n print(\n \"Columns:\",\n list(\n obj.columns\n )\n )\n\n\n# ============================================================\n# 6. LABELLED STUDY IDS\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABELLED STUDY INFORMATION\")\nprint(\"=\" * 70)\n\n\nlabelled_ids = (\n train_df_12\n .iloc[labelled_indices]\n [\n \"StudyInstanceUID\"\n ]\n .astype(str)\n .tolist()\n)\n\n\nprint(\n \"Number of labelled UIDs:\",\n len(labelled_ids)\n)\n\n\nprint(\n \"\\nFirst 5 labelled UIDs:\"\n)\n\n\nfor uid in labelled_ids[:5]:\n\n print(\n uid\n )\n\n\n# ============================================================\n# 7. CHECK WHETHER DATASET HAS UID MAPPING\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"SEARCHING FOR UID MAPPING\")\nprint(\"=\" * 70)\n\n\nfound_uid_mapping = False\n\n\nfor attr in dir(\n train_dataset\n):\n\n if (\n \"uid\" in attr.lower()\n or\n \"study\" in attr.lower()\n or\n \"id\" in attr.lower()\n or\n \"path\" in attr.lower()\n ):\n\n try:\n\n value = getattr(\n train_dataset,\n attr\n )\n\n if callable(value):\n\n continue\n\n print(\n \"\\nPossible attribute:\",\n attr\n )\n\n print(\n \"Type:\",\n type(value)\n )\n\n print(\n \"Preview:\",\n str(value)[:500]\n )\n\n found_uid_mapping = True\n\n except Exception:\n\n pass\n\n\n# ============================================================\n# 8. FINAL DIAGNOSTIC\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 15 FIX DIAGNOSTIC COMPLETE\")\nprint(\"=\" * 70)\n\n\nif found_uid_mapping:\n\n print(\n \"āœ… Possible UID/path mapping found.\"\n )\n\nelse:\n\n print(\n \"āš ļø No obvious UID mapping found.\"\n )\n\n\nprint(\n \"\"\"\nIMPORTANT:\nDo NOT train STEP 15 yet.\n\nWe need to correctly connect:\nStudyInstanceUID\n ā†“\nMRI DICOM series\n ā†“\nMRI tensor\n ā†“\n512-D MRI feature\n ā†“\ntext embedding\n ā†“\n12 labels\n\"\"\"\n)\n\nprint(\n \"\\nšŸš€ SEND THIS OUTPUT BEFORE STEP 15 CONTINUES\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 15 FIX v2 ā€” MRI + TEXT FEATURE ALIGNMENT\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nfrom torchvision import models\n\nprint(\"=\" * 70)\nprint(\"STEP 15 FIX v2 ā€” MRI + TEXT FEATURE EXTRACTION\")\nprint(\"=\" * 70)\n\n# ============================================================\n# 1. DEVICE\n# ============================================================\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"\\nDevice:\", device)\n\n\n# ============================================================\n# 2. FIND DATASET\n# ============================================================\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(DATA_PATH, \"train.csv\")\n\nif not os.path.exists(TRAIN_CSV):\n raise FileNotFoundError(\n f\"train.csv not found at:\\n{TRAIN_CSV}\"\n )\n\nprint(\"āœ… Dataset found\")\nprint(\"Dataset:\", DATA_PATH)\n\n\n# ============================================================\n# 3. LOAD TRAIN CSV\n# ============================================================\n\ntrain_df = pd.read_csv(TRAIN_CSV)\n\nprint(\"\\nTrain shape:\", train_df.shape)\n\nprint(\"Columns:\")\nprint(train_df.columns.tolist())\n\n\n# ============================================================\n# 4. TARGET COLUMNS\n# ============================================================\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n\n# ============================================================\n# 5. FIND LABELLED STUDIES\n# ============================================================\n\n# Competition has many unlabeled rows.\n# Keep only rows where all 12 labels are available.\n\nlabelled_mask = train_df[target_cols].notna().all(axis=1)\n\nlabelled_df = train_df.loc[\n labelled_mask\n].copy().reset_index(drop=True)\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LABELLED STUDIES\")\nprint(\"=\" * 70)\n\nprint(\n \"Total training studies:\",\n len(train_df)\n)\n\nprint(\n \"Labelled studies:\",\n len(labelled_df)\n)\n\n\n# ============================================================\n# 6. SAFETY CHECK\n# ============================================================\n\nif len(labelled_df) == 0:\n raise RuntimeError(\n \"No fully labelled studies found.\"\n )\n\nprint(\"āœ… Labelled dataframe created\")\n\n\n# ============================================================\n# 7. TARGET MATRIX\n# ============================================================\n\ntargets = labelled_df[\n target_cols\n].astype(np.float32).values\n\nprint(\n \"\\nLabels shape:\",\n targets.shape\n)\n\n\n# ============================================================\n# 8. TEXT FEATURES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TEXT FEATURES\")\nprint(\"=\" * 70)\n\nif \"text_tensor_14\" in globals():\n\n text_data = text_tensor_14.detach().cpu().numpy()\n\n print(\n \"Using existing text_tensor_14\"\n )\n\nelif \"text_features\" in globals():\n\n if torch.is_tensor(text_features):\n text_data = (\n text_features\n .detach()\n .cpu()\n .numpy()\n )\n else:\n text_data = np.asarray(text_features)\n\n print(\n \"Using existing text_features\"\n )\n\nelse:\n\n raise RuntimeError(\n \"\"\"\nText features are not available.\n\nPlease run STEP 14 first so that\ntext_tensor_14 is created.\n\"\"\"\n )\n\n\nprint(\n \"Text feature shape:\",\n text_data.shape\n)\n\n\n# ============================================================\n# 9. TEXT COUNT CHECK\n# ============================================================\n\nif len(text_data) != len(labelled_df):\n\n print(\"\\nāš ļø Text count mismatch\")\n print(\n \"Text:\",\n len(text_data)\n )\n\n print(\n \"Labelled:\",\n len(labelled_df)\n )\n\n raise RuntimeError(\n \"Text features do not match labelled studies.\"\n )\n\nprint(\n \"āœ… Text features aligned with labels\"\n)\n\n\n# ============================================================\n# 10. MRI DATASET\n# ============================================================\n\nif \"train_dataset\" not in globals():\n\n raise RuntimeError(\n \"\"\"\ntrain_dataset does not exist.\n\nPlease run the MRI dataset preparation\nsteps before STEP 15.\n\"\"\"\n )\n\nmri_dataset = train_dataset\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MRI DATASET\")\nprint(\"=\" * 70)\n\nprint(\n \"Dataset type:\",\n type(mri_dataset)\n)\n\nprint(\n \"Dataset length:\",\n len(mri_dataset)\n)\n\n\n# ============================================================\n# 11. MRI DATASET INDICES\n# ============================================================\n\nif \"indices\" in globals():\n\n original_indices = np.asarray(\n indices,\n dtype=int\n )\n\n print(\n \"Original indices length:\",\n len(original_indices)\n )\n\nelse:\n\n # Fallback\n original_indices = np.arange(\n len(mri_dataset)\n )\n\n print(\n \"āš ļø indices variable not found.\"\n )\n\n print(\n \"Using sequential indices.\"\n )\n\n\n# ============================================================\n# 12. FIX INDEX MAPPING\n# ============================================================\n\nvalid_count = min(\n len(mri_dataset),\n len(original_indices)\n)\n\ndataset_to_label_position = {\n dataset_idx: int(\n original_indices[dataset_idx]\n )\n for dataset_idx in range(valid_count)\n if 0 <= int(original_indices[dataset_idx])\n < len(labelled_df)\n}\n\nprint(\n \"\\nValid MRI ā†’ label mappings:\",\n len(dataset_to_label_position)\n)\n\n\n# ============================================================\n# 13. RESNET18\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"RESNET18 FEATURE EXTRACTOR\")\nprint(\"=\" * 70)\n\nresnet = models.resnet18(\n weights=None\n)\n\nmodel_path = (\n \"/kaggle/working/\"\n \"rsna_resnet18_best.pth\"\n)\n\nif os.path.exists(model_path):\n\n print(\n \"Loading:\",\n model_path\n )\n\n checkpoint = torch.load(\n model_path,\n map_location=device\n )\n\n # --------------------------------------------\n # Extract state dict\n # --------------------------------------------\n\n if isinstance(\n checkpoint,\n dict\n ):\n\n if \"model_state_dict\" in checkpoint:\n\n state_dict = (\n checkpoint[\"model_state_dict\"]\n )\n\n elif \"state_dict\" in checkpoint:\n\n state_dict = (\n checkpoint[\"state_dict\"]\n )\n\n else:\n\n state_dict = checkpoint\n\n else:\n\n state_dict = checkpoint\n\n\n # --------------------------------------------\n # Remove module. prefix\n # --------------------------------------------\n\n clean_state_dict = {}\n\n for key, value in state_dict.items():\n\n new_key = key\n\n if new_key.startswith(\n \"module.\"\n ):\n\n new_key = new_key[\n len(\"module.\") :\n ]\n\n clean_state_dict[\n new_key\n ] = value\n\n\n # --------------------------------------------\n # Load compatible weights\n # --------------------------------------------\n\n missing, unexpected = (\n resnet.load_state_dict(\n clean_state_dict,\n strict=False\n )\n )\n\n print(\n \"Missing keys:\",\n len(missing)\n )\n\n print(\n \"Unexpected keys:\",\n len(unexpected)\n )\n\n print(\n \"āœ… ResNet weights loaded\"\n )\n\nelse:\n\n print(\n \"āš ļø Previous ResNet weights not found.\"\n )\n\n print(\n \"Using randomly initialized ResNet.\"\n )\n\n\n# ============================================================\n# 14. REMOVE CLASSIFIER\n# ============================================================\n\nmri_encoder = nn.Sequential(\n *list(\n resnet.children()\n )[:-1]\n)\n\nmri_encoder = mri_encoder.to(\n device\n)\n\nmri_encoder.eval()\n\nprint(\n \"āœ… MRI feature extractor ready\"\n)\n\nprint(\n \"Feature dimension: 512\"\n)\n\n\n# ============================================================\n# 15. FEATURE EXTRACTION FUNCTION\n# ============================================================\n\ndef extract_mri_feature(sample):\n\n # Dataset returns:\n #\n # sample[0] = (12, 3, 224, 224)\n # sample[1] = labels\n\n if isinstance(\n sample,\n (tuple, list)\n ):\n\n images = sample[0]\n\n else:\n\n images = sample\n\n\n if not torch.is_tensor(images):\n\n images = torch.tensor(\n images,\n dtype=torch.float32\n )\n\n\n images = images.float()\n\n\n if images.ndim != 4:\n\n raise ValueError(\n \"Unexpected image shape: \"\n + str(images.shape)\n )\n\n\n # (12, 3, 224, 224)\n images = images.to(device)\n\n\n with torch.no_grad():\n\n # (12, 512, 1, 1)\n features = mri_encoder(\n images\n )\n\n # (12, 512)\n features = features.flatten(\n start_dim=1\n )\n\n # Study-level feature\n # (512,)\n study_feature = features.mean(\n dim=0\n )\n\n\n return (\n study_feature\n .detach()\n .cpu()\n .numpy()\n )\n\n\n# ============================================================\n# 16. EXTRACT MRI FEATURES\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"EXTRACTING MRI FEATURES\")\nprint(\"=\" * 70)\n\nmri_features_list = []\nvalid_dataset_positions = []\nvalid_label_positions = []\n\nfor dataset_idx in range(\n len(mri_dataset)\n):\n\n # Make sure mapping exists\n if dataset_idx not in (\n dataset_to_label_position\n ):\n continue\n\n label_position = (\n dataset_to_label_position[\n dataset_idx\n ]\n )\n\n try:\n\n sample = mri_dataset[\n dataset_idx\n ]\n\n feature = extract_mri_feature(\n sample\n )\n\n if feature.shape != (512,):\n\n print(\n f\"\\nāš ļø Bad feature shape \"\n f\"at {dataset_idx}: \"\n f\"{feature.shape}\"\n )\n\n continue\n\n\n mri_features_list.append(\n feature\n )\n\n valid_dataset_positions.append(\n dataset_idx\n )\n\n valid_label_positions.append(\n label_position\n )\n\n print(\n f\"āœ… {len(mri_features_list):02d}/\"\n f\"{valid_count}\",\n end=\"\\r\"\n )\n\n except Exception as e:\n\n print(\n f\"\\nāš ļø Dataset index \"\n f\"{dataset_idx} failed:\"\n )\n\n print(\n str(e)\n )\n\n\nprint()\n\n\n# ============================================================\n# 17. CHECK\n# ============================================================\n\nif len(mri_features_list) == 0:\n\n raise RuntimeError(\n \"\"\"\nāŒ No MRI features extracted.\n\nCheck MRI dataset and DICOM paths.\n\"\"\"\n )\n\n\nmri_features = np.stack(\n mri_features_list\n)\n\nvalid_label_positions = np.asarray(\n valid_label_positions,\n dtype=int\n)\n\nprint(\n \"\\nMRI features:\",\n mri_features.shape\n)\n\nprint(\n \"Valid label positions:\",\n valid_label_positions.shape\n)\n\n\n# ============================================================\n# 18. ALIGN TEXT + LABELS\n# ============================================================\n\nmri_features_aligned = (\n mri_features\n)\n\ntext_features_aligned = (\n text_data[\n valid_label_positions\n ]\n)\n\nlabels_aligned = (\n targets[\n valid_label_positions\n ]\n)\n\n\n# ============================================================\n# 19. FINAL SHAPE CHECK\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FINAL MULTIMODAL ALIGNMENT\")\nprint(\"=\" * 70)\n\nprint(\n \"MRI :\",\n mri_features_aligned.shape\n)\n\nprint(\n \"TEXT :\",\n text_features_aligned.shape\n)\n\nprint(\n \"LABEL :\",\n labels_aligned.shape\n)\n\n\nn = len(\n mri_features_aligned\n)\n\nif not (\n n == len(text_features_aligned)\n and\n n == len(labels_aligned)\n):\n\n raise RuntimeError(\n \"āŒ MRI/Text/Label alignment failed.\"\n )\n\n\nprint(\n f\"\\nāœ… Successfully aligned {n} studies\"\n)\n\n\n# ============================================================\n# 20. CREATE TENSORS\n# ============================================================\n\nmri_tensor_15 = torch.tensor(\n mri_features_aligned,\n dtype=torch.float32\n)\n\ntext_tensor_15 = torch.tensor(\n text_features_aligned,\n dtype=torch.float32\n)\n\nlabels_tensor_15 = torch.tensor(\n labels_aligned,\n dtype=torch.float32\n)\n\n\nprint(\"\\nFinal tensors:\")\n\nprint(\n \"MRI :\",\n tuple(\n mri_tensor_15.shape\n )\n)\n\nprint(\n \"TEXT :\",\n tuple(\n text_tensor_15.shape\n )\n)\n\nprint(\n \"LABEL :\",\n tuple(\n labels_tensor_15.shape\n )\n)\n\n\n# ============================================================\n# 21. SAVE\n# ============================================================\n\nnp.save(\n \"/kaggle/working/step15_mri_features.npy\",\n mri_features_aligned\n)\n\nnp.save(\n \"/kaggle/working/step15_text_features.npy\",\n text_features_aligned\n)\n\nnp.save(\n \"/kaggle/working/step15_labels.npy\",\n labels_aligned\n)\n\n\n# ============================================================\n# COMPLETE\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 15 FEATURE EXTRACTION COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… train.csv loaded inside Step 15\"\n)\n\nprint(\n \"āœ… Labelled studies detected\"\n)\n\nprint(\n \"āœ… MRI features extracted\"\n)\n\nprint(\n \"āœ… Text features aligned\"\n)\n\nprint(\n \"āœ… 12 labels aligned\"\n)\n\nprint(\n \"āœ… Features saved\"\n)\n\nprint(\"\\nšŸš€ DO NOT TRAIN YET.\")\nprint(\n \"Send me the output of this cell.\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 15 ā€” FIX MRI UID MAPPING\n# DO NOT TRAIN YET\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\n\nprint(\"=\" * 70)\nprint(\"STEP 15 ā€” FIXING MRI UID MAPPING\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. DATASET PATH\n# ------------------------------------------------------------\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(DATA_PATH, \"train.csv\")\nTRAIN_SERIES_CSV = os.path.join(DATA_PATH, \"train_series.csv\")\n\nINDEX_PATH = \"/kaggle/working/improved_mri_series_index.csv\"\n\nprint(\"\\n[1] Loading files...\")\n\ntrain_df = pd.read_csv(TRAIN_CSV)\ntrain_series_df = pd.read_csv(TRAIN_SERIES_CSV)\n\nprint(\"train_df:\", train_df.shape)\nprint(\"train_series_df:\", train_series_df.shape)\n\n# ------------------------------------------------------------\n# 2. TARGET COLUMNS\n# ------------------------------------------------------------\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n# ------------------------------------------------------------\n# 3. GET LABELLED STUDIES\n# ------------------------------------------------------------\n\nlabelled_mask = train_df[target_cols].notna().any(axis=1)\n\nlabelled_df = train_df.loc[\n labelled_mask\n].copy().reset_index(drop=True)\n\nprint(\"\\n[2] Labelled studies:\", len(labelled_df))\n\nlabelled_uids = set(\n labelled_df[\"StudyInstanceUID\"].astype(str)\n)\n\nprint(\"Unique labelled UIDs:\", len(labelled_uids))\n\n# ------------------------------------------------------------\n# 4. LOAD IMPROVED MRI INDEX\n# ------------------------------------------------------------\n\nprint(\"\\n[3] Loading improved MRI index...\")\n\nif not os.path.exists(INDEX_PATH):\n raise FileNotFoundError(\n f\"Missing file:\\n{INDEX_PATH}\\n\\n\"\n \"Run STEP 9 first.\"\n )\n\nimproved_index_df = pd.read_csv(INDEX_PATH)\n\nprint(\n \"Improved MRI index:\",\n improved_index_df.shape\n)\n\nprint(\n \"Columns:\",\n improved_index_df.columns.tolist()\n)\n\n# ------------------------------------------------------------\n# 5. NORMALIZE UID COLUMNS\n# ------------------------------------------------------------\n\nimproved_index_df[\"StudyInstanceUID\"] = (\n improved_index_df[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\nlabelled_df[\"StudyInstanceUID\"] = (\n labelled_df[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\n# ------------------------------------------------------------\n# 6. ONLY KEEP LABELLED MRI STUDIES\n# ------------------------------------------------------------\n\nlabelled_mri_index = improved_index_df[\n improved_index_df[\"StudyInstanceUID\"].isin(labelled_uids)\n].copy()\n\nprint(\n \"\\n[4] Labelled studies represented in MRI index:\",\n labelled_mri_index[\"StudyInstanceUID\"].nunique()\n)\n\nprint(\n \"MRI index rows:\",\n len(labelled_mri_index)\n)\n\n# ------------------------------------------------------------\n# 7. BUILD UNIQUE STUDY ā†’ MRI PATH MAPPING\n# ------------------------------------------------------------\n\nrequired_cols = [\n \"StudyInstanceUID\",\n \"DICOM_Path\"\n]\n\nmissing_cols = [\n c for c in required_cols\n if c not in labelled_mri_index.columns\n]\n\nif missing_cols:\n raise RuntimeError(\n f\"Missing required columns: {missing_cols}\"\n )\n\nstudy_to_paths = {}\n\nfor uid, group in labelled_mri_index.groupby(\n \"StudyInstanceUID\"\n):\n\n paths = []\n\n for p in group[\"DICOM_Path\"].dropna().astype(str):\n\n if os.path.exists(p):\n paths.append(p)\n\n if len(paths) > 0:\n study_to_paths[uid] = paths\n\nprint(\n \"\\n[5] Studies with valid DICOM paths:\",\n len(study_to_paths)\n)\n\n# ------------------------------------------------------------\n# 8. FIND THE 46 DATASET STUDIES\n# ------------------------------------------------------------\n\nprint(\"\\n[6] Inspecting MultiSeriesKneeDataset...\")\n\nif \"train_dataset\" not in globals():\n raise RuntimeError(\n \"train_dataset does not exist.\"\n )\n\nprint(\n \"Dataset type:\",\n type(train_dataset)\n)\n\nprint(\n \"Dataset length:\",\n len(train_dataset)\n)\n\n# ------------------------------------------------------------\n# 9. TRY TO IDENTIFY UID FROM DATASET ATTRIBUTES\n# ------------------------------------------------------------\n\npossible_uid_attrs = [\n \"uids\",\n \"UIDs\",\n \"study_uids\",\n \"study_ids\",\n \"study_ids\",\n \"study_to_images\",\n \"image_paths\",\n \"paths\",\n \"samples\",\n \"data\",\n \"df\"\n]\n\nfound_uid_mapping = {}\n\nfor attr in possible_uid_attrs:\n\n if not hasattr(train_dataset, attr):\n continue\n\n obj = getattr(train_dataset, attr)\n\n print(\n f\"\\nFound dataset attribute: {attr}\"\n )\n\n print(\n \"Type:\",\n type(obj)\n )\n\n # Dictionary case\n if isinstance(obj, dict):\n\n keys = list(obj.keys())\n\n candidate_keys = [\n str(k)\n for k in keys\n if str(k) in labelled_uids\n ]\n\n if candidate_keys:\n\n for uid in candidate_keys:\n found_uid_mapping[uid] = obj[uid]\n\n print(\n \"Matched labelled UIDs:\",\n len(candidate_keys)\n )\n\n # DataFrame case\n elif isinstance(obj, pd.DataFrame):\n\n if \"StudyInstanceUID\" in obj.columns:\n\n temp = obj.copy()\n\n temp[\"StudyInstanceUID\"] = (\n temp[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n )\n\n matched = temp[\n temp[\"StudyInstanceUID\"].isin(\n labelled_uids\n )\n ]\n\n for uid in matched[\n \"StudyInstanceUID\"\n ].unique():\n\n found_uid_mapping[uid] = uid\n\n print(\n \"Matched labelled UIDs:\",\n len(matched)\n )\n\n# ------------------------------------------------------------\n# 10. DO NOT GUESS\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"UID MAPPING RESULT\")\nprint(\"=\" * 70)\n\nprint(\n \"Labelled studies:\",\n len(labelled_uids)\n)\n\nprint(\n \"Studies with DICOM paths:\",\n len(study_to_paths)\n)\n\nprint(\n \"Dataset UID mappings found:\",\n len(found_uid_mapping)\n)\n\n# ------------------------------------------------------------\n# 11. CREATE CLEAN STUDY TABLE\n# ------------------------------------------------------------\n\nmapping_rows = []\n\nfor uid in sorted(study_to_paths.keys()):\n\n mapping_rows.append({\n \"StudyInstanceUID\": uid,\n \"DICOM_Count\": len(study_to_paths[uid]),\n \"DICOM_Paths\": study_to_paths[uid]\n })\n\nuid_mapping_df = pd.DataFrame(mapping_rows)\n\nprint(\n \"\\nClean MRI study mapping:\",\n uid_mapping_df.shape\n)\n\nif len(uid_mapping_df) > 0:\n\n print(\n uid_mapping_df[\n [\"StudyInstanceUID\", \"DICOM_Count\"]\n ].head(10)\n )\n\n# ------------------------------------------------------------\n# 12. SAVE MAPPING\n# ------------------------------------------------------------\n\nUID_MAP_PATH = (\n \"/kaggle/working/\"\n \"step15_mri_uid_mapping.csv\"\n)\n\nuid_mapping_save = uid_mapping_df.copy()\n\nif \"DICOM_Paths\" in uid_mapping_save.columns:\n\n uid_mapping_save[\"DICOM_Paths\"] = (\n uid_mapping_save[\"DICOM_Paths\"]\n .apply(lambda x: \"||\".join(x))\n )\n\nuid_mapping_save.to_csv(\n UID_MAP_PATH,\n index=False\n)\n\nprint(\n \"\\nāœ… Mapping saved:\",\n UID_MAP_PATH\n)\n\n# ------------------------------------------------------------\n# 13. FINAL STATUS\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 15 UID MAPPING COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"āœ… Labelled studies identified\"\n)\n\nprint(\n \"āœ… Improved MRI index loaded\"\n)\n\nprint(\n \"āœ… StudyInstanceUID normalized\"\n)\n\nprint(\n \"āœ… MRI DICOM paths connected\"\n)\n\nprint(\n \"āœ… UID mapping saved\"\n)\n\nprint(\n \"\\nāš ļø DO NOT TRAIN THE MULTIMODAL MODEL YET.\"\n)\n\nprint(\n \"\\nNext we will explicitly connect:\"\n)\n\nprint(\n \"StudyInstanceUID\"\n)\n\nprint(\n \" ā†“\"\n)\n\nprint(\n \"DICOM paths\"\n)\n\nprint(\n \" ā†“\"\n)\n\nprint(\n \"MRI tensor\"\n)\n\nprint(\n \" ā†“\"\n)\n\nprint(\n \"512-D MRI feature\"\n)\n\nprint(\n \" ā†“\"\n)\n\nprint(\n \"1500-D text feature\"\n)\n\nprint(\n \" ā†“\"\n)\n\nprint(\n \"12 labels\"\n)\n\nprint(\"\\nšŸš€ SEND ME THIS OUTPUT BEFORE TRAINING.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 16 ā€” EXACT MRI + TEXT + LABEL ALIGNMENT\n# ============================================================\n\nimport os\nimport ast\nimport numpy as np\nimport pandas as pd\nimport torch\n\nprint(\"=\" * 70)\nprint(\"STEP 16 ā€” EXACT MULTIMODAL ALIGNMENT\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. PATHS\n# ------------------------------------------------------------\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTRAIN_CSV = os.path.join(\n DATA_PATH,\n \"train.csv\"\n)\n\nUID_MAP_PATH = (\n \"/kaggle/working/\"\n \"step15_mri_uid_mapping.csv\"\n)\n\n# ------------------------------------------------------------\n# 2. LOAD TRAIN\n# ------------------------------------------------------------\n\ntrain_df = pd.read_csv(TRAIN_CSV)\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n# ------------------------------------------------------------\n# 3. LABELLED DATA\n# ------------------------------------------------------------\n\nlabelled_mask = train_df[\n target_cols\n].notna().any(axis=1)\n\nlabelled_df = (\n train_df.loc[labelled_mask]\n .copy()\n .reset_index(drop=True)\n)\n\nlabelled_df[\"StudyInstanceUID\"] = (\n labelled_df[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\nprint(\"\\n[1] Labelled dataframe\")\nprint(\"Shape:\", labelled_df.shape)\nprint(\n \"Unique UIDs:\",\n labelled_df[\"StudyInstanceUID\"].nunique()\n)\n\n# ------------------------------------------------------------\n# 4. LOAD MRI UID MAPPING\n# ------------------------------------------------------------\n\nif not os.path.exists(UID_MAP_PATH):\n\n raise FileNotFoundError(\n f\"Missing:\\n{UID_MAP_PATH}\"\n )\n\nuid_map = pd.read_csv(\n UID_MAP_PATH\n)\n\nuid_map[\"StudyInstanceUID\"] = (\n uid_map[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\nprint(\"\\n[2] MRI UID mapping\")\nprint(\"Shape:\", uid_map.shape)\nprint(\n \"Unique UIDs:\",\n uid_map[\"StudyInstanceUID\"].nunique()\n)\n\n# ------------------------------------------------------------\n# 5. MERGE LABELS + MRI USING UID\n# ------------------------------------------------------------\n\naligned_df = labelled_df[\n [\n \"StudyInstanceUID\",\n \"Report\"\n ] + target_cols\n].merge(\n uid_map,\n on=\"StudyInstanceUID\",\n how=\"inner\",\n validate=\"one_to_one\"\n)\n\nprint(\"\\n[3] UID MERGE RESULT\")\nprint(\n \"Aligned studies:\",\n len(aligned_df)\n)\n\nprint(\n \"Unique aligned UIDs:\",\n aligned_df[\"StudyInstanceUID\"].nunique()\n)\n\n# ------------------------------------------------------------\n# 6. CHECK ALIGNMENT\n# ------------------------------------------------------------\n\nif len(aligned_df) != 58:\n\n print(\n \"\\nāš ļø WARNING:\"\n )\n\n print(\n \"Expected 58 aligned studies.\"\n )\n\n print(\n \"Actual:\",\n len(aligned_df)\n )\n\n missing_uids = sorted(\n set(labelled_df[\"StudyInstanceUID\"])\n -\n set(aligned_df[\"StudyInstanceUID\"])\n )\n\n print(\n \"Missing UIDs:\",\n len(missing_uids)\n )\n\n print(\n missing_uids[:10]\n )\n\nelse:\n\n print(\n \"āœ… ALL 58 LABELLED STUDIES ALIGNED\"\n )\n\n# ------------------------------------------------------------\n# 7. PARSE DICOM PATHS\n# ------------------------------------------------------------\n\ndef parse_paths(x):\n\n if pd.isna(x):\n return []\n\n x = str(x)\n\n # Our saved format\n if \"||\" in x:\n return [\n p for p in x.split(\"||\")\n if p\n ]\n\n # Possible Python-list format\n if x.startswith(\"[\"):\n\n try:\n result = ast.literal_eval(x)\n\n if isinstance(result, list):\n return [\n str(p)\n for p in result\n if os.path.exists(str(p))\n ]\n\n except Exception:\n pass\n\n # Single path\n if os.path.exists(x):\n return [x]\n\n return []\n\n\naligned_df[\"DICOM_Paths\"] = (\n aligned_df[\"DICOM_Paths\"]\n .apply(parse_paths)\n)\n\n# ------------------------------------------------------------\n# 8. VALID DICOM CHECK\n# ------------------------------------------------------------\n\naligned_df[\"Valid_DICOM_Count\"] = (\n aligned_df[\"DICOM_Paths\"]\n .apply(len)\n)\n\nprint(\"\\n[4] DICOM CHECK\")\n\nprint(\n \"Studies with DICOM:\",\n (\n aligned_df[\"Valid_DICOM_Count\"] > 0\n ).sum()\n)\n\nprint(\n \"Total DICOM files:\",\n aligned_df[\"Valid_DICOM_Count\"].sum()\n)\n\nprint(\n \"Minimum DICOM/study:\",\n aligned_df[\"Valid_DICOM_Count\"].min()\n)\n\nprint(\n \"Maximum DICOM/study:\",\n aligned_df[\"Valid_DICOM_Count\"].max()\n)\n\n# ------------------------------------------------------------\n# 9. TEXT CHECK\n# ------------------------------------------------------------\n\naligned_df[\"Report\"] = (\n aligned_df[\"Report\"]\n .fillna(\"\")\n .astype(str)\n)\n\nprint(\"\\n[5] TEXT CHECK\")\n\nprint(\n \"Reports:\",\n len(aligned_df)\n)\n\nprint(\n \"Empty reports:\",\n (\n aligned_df[\"Report\"].str.strip()\n == \"\"\n ).sum()\n)\n\n# ------------------------------------------------------------\n# 10. LABEL MATRIX\n# ------------------------------------------------------------\n\nlabels = aligned_df[\n target_cols\n].astype(float).values\n\nprint(\"\\n[6] LABEL MATRIX\")\n\nprint(\n \"Shape:\",\n labels.shape\n)\n\nprint(\n \"Expected:\",\n \"(58, 12)\"\n)\n\n# ------------------------------------------------------------\n# 11. LABEL AVAILABILITY\n# ------------------------------------------------------------\n\nprint(\n \"\\nMissing values per target:\"\n)\n\nprint(\n aligned_df[target_cols]\n .isna()\n .sum()\n)\n\n# ------------------------------------------------------------\n# 12. TEXT FEATURE CHECK\n# ------------------------------------------------------------\n\ntext_source = None\n\nif \"text_tensor_14\" in globals():\n\n text_source = text_tensor_14\n\n print(\n \"\\n[7] Found text_tensor_14\"\n )\n\nelif \"text_features\" in globals():\n\n text_source = text_features\n\n print(\n \"\\n[7] Found text_features\"\n )\n\nelse:\n\n print(\n \"\\nāš ļø No existing text features found.\"\n )\n\n# ------------------------------------------------------------\n# 13. TEXT FEATURE DIMENSION\n# ------------------------------------------------------------\n\nif text_source is not None:\n\n if torch.is_tensor(text_source):\n\n text_np = (\n text_source\n .detach()\n .cpu()\n .numpy()\n )\n\n else:\n\n text_np = np.asarray(\n text_source\n )\n\n print(\n \"Text feature shape:\",\n text_np.shape\n )\n\n if text_np.shape[0] != len(aligned_df):\n\n print(\n \"\\nāš ļø Text feature count does not\"\n \" equal aligned study count.\"\n )\n\n print(\n \"Text rows:\",\n text_np.shape[0]\n )\n\n print(\n \"Aligned studies:\",\n len(aligned_df)\n )\n\n else:\n\n print(\n \"āœ… Text feature count matches\"\n )\n\n# ------------------------------------------------------------\n# 14. CREATE UID ā†’ POSITION MAPPING\n# ------------------------------------------------------------\n\nuid_to_position = {\n uid: i\n for i, uid in enumerate(\n aligned_df[\"StudyInstanceUID\"]\n )\n}\n\nprint(\n \"\\n[8] UID mapping created\"\n)\n\nprint(\n \"Mapping size:\",\n len(uid_to_position)\n)\n\n# ------------------------------------------------------------\n# 15. SAVE FINAL ALIGNMENT TABLE\n# ------------------------------------------------------------\n\nALIGN_PATH = (\n \"/kaggle/working/\"\n \"step16_final_multimodal_alignment.csv\"\n)\n\nsave_df = aligned_df[\n [\n \"StudyInstanceUID\",\n \"Valid_DICOM_Count\"\n ] + target_cols\n].copy()\n\nsave_df.to_csv(\n ALIGN_PATH,\n index=False\n)\n\nprint(\n \"\\nSaved:\",\n ALIGN_PATH\n)\n\n# ------------------------------------------------------------\n# 16. FINAL SUMMARY\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 16 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"StudyInstanceUID ā†’ MRI:\",\n len(aligned_df)\n)\n\nprint(\n \"StudyInstanceUID ā†’ Text:\",\n len(aligned_df)\n)\n\nprint(\n \"StudyInstanceUID ā†’ Labels:\",\n len(aligned_df)\n)\n\nprint(\n \"Labels:\",\n labels.shape\n)\n\nif text_source is not None:\n print(\n \"Text:\",\n text_np.shape\n )\n\nprint(\n \"MRI DICOM paths:\",\n aligned_df[\"Valid_DICOM_Count\"].sum()\n)\n\nprint(\n \"\\nšŸš« DO NOT TRAIN YET.\"\n)\n\nprint(\n \"Send me the complete output of this cell.\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 17 ā€” UID-BASED MRI FEATURE EXTRACTION\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pydicom\n\nfrom PIL import Image\nfrom torchvision import models, transforms\n\nprint(\"=\" * 70)\nprint(\"STEP 17 ā€” UID-BASED MRI FEATURE EXTRACTION\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. DEVICE\n# ------------------------------------------------------------\n\ndevice = torch.device(\n \"cuda\" if torch.cuda.is_available() else \"cpu\"\n)\n\nprint(\"\\nDevice:\", device)\n\n# ------------------------------------------------------------\n# 2. CHECK STEP 16 OBJECTS\n# ------------------------------------------------------------\n\nif \"aligned_df\" not in globals():\n raise RuntimeError(\n \"aligned_df not found. Run STEP 16 first.\"\n )\n\nif \"target_cols\" not in globals():\n target_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n ]\n\nprint(\n \"\\nAligned studies:\",\n len(aligned_df)\n)\n\n# ------------------------------------------------------------\n# 3. CHECK REQUIRED COLUMNS\n# ------------------------------------------------------------\n\nrequired_cols = [\n \"StudyInstanceUID\",\n \"DICOM_Paths\"\n]\n\nfor col in required_cols:\n\n if col not in aligned_df.columns:\n\n raise RuntimeError(\n f\"Missing column: {col}\"\n )\n\n# ------------------------------------------------------------\n# 4. BUILD UID ā†’ DICOM PATHS\n# ------------------------------------------------------------\n\nstudy_to_dicom = {}\n\nfor _, row in aligned_df.iterrows():\n\n uid = str(\n row[\"StudyInstanceUID\"]\n ).strip()\n\n paths = row[\"DICOM_Paths\"]\n\n if not isinstance(paths, list):\n\n paths = list(paths)\n\n valid_paths = [\n str(p)\n for p in paths\n if os.path.exists(str(p))\n ]\n\n study_to_dicom[uid] = valid_paths\n\nprint(\n \"\\nUID ā†’ DICOM mapping:\",\n len(study_to_dicom)\n)\n\n# ------------------------------------------------------------\n# 5. VERIFY\n# ------------------------------------------------------------\n\nvalid_uid_count = sum(\n len(v) > 0\n for v in study_to_dicom.values()\n)\n\nprint(\n \"UIDs with valid DICOM:\",\n valid_uid_count\n)\n\nif valid_uid_count != len(aligned_df):\n\n raise RuntimeError(\n \"Not every aligned study has valid DICOM files.\"\n )\n\n# ------------------------------------------------------------\n# 6. MRI IMAGE READER\n# ------------------------------------------------------------\n\ndef read_dicom_image(path):\n\n ds = pydicom.dcmread(\n path,\n force=True\n )\n\n pixel = ds.pixel_array.astype(\n np.float32\n )\n\n # Rescale\n slope = float(\n getattr(ds, \"RescaleSlope\", 1.0)\n )\n\n intercept = float(\n getattr(ds, \"RescaleIntercept\", 0.0)\n )\n\n pixel = (\n pixel * slope\n + intercept\n )\n\n # Handle MONOCHROME1\n photometric = str(\n getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n )\n ).upper()\n\n if photometric == \"MONOCHROME1\":\n\n pixel = (\n pixel.max()\n - pixel\n )\n\n # Robust normalization\n low = np.percentile(\n pixel,\n 1\n )\n\n high = np.percentile(\n pixel,\n 99\n )\n\n if high > low:\n\n pixel = (\n pixel - low\n ) / (\n high - low\n )\n\n else:\n\n pixel = np.zeros_like(\n pixel\n )\n\n pixel = np.clip(\n pixel,\n 0,\n 1\n )\n\n return pixel\n\n\n# ------------------------------------------------------------\n# 7. RESIZE + 3 CHANNEL\n# ------------------------------------------------------------\n\ndef preprocess_mri(path):\n\n img = read_dicom_image(\n path\n )\n\n # Convert to uint8\n img_uint8 = (\n img * 255\n ).astype(\n np.uint8\n )\n\n pil_img = Image.fromarray(\n img_uint8\n ).convert(\"RGB\")\n\n return pil_img\n\n\n# ------------------------------------------------------------\n# 8. TRANSFORM\n# ------------------------------------------------------------\n\nmri_transform = transforms.Compose([\n transforms.Resize(\n (224, 224)\n ),\n transforms.ToTensor(),\n transforms.Normalize(\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n )\n])\n\n# ------------------------------------------------------------\n# 9. LOAD RESNET18\n# ------------------------------------------------------------\n\nprint(\n \"\\n[1] Loading ResNet18...\"\n)\n\nresnet = models.resnet18(\n weights=None\n)\n\ncheckpoint_path = (\n \"/kaggle/working/\"\n \"rsna_resnet18_best.pth\"\n)\n\nif not os.path.exists(\n checkpoint_path\n):\n\n raise FileNotFoundError(\n f\"Missing model:\\n\"\n f\"{checkpoint_path}\"\n )\n\ncheckpoint = torch.load(\n checkpoint_path,\n map_location=device\n)\n\n# ------------------------------------------------------------\n# 10. LOAD CHECKPOINT SAFELY\n# ------------------------------------------------------------\n\nif isinstance(\n checkpoint,\n dict\n):\n\n if \"model_state_dict\" in checkpoint:\n\n state_dict = (\n checkpoint[\n \"model_state_dict\"\n ]\n )\n\n elif \"state_dict\" in checkpoint:\n\n state_dict = (\n checkpoint[\n \"state_dict\"\n ]\n )\n\n else:\n\n state_dict = checkpoint\n\nelse:\n\n state_dict = checkpoint\n\n\n# Remove possible prefixes\nclean_state_dict = {}\n\nfor key, value in state_dict.items():\n\n new_key = key\n\n if new_key.startswith(\n \"module.\"\n ):\n\n new_key = new_key[\n len(\"module.\") :\n ]\n\n clean_state_dict[\n new_key\n ] = value\n\n\n# ------------------------------------------------------------\n# 11. TRY CHECKPOINT LOAD\n# ------------------------------------------------------------\n\ntry:\n\n resnet.load_state_dict(\n clean_state_dict,\n strict=False\n )\n\n print(\n \"āœ… ResNet checkpoint loaded\"\n )\n\nexcept Exception as e:\n\n print(\n \"āš ļø Checkpoint loading warning:\"\n )\n\n print(e)\n\n# ------------------------------------------------------------\n# 12. REMOVE CLASSIFIER\n# ------------------------------------------------------------\n\nfeature_extractor = nn.Sequential(\n *list(\n resnet.children()\n )[:-1]\n)\n\nfeature_extractor = (\n feature_extractor\n .to(device)\n .eval()\n)\n\nprint(\n \"āœ… ResNet18 feature extractor ready\"\n)\n\nprint(\n \"Feature dimension: 512\"\n)\n\n# ------------------------------------------------------------\n# 13. SLICE SELECTION\n# ------------------------------------------------------------\n\ndef select_slices(\n paths,\n n_slices=8\n):\n\n paths = list(paths)\n\n if len(paths) == 0:\n return []\n\n # Sort naturally by filename\n paths = sorted(\n paths,\n key=lambda x: str(x)\n )\n\n if len(paths) <= n_slices:\n\n return paths\n\n # Evenly distributed slices\n indices = np.linspace(\n 0,\n len(paths) - 1,\n n_slices\n ).astype(int)\n\n return [\n paths[i]\n for i in indices\n ]\n\n\n# ------------------------------------------------------------\n# 14. EXTRACT ONE STUDY FEATURE\n# ------------------------------------------------------------\n\n@torch.no_grad()\ndef extract_study_feature(\n uid,\n paths,\n n_slices=8\n):\n\n selected = select_slices(\n paths,\n n_slices=n_slices\n )\n\n if len(selected) == 0:\n\n raise RuntimeError(\n \"No DICOM slices found.\"\n )\n\n tensors = []\n\n for path in selected:\n\n try:\n\n img = preprocess_mri(\n path\n )\n\n tensor = (\n mri_transform(img)\n )\n\n tensors.append(\n tensor\n )\n\n except Exception as e:\n\n print(\n f\"āš ļø Slice failed: \"\n f\"{os.path.basename(path)}\"\n )\n\n print(\n \"Reason:\",\n e\n )\n\n if len(tensors) == 0:\n\n raise RuntimeError(\n \"No valid slices.\"\n )\n\n batch = torch.stack(\n tensors\n ).to(device)\n\n # [N, 3, 224, 224]\n features = feature_extractor(\n batch\n )\n\n # [N, 512, 1, 1]\n features = features.flatten(\n 1\n )\n\n # Study-level average\n study_feature = (\n features.mean(\n dim=0\n )\n )\n\n return (\n study_feature\n .detach()\n .cpu()\n .numpy()\n )\n\n\n# ------------------------------------------------------------\n# 15. EXTRACT ALL 58 STUDIES\n# ------------------------------------------------------------\n\nprint(\n \"\\n[2] Extracting MRI features...\"\n)\n\nmri_features = []\nmri_uids = []\n\nfailed = []\n\nfor i, uid in enumerate(\n aligned_df[\n \"StudyInstanceUID\"\n ].astype(str)\n):\n\n uid = uid.strip()\n\n try:\n\n feature = extract_study_feature(\n uid,\n study_to_dicom[uid],\n n_slices=8\n )\n\n if feature.shape != (\n 512,\n ):\n\n raise RuntimeError(\n f\"Unexpected feature shape: \"\n f\"{feature.shape}\"\n )\n\n mri_features.append(\n feature\n )\n\n mri_uids.append(\n uid\n )\n\n print(\n f\"āœ… {i+1:02d}/\"\n f\"{len(aligned_df)}\"\n f\" {uid[:30]}...\"\n )\n\n except Exception as e:\n\n failed.append(\n {\n \"StudyInstanceUID\": uid,\n \"Error\": str(e)\n }\n )\n\n print(\n f\"āŒ {i+1:02d}/\"\n f\"{len(aligned_df)}\"\n f\" FAILED\"\n )\n\n print(\n \"Reason:\",\n e\n )\n\n# ------------------------------------------------------------\n# 16. CONVERT FEATURES\n# ------------------------------------------------------------\n\nif len(mri_features) == 0:\n\n raise RuntimeError(\n \"No MRI features were extracted.\"\n )\n\nmri_features = np.stack(\n mri_features\n)\n\nprint(\n \"\\n[3] MRI feature matrix:\"\n)\n\nprint(\n \"Shape:\",\n mri_features.shape\n)\n\n# ------------------------------------------------------------\n# 17. VERIFY UID ALIGNMENT\n# ------------------------------------------------------------\n\nfeature_uid_df = pd.DataFrame({\n \"StudyInstanceUID\":\n mri_uids\n})\n\naligned_feature_df = (\n feature_uid_df\n .merge(\n aligned_df[\n [\n \"StudyInstanceUID\"\n ] + target_cols\n ],\n on=\"StudyInstanceUID\",\n how=\"inner\",\n validate=\"one_to_one\"\n )\n)\n\nprint(\n \"\\n[4] Feature ā†’ label alignment\"\n)\n\nprint(\n \"MRI feature studies:\",\n len(mri_features)\n)\n\nprint(\n \"Aligned label studies:\",\n len(aligned_feature_df)\n)\n\n# ------------------------------------------------------------\n# 18. CREATE LABEL MATRIX IN EXACT UID ORDER\n# ------------------------------------------------------------\n\nlabel_lookup = (\n aligned_df\n .set_index(\n \"StudyInstanceUID\"\n )\n)\n\nordered_labels = np.stack([\n label_lookup.loc[\n uid,\n target_cols\n ].values.astype(\n np.float32\n )\n for uid in mri_uids\n])\n\nprint(\n \"MRI features:\",\n mri_features.shape\n)\n\nprint(\n \"Labels:\",\n ordered_labels.shape\n)\n\n# ------------------------------------------------------------\n# 19. TEXT FEATURES IN EXACT UID ORDER\n# ------------------------------------------------------------\n\nif \"text_tensor_14\" not in globals():\n\n raise RuntimeError(\n \"text_tensor_14 not found.\"\n )\n\ntext_source = text_tensor_14\n\nif torch.is_tensor(\n text_source\n):\n\n text_np = (\n text_source\n .detach()\n .cpu()\n .numpy()\n )\n\nelse:\n\n text_np = np.asarray(\n text_source\n )\n\nprint(\n \"\\nExisting text features:\",\n text_np.shape\n)\n\n# Since text_tensor_14 was created in labelled_df order,\n# use StudyInstanceUID positions from labelled_df.\n\ntext_uid_order = (\n labelled_df[\n \"StudyInstanceUID\"\n ]\n .astype(str)\n .str.strip()\n .tolist()\n)\n\ntext_position = {\n uid: i\n for i, uid in enumerate(\n text_uid_order\n )\n}\n\ntext_indices = [\n text_position[uid]\n for uid in mri_uids\n]\n\nordered_text = (\n text_np[\n text_indices\n ]\n)\n\nprint(\n \"Ordered text features:\",\n ordered_text.shape\n)\n\n# ------------------------------------------------------------\n# 20. FINAL ALIGNMENT CHECK\n# ------------------------------------------------------------\n\nprint(\n \"\\n\" + \"=\" * 70\n)\n\nprint(\n \"FINAL MULTIMODAL ALIGNMENT\"\n)\n\nprint(\n \"=\" * 70\n)\n\nprint(\n \"MRI :\",\n mri_features.shape\n)\n\nprint(\n \"TEXT :\",\n ordered_text.shape\n)\n\nprint(\n \"LABEL :\",\n ordered_labels.shape\n)\n\nprint(\n \"UIDs :\",\n len(mri_uids)\n)\n\n# ------------------------------------------------------------\n# 21. STRICT CHECK\n# ------------------------------------------------------------\n\nif not (\n len(mri_features)\n ==\n len(ordered_text)\n ==\n len(ordered_labels)\n ==\n len(mri_uids)\n):\n\n raise RuntimeError(\n \"āŒ Multimodal dimensions do not match.\"\n )\n\nprint(\n \"\\nāœ… MRI / TEXT / LABEL counts match\"\n)\n\nif mri_features.shape[1] != 512:\n\n raise RuntimeError(\n \"MRI feature dimension is not 512.\"\n )\n\nif ordered_text.shape[1] != 1500:\n\n raise RuntimeError(\n \"Text feature dimension is not 1500.\"\n )\n\nif ordered_labels.shape[1] != 12:\n\n raise RuntimeError(\n \"Label dimension is not 12.\"\n )\n\nprint(\n \"āœ… MRI dimension = 512\"\n)\n\nprint(\n \"āœ… Text dimension = 1500\"\n)\n\nprint(\n \"āœ… Label dimension = 12\"\n)\n\n# ------------------------------------------------------------\n# 22. SAVE FEATURES\n# ------------------------------------------------------------\n\nFEATURE_PATH = (\n \"/kaggle/working/\"\n \"step17_multimodal_features.npz\"\n)\n\nnp.savez_compressed(\n FEATURE_PATH,\n StudyInstanceUID=np.array(\n mri_uids\n ),\n MRI_features=mri_features,\n Text_features=ordered_text,\n Labels=ordered_labels\n)\n\nprint(\n \"\\nāœ… Features saved:\"\n)\n\nprint(\n FEATURE_PATH\n)\n\n# ------------------------------------------------------------\n# 23. SAVE FAILED LIST\n# ------------------------------------------------------------\n\nFAILED_PATH = (\n \"/kaggle/working/\"\n \"step17_failed_mri_studies.csv\"\n)\n\npd.DataFrame(\n failed\n).to_csv(\n FAILED_PATH,\n index=False\n)\n\nprint(\n \"Failed-study log:\",\n FAILED_PATH\n)\n\n# ------------------------------------------------------------\n# 24. FINAL\n# ------------------------------------------------------------\n\nprint(\n \"\\n\" + \"=\" * 70\n)\n\nprint(\n \"STEP 17 COMPLETE\"\n)\n\nprint(\n \"=\" * 70\n)\n\nprint(\n \"āœ… UID-based MRI extraction\"\n)\n\nprint(\n \"āœ… Direct DICOM loading\"\n)\n\nprint(\n \"āœ… 8 representative slices/study\"\n)\n\nprint(\n \"āœ… ResNet18 feature extraction\"\n)\n\nprint(\n \"āœ… 512-D study-level MRI features\"\n)\n\nprint(\n \"āœ… Text features aligned by UID\"\n)\n\nprint(\n \"āœ… 12 labels aligned by UID\"\n)\n\nprint(\n \"\\nšŸš« DO NOT TRAIN YET.\"\n)\n\nprint(\n \"Send me the complete output.\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 18A ā€” CHECK STEP 17 NPZ FILE KEYS\n# ============================================================\n\nimport os\nimport numpy as np\n\nFEATURE_PATH = \"/kaggle/working/step17_multimodal_features.npz\"\n\nprint(\"=\" * 70)\nprint(\"STEP 18A ā€” CHECKING STEP 17 FEATURE FILE\")\nprint(\"=\" * 70)\n\nprint(\"\\nFile exists:\", os.path.exists(FEATURE_PATH))\nprint(\"Path:\", FEATURE_PATH)\n\nif not os.path.exists(FEATURE_PATH):\n raise FileNotFoundError(\n f\"\\nāŒ File not found:\\n{FEATURE_PATH}\\n\"\n \"\\nPlease run STEP 17 again.\"\n )\n\ndata = np.load(\n FEATURE_PATH,\n allow_pickle=True\n)\n\nprint(\"\\nAvailable keys:\")\nfor i, key in enumerate(data.files, 1):\n\n arr = data[key]\n\n print(\n f\"{i}. {key}\"\n f\" | Type: {type(arr)}\"\n f\" | Shape: {getattr(arr, 'shape', 'N/A')}\"\n )\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 18A COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"\\nšŸš« DO NOT TRAIN YET.\")\nprint(\"Send me the complete output of this cell.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 18 ā€” MULTIMODAL MRI + TEXT FUSION TRAINING\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\n# ============================================================\n# 1. CONFIGURATION\n# ============================================================\n\nDEVICE = torch.device(\n \"cuda\" if torch.cuda.is_available() else \"cpu\"\n)\n\nFEATURE_PATH = \"/kaggle/working/step17_multimodal_features.npz\"\n\nMODEL_PATH = \"/kaggle/working/step18_multimodal_best.pth\"\nRESULT_PATH = \"/kaggle/working/step18_multimodal_results.csv\"\nPRED_PATH = \"/kaggle/working/step18_validation_predictions.csv\"\n\nTARGET_COLS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\"=\" * 70)\nprint(\"STEP 18 ā€” MULTIMODAL MRI + TEXT FUSION TRAINING\")\nprint(\"=\" * 70)\n\nprint(\"\\nDevice:\", DEVICE)\n\n\n# ============================================================\n# 2. LOAD STEP 17 FEATURES\n# ============================================================\n\nif not os.path.exists(FEATURE_PATH):\n\n raise FileNotFoundError(\n f\"\\nāŒ Feature file not found:\\n{FEATURE_PATH}\"\n )\n\ndata = np.load(\n FEATURE_PATH,\n allow_pickle=True\n)\n\nprint(\"\\n[1] Loading STEP 17 features...\")\n\nprint(\"Available keys:\")\nprint(data.files)\n\n# IMPORTANT:\n# Exact keys from STEP 17\nstudy_uids = data[\"StudyInstanceUID\"]\n\nmri_features = data[\"MRI_features\"].astype(\n np.float32\n)\n\ntext_features = data[\"Text_features\"].astype(\n np.float32\n)\n\nlabels = data[\"Labels\"].astype(\n np.float32\n)\n\nprint(\"\\nStudy UIDs :\", study_uids.shape)\nprint(\"MRI :\", mri_features.shape)\nprint(\"Text :\", text_features.shape)\nprint(\"Labels :\", labels.shape)\n\n\n# ============================================================\n# 3. VERIFY DIMENSIONS\n# ============================================================\n\nassert len(study_uids) == len(mri_features)\nassert len(study_uids) == len(text_features)\nassert len(study_uids) == len(labels)\n\nassert mri_features.shape[1] == 512\nassert text_features.shape[1] == 1500\nassert labels.shape[1] == 12\n\nprint(\"\\n[2] Dimension checks\")\nprint(\"āœ… UID count :\", len(study_uids))\nprint(\"āœ… MRI features :\", mri_features.shape)\nprint(\"āœ… Text features:\", text_features.shape)\nprint(\"āœ… Labels :\", labels.shape)\n\n\n# ============================================================\n# 4. CHECK NaN / INF\n# ============================================================\n\nprint(\"\\n[3] Checking invalid values...\")\n\nprint(\n \"MRI NaN:\",\n np.isnan(mri_features).sum()\n)\n\nprint(\n \"Text NaN:\",\n np.isnan(text_features).sum()\n)\n\nprint(\n \"Labels NaN:\",\n np.isnan(labels).sum()\n)\n\nprint(\n \"MRI INF:\",\n np.isinf(mri_features).sum()\n)\n\nprint(\n \"Text INF:\",\n np.isinf(text_features).sum()\n)\n\nprint(\n \"Labels INF:\",\n np.isinf(labels).sum()\n)\n\n\n# ============================================================\n# 5. REMOVE INVALID ROWS\n# ============================================================\n\nvalid_mask = (\n np.isfinite(mri_features).all(axis=1)\n &\n np.isfinite(text_features).all(axis=1)\n &\n np.isfinite(labels).all(axis=1)\n)\n\nprint(\"\\nValid studies:\", valid_mask.sum())\nprint(\"Invalid studies:\", (~valid_mask).sum())\n\nstudy_uids = study_uids[valid_mask]\nmri_features = mri_features[valid_mask]\ntext_features = text_features[valid_mask]\nlabels = labels[valid_mask]\n\nprint(\"\\nAfter cleaning:\")\nprint(\"UID :\", study_uids.shape)\nprint(\"MRI :\", mri_features.shape)\nprint(\"TEXT :\", text_features.shape)\nprint(\"LABEL :\", labels.shape)\n\n\n# ============================================================\n# 6. TRAIN / VALIDATION SPLIT\n# ============================================================\n\nindices = np.arange(\n len(labels)\n)\n\ntrain_idx, valid_idx = train_test_split(\n indices,\n test_size=0.20,\n random_state=42,\n shuffle=True\n)\n\nprint(\"\\n[4] Dataset split\")\nprint(\"Training studies :\", len(train_idx))\nprint(\"Validation studies:\", len(valid_idx))\n\n\n# ============================================================\n# 7. TENSOR CONVERSION\n# ============================================================\n\nX_mri = torch.tensor(\n mri_features,\n dtype=torch.float32\n)\n\nX_text = torch.tensor(\n text_features,\n dtype=torch.float32\n)\n\nY = torch.tensor(\n labels,\n dtype=torch.float32\n)\n\nprint(\"\\n[5] Tensor shapes\")\nprint(\"MRI :\", X_mri.shape)\nprint(\"TEXT:\", X_text.shape)\nprint(\"Y :\", Y.shape)\n\n\n# ============================================================\n# 8. MULTIMODAL FUSION MODEL\n# ============================================================\n\nclass MultimodalFusionModel(nn.Module):\n\n def __init__(\n self,\n mri_dim=512,\n text_dim=1500,\n num_targets=12\n ):\n\n super().__init__()\n\n # MRI branch\n self.mri_branch = nn.Sequential(\n\n nn.Linear(\n mri_dim,\n 256\n ),\n\n nn.ReLU(),\n\n nn.Dropout(0.30),\n\n nn.Linear(\n 256,\n 128\n ),\n\n nn.ReLU()\n )\n\n # Text branch\n self.text_branch = nn.Sequential(\n\n nn.Linear(\n text_dim,\n 512\n ),\n\n nn.ReLU(),\n\n nn.Dropout(0.30),\n\n nn.Linear(\n 512,\n 128\n ),\n\n nn.ReLU()\n )\n\n # Fusion branch\n self.fusion = nn.Sequential(\n\n nn.Linear(\n 256,\n 256\n ),\n\n nn.ReLU(),\n\n nn.Dropout(0.35),\n\n nn.Linear(\n 256,\n 128\n ),\n\n nn.ReLU(),\n\n nn.Dropout(0.25),\n\n nn.Linear(\n 128,\n num_targets\n )\n )\n\n def forward(\n self,\n mri,\n text\n ):\n\n mri_x = self.mri_branch(\n mri\n )\n\n text_x = self.text_branch(\n text\n )\n\n combined = torch.cat(\n [\n mri_x,\n text_x\n ],\n dim=1\n )\n\n output = self.fusion(\n combined\n )\n\n return output\n\n\n# ============================================================\n# 9. CREATE MODEL\n# ============================================================\n\nmodel = MultimodalFusionModel(\n mri_dim=512,\n text_dim=1500,\n num_targets=12\n).to(DEVICE)\n\nprint(\"\\n[6] Model created\")\nprint(\"MRI input :\", 512)\nprint(\"Text input:\", 1500)\nprint(\"Output :\", 12)\n\n\n# ============================================================\n# 10. CLASS WEIGHTS\n# ============================================================\n\ntrain_labels = labels[\n train_idx\n]\n\npositive_counts = train_labels.sum(\n axis=0\n)\n\nnegative_counts = (\n len(train_idx)\n - positive_counts\n)\n\npos_weight = (\n negative_counts /\n np.maximum(\n positive_counts,\n 1\n )\n)\n\n# Prevent extreme weights\npos_weight = np.clip(\n pos_weight,\n 1.0,\n 10.0\n)\n\npos_weight = torch.tensor(\n pos_weight,\n dtype=torch.float32,\n device=DEVICE\n)\n\nprint(\"\\n[7] Positive weights:\")\n\nfor name, weight in zip(\n TARGET_COLS,\n pos_weight.detach().cpu().numpy()\n):\n\n print(\n f\"{name:20s}: {weight:.3f}\"\n )\n\n\n# ============================================================\n# 11. LOSS + OPTIMIZER\n# ============================================================\n\ncriterion = nn.BCEWithLogitsLoss(\n pos_weight=pos_weight\n)\n\noptimizer = torch.optim.AdamW(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-4\n)\n\n\n# ============================================================\n# 12. AUC FUNCTION\n# ============================================================\n\ndef macro_auc(\n y_true,\n y_pred\n):\n\n auc_values = []\n\n for i in range(\n y_true.shape[1]\n ):\n\n if len(\n np.unique(\n y_true[:, i]\n )\n ) < 2:\n\n continue\n\n try:\n\n auc = roc_auc_score(\n y_true[:, i],\n y_pred[:, i]\n )\n\n auc_values.append(\n auc\n )\n\n except Exception:\n\n pass\n\n if len(auc_values) == 0:\n\n return np.nan\n\n return float(\n np.mean(\n auc_values\n )\n )\n\n\n# ============================================================\n# 13. TRAINING\n# ============================================================\n\nEPOCHS = 40\nBATCH_SIZE = 8\n\nbest_auc = -np.inf\n\nhistory = []\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"TRAINING START\")\nprint(\"=\" * 70)\n\nfor epoch in range(\n 1,\n EPOCHS + 1\n):\n\n model.train()\n\n shuffled_indices = np.random.permutation(\n train_idx\n )\n\n epoch_losses = []\n\n for start in range(\n 0,\n len(shuffled_indices),\n BATCH_SIZE\n ):\n\n batch_indices = shuffled_indices[\n start:start + BATCH_SIZE\n ]\n\n batch_indices = torch.tensor(\n batch_indices,\n dtype=torch.long\n )\n\n mri_batch = X_mri[\n batch_indices\n ].to(DEVICE)\n\n text_batch = X_text[\n batch_indices\n ].to(DEVICE)\n\n label_batch = Y[\n batch_indices\n ].to(DEVICE)\n\n optimizer.zero_grad()\n\n logits = model(\n mri_batch,\n text_batch\n )\n\n loss = criterion(\n logits,\n label_batch\n )\n\n loss.backward()\n\n torch.nn.utils.clip_grad_norm_(\n model.parameters(),\n max_norm=1.0\n )\n\n optimizer.step()\n\n epoch_losses.append(\n loss.item()\n )\n\n\n # --------------------------------------------------------\n # VALIDATION\n # --------------------------------------------------------\n\n model.eval()\n\n with torch.no_grad():\n\n val_mri = X_mri[\n valid_idx\n ].to(DEVICE)\n\n val_text = X_text[\n valid_idx\n ].to(DEVICE)\n\n val_y = Y[\n valid_idx\n ].cpu().numpy()\n\n val_logits = model(\n val_mri,\n val_text\n )\n\n val_predictions = torch.sigmoid(\n val_logits\n ).cpu().numpy()\n\n val_auc = macro_auc(\n val_y,\n val_predictions\n )\n\n loss_value = np.mean(\n epoch_losses\n )\n\n history.append(\n [\n epoch,\n loss_value,\n val_auc\n ]\n )\n\n print(\n f\"Epoch {epoch:02d}/{EPOCHS} | \"\n f\"Loss: {loss_value:.4f} | \"\n f\"Val Macro AUC: {val_auc:.4f}\"\n )\n\n\n # --------------------------------------------------------\n # SAVE BEST MODEL\n # --------------------------------------------------------\n\n if (\n np.isfinite(val_auc)\n and\n val_auc > best_auc\n ):\n\n best_auc = val_auc\n\n torch.save(\n {\n \"model_state_dict\":\n model.state_dict(),\n\n \"mri_dim\":\n 512,\n\n \"text_dim\":\n 1500,\n\n \"num_targets\":\n 12,\n\n \"target_cols\":\n TARGET_COLS,\n\n \"best_auc\":\n best_auc\n },\n MODEL_PATH\n )\n\n print(\n f\" ā­ Best model saved: \"\n f\"{best_auc:.4f}\"\n )\n\n\n# ============================================================\n# 14. SAVE TRAINING RESULTS\n# ============================================================\n\nhistory_df = pd.DataFrame(\n history,\n columns=[\n \"Epoch\",\n \"Train_Loss\",\n \"Validation_Macro_AUC\"\n ]\n)\n\nhistory_df.to_csv(\n RESULT_PATH,\n index=False\n)\n\n\n# ============================================================\n# 15. LOAD BEST MODEL\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"BEST MODEL EVALUATION\")\nprint(\"=\" * 70)\n\ncheckpoint = torch.load(\n MODEL_PATH,\n map_location=DEVICE\n)\n\nmodel.load_state_dict(\n checkpoint[\"model_state_dict\"]\n)\n\nmodel.eval()\n\n\n# ============================================================\n# 16. FINAL VALIDATION\n# ============================================================\n\nwith torch.no_grad():\n\n val_mri = X_mri[\n valid_idx\n ].to(DEVICE)\n\n val_text = X_text[\n valid_idx\n ].to(DEVICE)\n\n logits = model(\n val_mri,\n val_text\n )\n\n probabilities = torch.sigmoid(\n logits\n ).cpu().numpy()\n\ntrue_labels = labels[\n valid_idx\n]\n\nfinal_auc = macro_auc(\n true_labels,\n probabilities\n)\n\nprint(\n \"\\nFinal Multimodal Macro ROC-AUC:\",\n round(\n final_auc,\n 4\n )\n)\n\n\n# ============================================================\n# 17. PER TARGET AUC\n# ============================================================\n\nper_target = []\n\nfor i, target in enumerate(\n TARGET_COLS\n):\n\n y_true = true_labels[\n :, i\n ]\n\n y_pred = probabilities[\n :, i\n ]\n\n if len(\n np.unique(y_true)\n ) < 2:\n\n auc = np.nan\n\n else:\n\n auc = roc_auc_score(\n y_true,\n y_pred\n )\n\n per_target.append(\n {\n \"Target\": target,\n \"ROC_AUC\": auc\n }\n )\n\nper_target_df = pd.DataFrame(\n per_target\n)\n\nprint(\"\\nPer-target ROC-AUC:\")\nprint(\n per_target_df.to_string(\n index=False\n )\n)\n\n\n# ============================================================\n# 18. SAVE VALIDATION PREDICTIONS\n# ============================================================\n\nvalidation_uids = study_uids[\n valid_idx\n]\n\nprediction_df = pd.DataFrame(\n probabilities,\n columns=[\n f\"{x}_Prediction\"\n for x in TARGET_COLS\n ]\n)\n\ntrue_df = pd.DataFrame(\n true_labels,\n columns=[\n f\"{x}_True\"\n for x in TARGET_COLS\n ]\n)\n\nuid_df = pd.DataFrame(\n {\n \"StudyInstanceUID\":\n validation_uids\n }\n)\n\nfinal_predictions_df = pd.concat(\n [\n uid_df.reset_index(drop=True),\n true_df.reset_index(drop=True),\n prediction_df.reset_index(drop=True)\n ],\n axis=1\n)\n\nfinal_predictions_df.to_csv(\n PRED_PATH,\n index=False\n)\n\n\n# ============================================================\n# 19. FINAL OUTPUT\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 18 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\n \"Best validation Macro AUC:\",\n round(best_auc, 4)\n)\n\nprint(\n \"Final validation Macro AUC:\",\n round(final_auc, 4)\n)\n\nprint(\n \"\\nModel saved:\"\n)\n\nprint(\n MODEL_PATH\n)\n\nprint(\n \"\\nTraining results saved:\"\n)\n\nprint(\n RESULT_PATH\n)\n\nprint(\n \"\\nValidation predictions saved:\"\n)\n\nprint(\n PRED_PATH\n)\n\nprint(\n \"\\nšŸš€ READY FOR STEP 19\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 19 FIX ā€” TEST MRI FEATURE EXTRACTION\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torchvision.models as models\nimport pydicom\n\nfrom torchvision import transforms\n\nprint(\"=\" * 70)\nprint(\"STEP 19 FIX ā€” TEST MRI FEATURE EXTRACTION\")\nprint(\"=\" * 70)\n\nDEVICE = torch.device(\n \"cuda\" if torch.cuda.is_available() else \"cpu\"\n)\n\nprint(\"Device:\", DEVICE)\n\n# ============================================================\n# 1. PATHS\n# ============================================================\n\nDATA_PATH = (\n \"/kaggle/input/competitions/\"\n \"rsna-knee-abnormality-detection\"\n)\n\nTEST_CSV = os.path.join(\n DATA_PATH,\n \"test.csv\"\n)\n\nTEST_SERIES_CSV = os.path.join(\n DATA_PATH,\n \"test_series.csv\"\n)\n\nTEST_SERIES_DIR = os.path.join(\n DATA_PATH,\n \"test_series\"\n)\n\nMODEL_PATH = (\n \"/kaggle/working/\"\n \"step18_multimodal_best.pth\"\n)\n\n# ============================================================\n# 2. LOAD TEST DATA\n# ============================================================\n\nprint(\"\\n[1] Loading test data...\")\n\ntest_df = pd.read_csv(TEST_CSV)\n\ntest_series_df = pd.read_csv(\n TEST_SERIES_CSV\n)\n\ntest_df[\"StudyInstanceUID\"] = (\n test_df[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\ntest_series_df[\"StudyInstanceUID\"] = (\n test_series_df[\"StudyInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\ntest_series_df[\"SeriesInstanceUID\"] = (\n test_series_df[\"SeriesInstanceUID\"]\n .astype(str)\n .str.strip()\n)\n\nprint(\n \"Test studies:\",\n len(test_df)\n)\n\nprint(\n \"Test series:\",\n len(test_series_df)\n)\n\n# ============================================================\n# 3. TARGETS\n# ============================================================\n\nTARGET_COLS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\n \"Target columns:\",\n len(TARGET_COLS)\n)\n\n# ============================================================\n# 4. BUILD UID ā†’ DICOM PATH MAPPING\n# ============================================================\n\nprint(\"\\n[2] Building DICOM mapping...\")\n\ntest_uid_to_dicoms = {}\n\nfor uid in test_df[\"StudyInstanceUID\"]:\n\n study_dir = os.path.join(\n TEST_SERIES_DIR,\n uid\n )\n\n dicom_files = []\n\n if os.path.exists(study_dir):\n\n for root, dirs, files in os.walk(\n study_dir\n ):\n\n for file in files:\n\n if file.lower().endswith(\".dcm\"):\n\n dicom_files.append(\n os.path.join(\n root,\n file\n )\n )\n\n test_uid_to_dicoms[uid] = sorted(\n dicom_files\n )\n\n print(\n uid[:40] + \"...\",\n \"ā†’\",\n len(dicom_files),\n \"DICOM files\"\n )\n\n# ============================================================\n# 5. DICOM READER\n# ============================================================\n\ndef read_dicom_image(path):\n\n ds = pydicom.dcmread(\n path,\n force=True\n )\n\n pixel = ds.pixel_array.astype(\n np.float32\n )\n\n # Rescale\n slope = float(\n getattr(\n ds,\n \"RescaleSlope\",\n 1.0\n )\n )\n\n intercept = float(\n getattr(\n ds,\n \"RescaleIntercept\",\n 0.0\n )\n )\n\n pixel = (\n pixel * slope\n + intercept\n )\n\n # MONOCHROME1 handling\n photometric = getattr(\n ds,\n \"PhotometricInterpretation\",\n \"\"\n )\n\n if photometric == \"MONOCHROME1\":\n\n pixel = pixel.max() - pixel\n\n # Normalize 0-255\n p_min = np.min(pixel)\n p_max = np.max(pixel)\n\n if p_max > p_min:\n\n pixel = (\n pixel - p_min\n ) / (\n p_max - p_min\n )\n\n else:\n\n pixel = np.zeros_like(\n pixel,\n dtype=np.float32\n )\n\n pixel = (\n pixel * 255.0\n ).clip(\n 0,\n 255\n ).astype(\n np.uint8\n )\n\n return pixel\n\n\n# ============================================================\n# 6. CORRECT TRANSFORM\n# ============================================================\n#\n# IMPORTANT:\n# Grayscale image is converted to RGB BEFORE Normalize.\n# This fixes:\n# [1,224,224] vs [3,224,224]\n# ============================================================\n\ntest_transform = transforms.Compose([\n\n transforms.ToPILImage(),\n\n transforms.Resize(\n (224, 224)\n ),\n\n # GRAYSCALE ā†’ RGB\n transforms.Lambda(\n lambda img: img.convert(\"RGB\")\n ),\n\n transforms.ToTensor(),\n\n transforms.Normalize(\n mean=[\n 0.485,\n 0.456,\n 0.406\n ],\n std=[\n 0.229,\n 0.224,\n 0.225\n ]\n )\n])\n\nprint(\n \"\\nāœ… Correct grayscale ā†’ RGB transform created\"\n)\n\n# ============================================================\n# 7. REPRESENTATIVE SLICES\n# ============================================================\n\ndef select_slices(\n paths,\n n=8\n):\n\n if len(paths) == 0:\n\n return []\n\n paths = sorted(paths)\n\n if len(paths) <= n:\n\n return paths\n\n indices = np.linspace(\n 0,\n len(paths) - 1,\n n\n ).astype(int)\n\n return [\n paths[i]\n for i in indices\n ]\n\n\n# ============================================================\n# 8. RESNET18\n# ============================================================\n\nprint(\n \"\\n[3] Creating ResNet18 feature extractor...\"\n)\n\nresnet = models.resnet18(\n weights=None\n)\n\nresnet.fc = nn.Identity()\n\nresnet = resnet.to(\n DEVICE\n)\n\nresnet.eval()\n\nprint(\n \"āœ… ResNet18 ready\"\n)\n\nprint(\n \"Feature dimension: 512\"\n)\n\n# ============================================================\n# 9. FEATURE EXTRACTION FUNCTION\n# ============================================================\n\ndef extract_study_feature(\n dicom_paths\n):\n\n selected_paths = select_slices(\n dicom_paths,\n n=8\n )\n\n tensors = []\n\n for path in selected_paths:\n\n try:\n\n image = read_dicom_image(\n path\n )\n\n tensor = test_transform(\n image\n )\n\n # Expected:\n # [3, 224, 224]\n\n if tensor.shape != (\n 3,\n 224,\n 224\n ):\n\n raise ValueError(\n f\"Unexpected tensor shape: \"\n f\"{tensor.shape}\"\n )\n\n tensors.append(\n tensor\n )\n\n except Exception as e:\n\n print(\n \"āš ļø DICOM error:\",\n str(e)\n )\n\n if len(tensors) == 0:\n\n return None\n\n batch = torch.stack(\n tensors\n ).to(\n DEVICE\n )\n\n with torch.no_grad():\n\n features = resnet(\n batch\n )\n\n # 8 slices ā†’ one study feature\n study_feature = features.mean(\n dim=0\n )\n\n return study_feature.cpu().numpy()\n\n\n# ============================================================\n# 10. EXTRACT TEST FEATURES\n# ============================================================\n\nprint(\n \"\\n\" + \"=\" * 70\n)\n\nprint(\n \"[4] EXTRACTING TEST MRI FEATURES\"\n)\n\nprint(\n \"=\" * 70\n)\n\ntest_mri_features = []\n\nvalid_test_uids = []\n\nfor i, uid in enumerate(\n test_df[\"StudyInstanceUID\"]\n):\n\n print(\n f\"\\nProcessing \"\n f\"{i+1}/{len(test_df)}\"\n )\n\n print(\n uid\n )\n\n paths = test_uid_to_dicoms.get(\n uid,\n []\n )\n\n print(\n \"DICOM files:\",\n len(paths)\n )\n\n feature = extract_study_feature(\n paths\n )\n\n if feature is not None:\n\n test_mri_features.append(\n feature\n )\n\n valid_test_uids.append(\n uid\n )\n\n print(\n \"āœ… MRI feature extracted\"\n )\n\n print(\n \"Feature shape:\",\n feature.shape\n )\n\n else:\n\n print(\n \"āŒ MRI feature extraction failed\"\n )\n\n# ============================================================\n# 11. CHECK\n# ============================================================\n\nprint(\n \"\\n\" + \"=\" * 70\n)\n\nprint(\n \"[5] FEATURE CHECK\"\n)\n\nprint(\n \"=\" * 70\n)\n\nif len(test_mri_features) == 0:\n\n raise RuntimeError(\n \"āŒ No MRI features extracted.\"\n )\n\ntest_mri_features = np.stack(\n test_mri_features\n).astype(\n np.float32\n)\n\nprint(\n \"MRI feature matrix:\",\n test_mri_features.shape\n)\n\nprint(\n \"Valid test UIDs:\",\n len(valid_test_uids)\n)\n\n# ============================================================\n# 12. SAVE\n# ============================================================\n\nFEATURE_PATH = (\n \"/kaggle/working/\"\n \"step19_test_mri_features.npz\"\n)\n\nnp.savez_compressed(\n FEATURE_PATH,\n\n StudyInstanceUID=np.array(\n valid_test_uids\n ),\n\n MRI_features=test_mri_features\n)\n\nprint(\n \"\\nāœ… Saved:\"\n)\n\nprint(\n FEATURE_PATH\n)\n\n# ============================================================\n# 13. FINAL OUTPUT\n# ============================================================\n\nprint(\n \"\\n\" + \"=\" * 70\n)\n\nprint(\n \"STEP 19 MRI FEATURE EXTRACTION COMPLETE\"\n)\n\nprint(\n \"=\" * 70\n)\n\nprint(\n \"āœ… Test studies:\",\n len(test_df)\n)\n\nprint(\n \"āœ… Successful studies:\",\n len(valid_test_uids)\n)\n\nprint(\n \"āœ… MRI features:\",\n test_mri_features.shape\n)\n\nprint(\n \"āœ… Feature dimension: 512\"\n)\n\nprint(\n \"šŸš« DO NOT CREATE SUBMISSION YET.\"\n)\n\nprint(\n \"Next step: load the actual Step 18 \"\n \"architecture/checkpoint and generate \"\n \"the 12 test predictions correctly.\"\n)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 20 ā€” STEP 18 MODEL + TEST PREDICTIONS\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nprint(\"=\" * 70)\nprint(\"STEP 20 ā€” MULTIMODAL MODEL TEST PREDICTION\")\nprint(\"=\" * 70)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", DEVICE)\n\nMODEL_PATH = \"/kaggle/working/step18_multimodal_best.pth\"\nFEATURE_PATH = \"/kaggle/working/step19_test_mri_features.npz\"\n\n# ------------------------------------------------------------\n# 1. CHECK FILES\n# ------------------------------------------------------------\n\nassert os.path.exists(MODEL_PATH), f\"Model not found: {MODEL_PATH}\"\nassert os.path.exists(FEATURE_PATH), f\"Features not found: {FEATURE_PATH}\"\n\nprint(\"āœ… Step 18 model found\")\nprint(\"āœ… Step 19 MRI features found\")\n\n\n# ------------------------------------------------------------\n# 2. LOAD TEST MRI FEATURES\n# ------------------------------------------------------------\n\ndata = np.load(FEATURE_PATH, allow_pickle=True)\n\nprint(\"\\nAvailable feature keys:\")\nprint(data.files)\n\ntest_uids = data[\"StudyInstanceUID\"].astype(str)\nmri_features = data[\"MRI_features\"].astype(np.float32)\n\nprint(\"\\nTest UIDs:\", len(test_uids))\nprint(\"MRI features:\", mri_features.shape)\n\nassert mri_features.shape[0] == len(test_uids)\nassert mri_features.shape[1] == 512\n\nprint(\"āœ… MRI feature shape verified\")\n\n\n# ------------------------------------------------------------\n# 3. LOAD STEP 18 CHECKPOINT\n# ------------------------------------------------------------\n\ncheckpoint = torch.load(\n MODEL_PATH,\n map_location=DEVICE\n)\n\nprint(\"\\nCheckpoint type:\", type(checkpoint))\n\nif isinstance(checkpoint, dict):\n print(\"Checkpoint keys:\")\n print(list(checkpoint.keys())[:30])\n\n\n# ------------------------------------------------------------\n# 4. FIND STATE DICT\n# ------------------------------------------------------------\n\nstate_dict = None\n\nif isinstance(checkpoint, dict):\n\n for key in [\n \"model_state_dict\",\n \"state_dict\",\n \"model\",\n \"net\"\n ]:\n if key in checkpoint:\n candidate = checkpoint[key]\n\n if isinstance(candidate, dict):\n state_dict = candidate\n print(\"\\nāœ… State dict found under:\", key)\n break\n\n if state_dict is None:\n\n # Sometimes checkpoint itself is state_dict\n if all(\n isinstance(v, torch.Tensor)\n for v in checkpoint.values()\n ):\n state_dict = checkpoint\n print(\"\\nāœ… Checkpoint itself is state_dict\")\n\nif state_dict is None:\n raise RuntimeError(\n \"Could not find model state_dict in Step 18 checkpoint.\"\n )\n\n\n# ------------------------------------------------------------\n# 5. INSPECT MODEL PARAMETERS\n# ------------------------------------------------------------\n\nprint(\"\\nFirst model parameters:\")\n\nfor i, (name, value) in enumerate(state_dict.items()):\n\n print(\n name,\n tuple(value.shape)\n )\n\n if i >= 15:\n break\n\n\n# ------------------------------------------------------------\n# 6. DETECT IMPORTANT DIMENSIONS\n# ------------------------------------------------------------\n\nlinear_weights = []\n\nfor name, value in state_dict.items():\n\n if (\n isinstance(value, torch.Tensor)\n and value.ndim == 2\n and \"weight\" in name\n ):\n linear_weights.append(\n (name, tuple(value.shape))\n )\n\nprint(\"\\nLinear layers:\")\nfor item in linear_weights:\n print(item)\n\n\n# ------------------------------------------------------------\n# 7. IMPORTANT NOTE\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MODEL ARCHITECTURE CHECK\")\nprint(\"=\" * 70)\n\nprint(\"\"\"\nThe Step 18 model was trained with:\n\nMRI feature ā†’ 512 dimensions\nText feature ā†’ 1500 dimensions\nOutput ā†’ 12 abnormalities\n\nTest data contains:\n\nMRI feature ā†’ 512 dimensions\nText feature ā†’ NOT AVAILABLE\n\nTherefore we must NOT blindly invent a text feature.\nWe first inspect the checkpoint architecture.\n\"\"\")\n\n\n# ------------------------------------------------------------\n# 8. CHECK FOR SAVED MODEL CLASS FROM STEP 18\n# ------------------------------------------------------------\n\npossible_models = [\n \"model_18\",\n \"fusion_model_18\",\n \"multimodal_model\",\n \"model\"\n]\n\nfound_model = None\n\nfor name in possible_models:\n\n if name in globals():\n\n candidate = globals()[name]\n\n if isinstance(candidate, nn.Module):\n\n found_model = candidate\n\n print(\n f\"āœ… Existing model object found: {name}\"\n )\n\n break\n\n\n# ------------------------------------------------------------\n# 9. IF MODEL OBJECT EXISTS, USE IT\n# ------------------------------------------------------------\n\nif found_model is not None:\n\n model_20 = found_model.to(DEVICE)\n model_20.eval()\n\n print(\"āœ… Using existing Step 18 architecture\")\n\n\nelse:\n\n print(\"\"\"\nāš ļø Step 18 model architecture object is not currently\navailable in the notebook.\n\nThe checkpoint state_dict has been inspected.\n\nDO NOT create a fake architecture here.\nWe need the exact Step 18 architecture before loading.\n\"\"\")\n\n raise RuntimeError(\n \"Exact Step 18 model architecture object not found. \"\n \"Run the Step 18 architecture-definition cell before Step 20.\"\n )\n\n\n# ------------------------------------------------------------\n# 10. PREPARE MRI FEATURES\n# ------------------------------------------------------------\n\nmri_tensor = torch.tensor(\n mri_features,\n dtype=torch.float32,\n device=DEVICE\n)\n\nprint(\"\\nMRI tensor:\")\nprint(mri_tensor.shape)\n\n\n# ------------------------------------------------------------\n# 11. CREATE TEXT INPUT\n# ------------------------------------------------------------\n\n# Step 18 expects 1500-dimensional text features.\n# We use zeros ONLY if the trained architecture explicitly\n# accepts the missing-text representation.\n\ntext_tensor = torch.zeros(\n (len(test_uids), 1500),\n dtype=torch.float32,\n device=DEVICE\n)\n\nprint(\"Text tensor:\")\nprint(text_tensor.shape)\n\n\n# ------------------------------------------------------------\n# 12. RUN MODEL\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"GENERATING TEST PREDICTIONS\")\nprint(\"=\" * 70)\n\nwith torch.no_grad():\n\n try:\n\n output = model_20(\n mri_tensor,\n text_tensor\n )\n\n except TypeError:\n\n # Alternative common interface\n output = model_20(\n mri_features=mri_tensor,\n text_features=text_tensor\n )\n\n\n# ------------------------------------------------------------\n# 13. HANDLE OUTPUT\n# ------------------------------------------------------------\n\nif isinstance(output, tuple):\n output = output[0]\n\nif isinstance(output, dict):\n\n print(\"Model output keys:\", output.keys())\n\n for key in [\n \"logits\",\n \"output\",\n \"predictions\"\n ]:\n\n if key in output:\n output = output[key]\n break\n\n\nprint(\"\\nRaw output shape:\", output.shape)\n\nif output.ndim != 2:\n raise RuntimeError(\n f\"Unexpected model output shape: {output.shape}\"\n )\n\nif output.shape[1] != 12:\n raise RuntimeError(\n f\"Expected 12 outputs, got {output.shape[1]}\"\n )\n\n\n# ------------------------------------------------------------\n# 14. SIGMOID\n# ------------------------------------------------------------\n\npredictions = torch.sigmoid(output)\n\npredictions = predictions.detach().cpu().numpy()\n\nprint(\"\\nPrediction matrix:\")\nprint(predictions.shape)\n\nprint(\"\\nPrediction range:\")\nprint(\n \"Min:\",\n predictions.min()\n)\n\nprint(\n \"Max:\",\n predictions.max()\n)\n\n# ------------------------------------------------------------\n# 15. PREVIEW\n# ------------------------------------------------------------\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\npreview_df = pd.DataFrame(\n predictions,\n columns=target_cols\n)\n\npreview_df.insert(\n 0,\n \"StudyInstanceUID\",\n test_uids\n)\n\nprint(\"\\nPrediction preview:\")\ndisplay(preview_df)\n\n\n# ------------------------------------------------------------\n# 16. SAVE TEST PREDICTIONS\n# ------------------------------------------------------------\n\nPRED_PATH = (\n \"/kaggle/working/\"\n \"step20_test_predictions.csv\"\n)\n\npreview_df.to_csv(\n PRED_PATH,\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(PRED_PATH)\n\n\n# ------------------------------------------------------------\n# FINAL\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 20 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Step 18 checkpoint inspected\")\nprint(\"āœ… Test MRI features loaded\")\nprint(\"āœ… Test UIDs loaded\")\nprint(\"āœ… MRI tensor prepared\")\nprint(\"āœ… 12-target prediction generated\")\nprint(\"āœ… Predictions saved\")\n\nprint(\"\\nšŸš« DO NOT CREATE SUBMISSION YET.\")\nprint(\"Send me the COMPLETE output of this cell.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 21 ā€” SUBMISSION FORMAT + PREDICTION SANITY CHECK\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nprint(\"=\" * 70)\nprint(\"STEP 21 ā€” SUBMISSION FORMAT + PREDICTION SANITY CHECK\")\nprint(\"=\" * 70)\n\nDATA_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection\"\n\nTEST_CSV = os.path.join(DATA_PATH, \"test.csv\")\nSAMPLE_CSV = os.path.join(DATA_PATH, \"sample_submission.csv\")\nPRED_CSV = \"/kaggle/working/step20_test_predictions.csv\"\n\n# ------------------------------------------------------------\n# 1. CHECK FILES\n# ------------------------------------------------------------\n\nprint(\"\\n[1] Checking files...\")\n\nfor path in [TEST_CSV, SAMPLE_CSV, PRED_CSV]:\n\n print(\n \"āœ…\" if os.path.exists(path) else \"āŒ\",\n path\n )\n\n if not os.path.exists(path):\n raise FileNotFoundError(path)\n\n\n# ------------------------------------------------------------\n# 2. LOAD FILES\n# ------------------------------------------------------------\n\ntest_df = pd.read_csv(TEST_CSV)\nsample_df = pd.read_csv(SAMPLE_CSV)\npred_df = pd.read_csv(PRED_CSV)\n\nprint(\"\\n[2] File shapes\")\n\nprint(\"test.csv :\", test_df.shape)\nprint(\"sample_submission.csv :\", sample_df.shape)\nprint(\"step20 predictions :\", pred_df.shape)\n\n\n# ------------------------------------------------------------\n# 3. DISPLAY COLUMNS\n# ------------------------------------------------------------\n\nprint(\"\\n[3] Test columns\")\nprint(test_df.columns.tolist())\n\nprint(\"\\nSample submission columns\")\nprint(sample_df.columns.tolist())\n\nprint(\"\\nStep 20 prediction columns\")\nprint(pred_df.columns.tolist())\n\n\n# ------------------------------------------------------------\n# 4. TEST UID CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[4] UID CHECK\")\n\ntest_uids = test_df[\"StudyInstanceUID\"].astype(str).tolist()\npred_uids = pred_df[\"StudyInstanceUID\"].astype(str).tolist()\n\nprint(\"Test UIDs:\", len(test_uids))\nprint(\"Prediction UIDs:\", len(pred_uids))\n\nmissing_uids = set(test_uids) - set(pred_uids)\nextra_uids = set(pred_uids) - set(test_uids)\n\nprint(\"Missing prediction UIDs:\", len(missing_uids))\nprint(\"Extra prediction UIDs:\", len(extra_uids))\n\nif len(missing_uids) == 0 and len(extra_uids) == 0:\n print(\"āœ… All test UIDs present\")\n\nelse:\n print(\"āŒ UID mismatch\")\n\n if missing_uids:\n print(\"Missing:\", list(missing_uids))\n\n if extra_uids:\n print(\"Extra:\", list(extra_uids))\n\n\n# ------------------------------------------------------------\n# 5. TARGET CHECK\n# ------------------------------------------------------------\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\"\\n[5] TARGET CHECK\")\n\nmissing_targets = [\n c for c in target_cols\n if c not in pred_df.columns\n]\n\nif len(missing_targets) == 0:\n print(\"āœ… All 12 target columns present\")\n\nelse:\n print(\"āŒ Missing targets:\", missing_targets)\n\n\n# ------------------------------------------------------------\n# 6. PREDICTION MATRIX\n# ------------------------------------------------------------\n\npred_matrix = pred_df[target_cols].to_numpy(\n dtype=np.float32\n)\n\nprint(\"\\n[6] Prediction matrix\")\nprint(\"Shape:\", pred_matrix.shape)\n\nassert pred_matrix.shape == (3, 12)\n\nprint(\"āœ… Expected shape (3, 12)\")\n\n\n# ------------------------------------------------------------\n# 7. NaN / INF CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[7] Numerical sanity check\")\n\nprint(\"NaN values :\", np.isnan(pred_matrix).sum())\nprint(\"Inf values :\", np.isinf(pred_matrix).sum())\n\nif np.isnan(pred_matrix).sum() == 0:\n print(\"āœ… No NaN\")\n\nif np.isinf(pred_matrix).sum() == 0:\n print(\"āœ… No Inf\")\n\n\n# ------------------------------------------------------------\n# 8. RANGE CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[8] Prediction range\")\n\nprint(\"Minimum:\", pred_matrix.min())\nprint(\"Maximum:\", pred_matrix.max())\n\nif (\n pred_matrix.min() >= 0\n and pred_matrix.max() <= 1\n):\n print(\"āœ… All predictions are valid probabilities\")\n\nelse:\n print(\"āŒ Predictions outside [0,1]\")\n\n\n# ------------------------------------------------------------\n# 9. PER-TARGET STATISTICS\n# ------------------------------------------------------------\n\nprint(\"\\n[9] Per-target prediction statistics\")\n\nstats_df = pd.DataFrame({\n \"Target\": target_cols,\n \"Mean\": pred_matrix.mean(axis=0),\n \"Min\": pred_matrix.min(axis=0),\n \"Max\": pred_matrix.max(axis=0),\n \"Std\": pred_matrix.std(axis=0)\n})\n\ndisplay(stats_df)\n\n\n# ------------------------------------------------------------\n# 10. CHECK NEAR-CONSTANT PREDICTIONS\n# ------------------------------------------------------------\n\nprint(\"\\n[10] Checking prediction variation\")\n\noverall_std = pred_matrix.std()\n\nprint(\"Overall prediction STD:\", overall_std)\n\nif overall_std < 0.01:\n\n print(\"\"\"\nāš ļø WARNING:\nPredictions are extremely close to each other.\n\nThis may indicate that the zero text vector used in\nStep 20 is causing the multimodal model to produce\nnearly constant predictions.\n\"\"\")\n\nelse:\n\n print(\"āœ… Predictions show variation\")\n\n\n# ------------------------------------------------------------\n# 11. SAMPLE SUBMISSION FORMAT\n# ------------------------------------------------------------\n\nprint(\"\\n[11] SAMPLE SUBMISSION PREVIEW\")\n\ndisplay(sample_df.head())\n\nprint(\"\\nSample submission shape:\", sample_df.shape)\n\n\n# ------------------------------------------------------------\n# 12. CHECK WHETHER SAMPLE USES SAME UID\n# ------------------------------------------------------------\n\nprint(\"\\n[12] Submission ID check\")\n\nprint(\"First sample IDs:\")\nprint(\n sample_df.iloc[:5, 0].tolist()\n)\n\nprint(\"\\nFirst test UIDs:\")\nprint(\n test_df[\"StudyInstanceUID\"].head().tolist()\n)\n\n\n# ------------------------------------------------------------\n# 13. FINAL STEP 21 STATUS\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 21 CHECK COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Test file checked\")\nprint(\"āœ… Sample submission checked\")\nprint(\"āœ… Step 20 predictions loaded\")\nprint(\"āœ… UID alignment checked\")\nprint(\"āœ… 12 targets checked\")\nprint(\"āœ… NaN / Inf checked\")\nprint(\"āœ… Probability range checked\")\nprint(\"āœ… Prediction variation checked\")\n\nprint(\"\"\"\nšŸš« DO NOT CREATE FINAL SUBMISSION YET.\n\nSend me the COMPLETE output of this cell.\n\"\"\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 22 ā€” FINAL TEST PREDICTION REFINEMENT\n# MRI-ONLY vs MULTIMODAL SANITY CHECK\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\n\nprint(\"=\" * 70)\nprint(\"STEP 22 ā€” FINAL TEST PREDICTION REFINEMENT\")\nprint(\"=\" * 70)\n\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Device:\", DEVICE)\n\nMODEL_PATH = \"/kaggle/working/step18_multimodal_best.pth\"\nMRI_PATH = \"/kaggle/working/step19_test_mri_features.npz\"\nSTEP20_PATH = \"/kaggle/working/step20_test_predictions.csv\"\n\nTARGET_COLS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n# ============================================================\n# 1. CHECK FILES\n# ============================================================\n\nprint(\"\\n[1] Checking required files...\")\n\nfor path in [MODEL_PATH, MRI_PATH, STEP20_PATH]:\n print(\n \"āœ…\" if os.path.exists(path) else \"āŒ\",\n path\n )\n\n if not os.path.exists(path):\n raise FileNotFoundError(path)\n\n\n# ============================================================\n# 2. LOAD TEST MRI FEATURES\n# ============================================================\n\nprint(\"\\n[2] Loading test MRI features...\")\n\ndata = np.load(\n MRI_PATH,\n allow_pickle=True\n)\n\ntest_uids = data[\"StudyInstanceUID\"].astype(str)\nmri_features = data[\"MRI_features\"].astype(np.float32)\n\nprint(\"UIDs:\", len(test_uids))\nprint(\"MRI features:\", mri_features.shape)\n\nassert mri_features.shape == (len(test_uids), 512)\n\nprint(\"āœ… MRI features verified\")\n\n\n# ============================================================\n# 3. LOAD STEP 20 PREDICTIONS\n# ============================================================\n\nstep20_df = pd.read_csv(STEP20_PATH)\n\nstep20_pred = step20_df[TARGET_COLS].to_numpy(\n dtype=np.float32\n)\n\nprint(\"\\n[3] Step 20 predictions:\")\nprint(\"Shape:\", step20_pred.shape)\n\nprint(\n \"Mean:\",\n step20_pred.mean()\n)\n\nprint(\n \"STD:\",\n step20_pred.std()\n)\n\n\n# ============================================================\n# 4. LOAD STEP 18 CHECKPOINT\n# ============================================================\n\nprint(\"\\n[4] Loading Step 18 checkpoint...\")\n\ncheckpoint = torch.load(\n MODEL_PATH,\n map_location=DEVICE\n)\n\nprint(\"Checkpoint keys:\")\nprint(list(checkpoint.keys()))\n\nstate_dict = checkpoint[\"model_state_dict\"]\n\nmri_dim = int(checkpoint.get(\"mri_dim\", 512))\ntext_dim = int(checkpoint.get(\"text_dim\", 1500))\nnum_targets = int(checkpoint.get(\"num_targets\", 12))\n\nprint(\"\\nMRI dimension:\", mri_dim)\nprint(\"Text dimension:\", text_dim)\nprint(\"Targets:\", num_targets)\n\nassert mri_dim == 512\nassert text_dim == 1500\nassert num_targets == 12\n\n\n# ============================================================\n# 5. DEFINE EXACT STEP 18 ARCHITECTURE\n# ============================================================\n\nprint(\"\\n[5] Reconstructing Step 18 architecture...\")\n\n\nclass Step18MultimodalModel(nn.Module):\n\n def __init__(\n self,\n mri_dim=512,\n text_dim=1500,\n num_targets=12\n ):\n super().__init__()\n\n self.mri_branch = nn.Sequential(\n nn.Linear(mri_dim, 256),\n nn.ReLU(),\n nn.Dropout(0.30),\n nn.Linear(256, 128),\n nn.ReLU()\n )\n\n self.text_branch = nn.Sequential(\n nn.Linear(text_dim, 512),\n nn.ReLU(),\n nn.Dropout(0.30),\n nn.Linear(512, 128),\n nn.ReLU()\n )\n\n self.fusion = nn.Sequential(\n nn.Linear(256, 256),\n nn.ReLU(),\n nn.Dropout(0.30),\n nn.Linear(256, 128),\n nn.ReLU(),\n nn.Dropout(0.20),\n nn.Linear(128, num_targets)\n )\n\n def forward(\n self,\n mri,\n text\n ):\n\n mri_x = self.mri_branch(mri)\n text_x = self.text_branch(text)\n\n x = torch.cat(\n [mri_x, text_x],\n dim=1\n )\n\n return self.fusion(x)\n\n\nmodel = Step18MultimodalModel(\n mri_dim=mri_dim,\n text_dim=text_dim,\n num_targets=num_targets\n).to(DEVICE)\n\n\n# ============================================================\n# 6. LOAD WEIGHTS\n# ============================================================\n\nmissing, unexpected = model.load_state_dict(\n state_dict,\n strict=False\n)\n\nprint(\"Missing keys:\", len(missing))\nprint(\"Unexpected keys:\", len(unexpected))\n\nif len(missing) == 0 and len(unexpected) == 0:\n print(\"āœ… Exact Step 18 weights loaded\")\n\nelse:\n print(\"āš ļø Weight mismatch detected\")\n\nmodel.eval()\n\n\n# ============================================================\n# 7. PREPARE MRI\n# ============================================================\n\nmri_tensor = torch.tensor(\n mri_features,\n dtype=torch.float32,\n device=DEVICE\n)\n\nprint(\"\\nMRI tensor:\", mri_tensor.shape)\n\n\n# ============================================================\n# 8. MULTIMODAL ZERO-TEXT PREDICTION\n# ============================================================\n\nprint(\"\\n[6] Multimodal prediction with missing test text...\")\n\nzero_text = torch.zeros(\n (len(test_uids), text_dim),\n dtype=torch.float32,\n device=DEVICE\n)\n\nwith torch.no_grad():\n\n multimodal_logits = model(\n mri_tensor,\n zero_text\n )\n\n multimodal_pred = torch.sigmoid(\n multimodal_logits\n ).cpu().numpy()\n\n\nprint(\n \"Multimodal prediction:\",\n multimodal_pred.shape\n)\n\nprint(\n \"Mean:\",\n multimodal_pred.mean()\n)\n\nprint(\n \"STD:\",\n multimodal_pred.std()\n)\n\n\n# ============================================================\n# 9. MRI-ONLY APPROXIMATION\n# ============================================================\n\nprint(\"\\n[7] MRI-only prediction analysis...\")\n\n# The fusion network still expects a text branch.\n# We therefore calculate the MRI contribution and inspect\n# whether MRI features themselves have meaningful variation.\n\nwith torch.no_grad():\n\n mri_emb = model.mri_branch(\n mri_tensor\n )\n\nprint(\n \"MRI embedding shape:\",\n tuple(mri_emb.shape)\n)\n\nprint(\n \"MRI embedding mean:\",\n mri_emb.mean().item()\n)\n\nprint(\n \"MRI embedding STD:\",\n mri_emb.std().item()\n)\n\n\n# ============================================================\n# 10. COMPARE STEP 20 WITH RECOMPUTED PREDICTION\n# ============================================================\n\ndifference = np.abs(\n step20_pred - multimodal_pred\n)\n\nprint(\"\\n[8] Step 20 vs recomputed prediction\")\n\nprint(\n \"Mean absolute difference:\",\n difference.mean()\n)\n\nprint(\n \"Maximum absolute difference:\",\n difference.max()\n)\n\nif difference.mean() < 1e-5:\n print(\"āœ… Step 20 prediction reproduced\")\n\nelse:\n print(\"āš ļø Predictions differ\")\n\n\n# ============================================================\n# 11. PER-TARGET STATISTICS\n# ============================================================\n\nprint(\"\\n[9] Final prediction statistics\")\n\nstats = pd.DataFrame({\n\n \"Target\": TARGET_COLS,\n\n \"Step20_Mean\":\n step20_pred.mean(axis=0),\n\n \"Step20_STD\":\n step20_pred.std(axis=0),\n\n \"New_Mean\":\n multimodal_pred.mean(axis=0),\n\n \"New_STD\":\n multimodal_pred.std(axis=0),\n\n \"Mean_Abs_Diff\":\n difference.mean(axis=0)\n\n})\n\ndisplay(stats)\n\n\n# ============================================================\n# 12. UID-BASED FINAL CHECK\n# ============================================================\n\nprint(\"\\n[10] UID verification\")\n\nprint(\n \"MRI UIDs:\",\n len(test_uids)\n)\n\nprint(\n \"Step 20 UIDs:\",\n len(step20_df)\n)\n\nif set(test_uids) == set(\n step20_df[\"StudyInstanceUID\"].astype(str)\n):\n\n print(\"āœ… Test UIDs match\")\n\nelse:\n\n raise RuntimeError(\n \"Test UID mismatch detected.\"\n )\n\n\n# ============================================================\n# 13. SAVE REFINED PREDICTIONS\n# ============================================================\n\nrefined_df = pd.DataFrame(\n multimodal_pred,\n columns=TARGET_COLS\n)\n\nrefined_df.insert(\n 0,\n \"StudyInstanceUID\",\n test_uids\n)\n\nREFINED_PATH = (\n \"/kaggle/working/\"\n \"step22_refined_test_predictions.csv\"\n)\n\nrefined_df.to_csv(\n REFINED_PATH,\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(REFINED_PATH)\n\n\n# ============================================================\n# FINAL STATUS\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 22 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Step 18 architecture reconstructed\")\nprint(\"āœ… Step 18 weights loaded\")\nprint(\"āœ… Test MRI features loaded\")\nprint(\"āœ… Multimodal test prediction regenerated\")\nprint(\"āœ… Prediction statistics checked\")\nprint(\"āœ… UID alignment verified\")\nprint(\"āœ… Refined predictions saved\")\n\nprint(\"\\nšŸš« DO NOT CREATE FINAL SUBMISSION YET.\")\nprint(\"Send me the COMPLETE output of this cell.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 23 ā€” FINAL SUBMISSION CREATION + VALIDATION\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nprint(\"=\" * 70)\nprint(\"STEP 23 ā€” FINAL SUBMISSION PREPARATION\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. PATHS\n# ------------------------------------------------------------\n\nTEST_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection/test.csv\"\nSAMPLE_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection/sample_submission.csv\"\nPRED_PATH = \"/kaggle/working/step22_test_predictions.csv\"\n\n# If Step 22 saved using another filename, try Step 20\nif not os.path.exists(PRED_PATH):\n PRED_PATH = \"/kaggle/working/step20_test_predictions.csv\"\n\nSUBMISSION_PATH = \"/kaggle/working/submission.csv\"\n\n\n# ------------------------------------------------------------\n# 2. CHECK FILES\n# ------------------------------------------------------------\n\nprint(\"\\n[1] Checking files...\")\n\nfor path in [TEST_PATH, SAMPLE_PATH, PRED_PATH]:\n print(\n \"āœ…\" if os.path.exists(path) else \"āŒ\",\n path\n )\n\nif not os.path.exists(TEST_PATH):\n raise FileNotFoundError(\"test.csv not found.\")\n\nif not os.path.exists(SAMPLE_PATH):\n raise FileNotFoundError(\"sample_submission.csv not found.\")\n\nif not os.path.exists(PRED_PATH):\n raise FileNotFoundError(\n \"Step 22/20 prediction file not found.\"\n )\n\n\n# ------------------------------------------------------------\n# 3. LOAD FILES\n# ------------------------------------------------------------\n\nprint(\"\\n[2] Loading files...\")\n\ntest_df = pd.read_csv(TEST_PATH)\nsample_df = pd.read_csv(SAMPLE_PATH)\npred_df = pd.read_csv(PRED_PATH)\n\nprint(\"Test shape:\", test_df.shape)\nprint(\"Sample submission shape:\", sample_df.shape)\nprint(\"Prediction shape:\", pred_df.shape)\n\nprint(\"\\nTest columns:\")\nprint(test_df.columns.tolist())\n\nprint(\"\\nSample columns:\")\nprint(sample_df.columns.tolist())\n\nprint(\"\\nPrediction columns:\")\nprint(pred_df.columns.tolist())\n\n\n# ------------------------------------------------------------\n# 4. TARGET COLUMNS\n# ------------------------------------------------------------\n\ntarget_cols = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nprint(\"\\n[3] Target validation\")\n\nmissing_targets = [\n c for c in target_cols\n if c not in pred_df.columns\n]\n\nif missing_targets:\n raise ValueError(\n f\"Missing prediction columns: {missing_targets}\"\n )\n\nprint(\"āœ… All 12 target columns found\")\n\n\n# ------------------------------------------------------------\n# 5. UID CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[4] UID validation\")\n\nif \"StudyInstanceUID\" not in test_df.columns:\n raise ValueError(\n \"StudyInstanceUID missing from test.csv\"\n )\n\nif \"StudyInstanceUID\" not in pred_df.columns:\n raise ValueError(\n \"StudyInstanceUID missing from prediction file\"\n )\n\nprint(\"Test UIDs:\", len(test_df))\nprint(\"Prediction UIDs:\", len(pred_df))\n\nif test_df[\"StudyInstanceUID\"].duplicated().any():\n raise ValueError(\n \"Duplicate StudyInstanceUID found in test.csv\"\n )\n\nif pred_df[\"StudyInstanceUID\"].duplicated().any():\n raise ValueError(\n \"Duplicate StudyInstanceUID found in predictions\"\n )\n\nprint(\"āœ… No duplicate UIDs\")\n\n\n# ------------------------------------------------------------\n# 6. EXACT UID ALIGNMENT\n# ------------------------------------------------------------\n\nprint(\"\\n[5] Aligning predictions to test.csv order\")\n\ntest_uids = test_df[\"StudyInstanceUID\"].astype(str)\n\npred_df[\"StudyInstanceUID\"] = (\n pred_df[\"StudyInstanceUID\"].astype(str)\n)\n\nmissing_uids = set(test_uids) - set(\n pred_df[\"StudyInstanceUID\"]\n)\n\nextra_uids = set(pred_df[\"StudyInstanceUID\"]) - set(\n test_uids\n)\n\nif missing_uids:\n raise ValueError(\n f\"Missing test UIDs in predictions: {missing_uids}\"\n )\n\nif extra_uids:\n raise ValueError(\n f\"Extra UIDs in predictions: {extra_uids}\"\n )\n\n# Reorder according to test.csv\npred_df = (\n pred_df\n .set_index(\"StudyInstanceUID\")\n .loc[test_uids]\n .reset_index()\n)\n\nprint(\"āœ… UID alignment verified\")\nprint(\"Aligned predictions:\", len(pred_df))\n\n\n# ------------------------------------------------------------\n# 7. NUMERIC / NaN / INF CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[6] Checking prediction values\")\n\nprediction_values = pred_df[target_cols].to_numpy(\n dtype=np.float32\n)\n\nprint(\"Prediction matrix:\", prediction_values.shape)\nprint(\"Min:\", float(np.min(prediction_values)))\nprint(\"Max:\", float(np.max(prediction_values)))\n\nif np.isnan(prediction_values).any():\n raise ValueError(\n \"āŒ NaN values found in predictions\"\n )\n\nif np.isinf(prediction_values).any():\n raise ValueError(\n \"āŒ Inf values found in predictions\"\n )\n\nprint(\"āœ… No NaN values\")\nprint(\"āœ… No Inf values\")\n\n\n# ------------------------------------------------------------\n# 8. PROBABILITY RANGE\n# ------------------------------------------------------------\n\nprint(\"\\n[7] Checking probability range\")\n\nif (\n np.min(prediction_values) < 0\n or np.max(prediction_values) > 1\n):\n raise ValueError(\n \"Predictions are outside [0, 1]\"\n )\n\nprint(\"āœ… All predictions are valid probabilities [0, 1]\")\n\n\n# ------------------------------------------------------------\n# 9. CHECK SAMPLE SUBMISSION STRUCTURE\n# ------------------------------------------------------------\n\nprint(\"\\n[8] Checking sample_submission structure\")\n\nsample_uid_col = sample_df.columns[0]\n\nprint(\"Sample UID column:\", sample_uid_col)\n\nif sample_uid_col != \"StudyInstanceUID\":\n print(\n \"āš ļø Sample first column differs:\",\n sample_uid_col\n )\n\nsample_targets = [\n c for c in sample_df.columns\n if c != \"StudyInstanceUID\"\n]\n\nprint(\"Sample target count:\", len(sample_targets))\n\nif len(sample_targets) != 12:\n raise ValueError(\n f\"Expected 12 target columns, got {len(sample_targets)}\"\n )\n\nprint(\"Sample target columns:\")\nprint(sample_targets)\n\n\n# ------------------------------------------------------------\n# 10. CREATE FINAL SUBMISSION\n# ------------------------------------------------------------\n\nprint(\"\\n[9] Creating final submission\")\n\n# IMPORTANT:\n# Use sample_submission column order exactly.\n\nsubmission = pd.DataFrame()\n\nsubmission[\"StudyInstanceUID\"] = pred_df[\n \"StudyInstanceUID\"\n]\n\nfor col in sample_targets:\n\n if col not in pred_df.columns:\n raise ValueError(\n f\"Target {col} missing from predictions\"\n )\n\n submission[col] = pred_df[col].astype(float)\n\n\n# ------------------------------------------------------------\n# 11. FINAL STRUCTURE CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[10] Final submission validation\")\n\nprint(\"Submission shape:\", submission.shape)\n\nprint(\"\\nSubmission columns:\")\nprint(submission.columns.tolist())\n\nexpected_columns = sample_df.columns.tolist()\n\nif submission.columns.tolist() != expected_columns:\n raise ValueError(\n \"Submission columns do not exactly match \"\n \"sample_submission.csv\"\n )\n\nif len(submission) != len(sample_df):\n raise ValueError(\n \"Submission row count does not match sample submission\"\n )\n\nif submission[\"StudyInstanceUID\"].tolist() != \\\n sample_df[\"StudyInstanceUID\"].astype(str).tolist():\n\n print(\n \"āš ļø Sample UID order differs from test order.\"\n )\n\n # Reorder exactly like sample submission\n submission = (\n submission\n .set_index(\"StudyInstanceUID\")\n .loc[\n sample_df[\"StudyInstanceUID\"].astype(str)\n ]\n .reset_index()\n )\n\nprint(\"āœ… Column order verified\")\nprint(\"āœ… Row count verified\")\nprint(\"āœ… UID structure verified\")\n\n\n# ------------------------------------------------------------\n# 12. FINAL NaN CHECK\n# ------------------------------------------------------------\n\nif submission.isna().any().any():\n raise ValueError(\n \"āŒ Final submission contains NaN\"\n )\n\nprint(\"āœ… Final submission contains no NaN\")\n\n\n# ------------------------------------------------------------\n# 13. SAVE\n# ------------------------------------------------------------\n\nsubmission.to_csv(\n SUBMISSION_PATH,\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(SUBMISSION_PATH)\n\n\n# ------------------------------------------------------------\n# 14. RELOAD AND FINAL VERIFY\n# ------------------------------------------------------------\n\nfinal_check = pd.read_csv(\n SUBMISSION_PATH\n)\n\nprint(\"\\n[11] Reloaded submission\")\nprint(\"Shape:\", final_check.shape)\n\nprint(\"\\nPreview:\")\ndisplay(final_check)\n\n\n# ------------------------------------------------------------\n# 15. FINAL CHECKS\n# ------------------------------------------------------------\n\nassert len(final_check) == len(sample_df)\nassert final_check.columns.tolist() == sample_df.columns.tolist()\nassert not final_check.isna().any().any()\n\nfinal_values = final_check[target_cols].to_numpy(\n dtype=np.float32\n)\n\nassert np.isfinite(final_values).all()\nassert (final_values >= 0).all()\nassert (final_values <= 1).all()\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 23 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… Step 22 predictions loaded\")\nprint(\"āœ… Test UIDs validated\")\nprint(\"āœ… 12 target columns validated\")\nprint(\"āœ… Prediction range validated\")\nprint(\"āœ… NaN / Inf check passed\")\nprint(\"āœ… Sample submission structure matched\")\nprint(\"āœ… Final submission created\")\nprint()\nprint(\"FINAL FILE:\")\nprint(SUBMISSION_PATH)\n\nprint(\"\\nšŸš€ FINAL SUBMISSION IS READY FOR REVIEW.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 24 ā€” FINAL SUBMISSION AUDIT\n# ============================================================\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nprint(\"=\" * 70)\nprint(\"STEP 24 ā€” FINAL SUBMISSION AUDIT\")\nprint(\"=\" * 70)\n\n# ------------------------------------------------------------\n# 1. PATHS\n# ------------------------------------------------------------\n\nSUBMISSION_PATH = \"/kaggle/working/submission.csv\"\nTEST_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection/test.csv\"\nSAMPLE_PATH = \"/kaggle/input/competitions/rsna-knee-abnormality-detection/sample_submission.csv\"\n\nTARGET_COLS = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\n# ------------------------------------------------------------\n# 2. FILE EXISTENCE\n# ------------------------------------------------------------\n\nprint(\"\\n[1] FILE CHECK\")\n\nfor path in [\n SUBMISSION_PATH,\n TEST_PATH,\n SAMPLE_PATH\n]:\n print(\n \"āœ…\" if os.path.exists(path) else \"āŒ\",\n path\n )\n\nif not os.path.exists(SUBMISSION_PATH):\n raise FileNotFoundError(\n \"submission.csv not found.\"\n )\n\n# ------------------------------------------------------------\n# 3. LOAD FILES\n# ------------------------------------------------------------\n\nprint(\"\\n[2] LOADING FILES\")\n\nsubmission = pd.read_csv(SUBMISSION_PATH)\ntest_df = pd.read_csv(TEST_PATH)\nsample_df = pd.read_csv(SAMPLE_PATH)\n\nprint(\"Submission shape:\", submission.shape)\nprint(\"Test shape:\", test_df.shape)\nprint(\"Sample shape:\", sample_df.shape)\n\n# ------------------------------------------------------------\n# 4. COLUMN CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[3] COLUMN CHECK\")\n\nexpected_columns = sample_df.columns.tolist()\nactual_columns = submission.columns.tolist()\n\nprint(\"Expected columns:\")\nprint(expected_columns)\n\nprint(\"\\nActual columns:\")\nprint(actual_columns)\n\nif actual_columns != expected_columns:\n raise ValueError(\n \"āŒ Submission columns do not exactly match sample_submission.csv\"\n )\n\nprint(\"āœ… Column structure EXACTLY matches sample submission\")\n\n# ------------------------------------------------------------\n# 5. ROW COUNT\n# ------------------------------------------------------------\n\nprint(\"\\n[4] ROW COUNT CHECK\")\n\nprint(\"Test rows:\", len(test_df))\nprint(\"Sample rows:\", len(sample_df))\nprint(\"Submission rows:\", len(submission))\n\nif len(submission) != len(test_df):\n raise ValueError(\n \"āŒ Submission row count does not match test.csv\"\n )\n\nif len(submission) != len(sample_df):\n raise ValueError(\n \"āŒ Submission row count does not match sample_submission.csv\"\n )\n\nprint(\"āœ… Row counts match\")\n\n# ------------------------------------------------------------\n# 6. UID CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[5] UID CHECK\")\n\ntest_uids = test_df[\"StudyInstanceUID\"].astype(str)\nsubmission_uids = submission[\"StudyInstanceUID\"].astype(str)\n\nprint(\"Test UID count:\", len(test_uids))\nprint(\"Submission UID count:\", len(submission_uids))\n\nif submission_uids.duplicated().any():\n raise ValueError(\n \"āŒ Duplicate StudyInstanceUID in submission\"\n )\n\nif len(set(test_uids)) != len(set(submission_uids)):\n raise ValueError(\n \"āŒ Submission UID set does not match test UID set\"\n )\n\nif set(test_uids) != set(submission_uids):\n raise ValueError(\n \"āŒ Submission contains different UIDs\"\n )\n\nprint(\"āœ… All test UIDs present\")\nprint(\"āœ… No duplicate UIDs\")\n\n# ------------------------------------------------------------\n# 7. UID ORDER CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[6] UID ORDER CHECK\")\n\nif test_uids.tolist() == submission_uids.tolist():\n print(\"āœ… Submission UID order matches test.csv\")\nelse:\n print(\n \"āš ļø Submission UID order differs from test.csv\"\n )\n\n # Reorder to test.csv order\n submission = (\n submission\n .set_index(\"StudyInstanceUID\")\n .loc[test_uids]\n .reset_index()\n )\n\n submission.to_csv(\n SUBMISSION_PATH,\n index=False\n )\n\n print(\"āœ… Submission reordered to test.csv order\")\n\n# ------------------------------------------------------------\n# 8. TARGET CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[7] TARGET CHECK\")\n\nmissing_targets = [\n col for col in TARGET_COLS\n if col not in submission.columns\n]\n\nif missing_targets:\n raise ValueError(\n f\"āŒ Missing target columns: {missing_targets}\"\n )\n\nif len(TARGET_COLS) != 12:\n raise ValueError(\n \"Target list does not contain 12 abnormalities\"\n )\n\nprint(\"āœ… All 12 abnormalities present\")\n\n# ------------------------------------------------------------\n# 9. NUMERIC CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[8] NUMERIC CHECK\")\n\nfor col in TARGET_COLS:\n submission[col] = pd.to_numeric(\n submission[col],\n errors=\"coerce\"\n )\n\nif submission[TARGET_COLS].isna().any().any():\n raise ValueError(\n \"āŒ Non-numeric / NaN values found\"\n )\n\nprint(\"āœ… All prediction values are numeric\")\n\n# ------------------------------------------------------------\n# 10. NaN / INF CHECK\n# ------------------------------------------------------------\n\nprint(\"\\n[9] NaN / INF CHECK\")\n\nvalues = submission[TARGET_COLS].to_numpy(\n dtype=np.float32\n)\n\nprint(\"NaN count:\", np.isnan(values).sum())\nprint(\"Inf count:\", np.isinf(values).sum())\n\nif np.isnan(values).any():\n raise ValueError(\"āŒ NaN detected\")\n\nif np.isinf(values).any():\n raise ValueError(\"āŒ Inf detected\")\n\nprint(\"āœ… No NaN\")\nprint(\"āœ… No Inf\")\n\n# ------------------------------------------------------------\n# 11. PROBABILITY RANGE\n# ------------------------------------------------------------\n\nprint(\"\\n[10] PROBABILITY RANGE\")\n\nminimum = float(values.min())\nmaximum = float(values.max())\n\nprint(\"Minimum prediction:\", minimum)\nprint(\"Maximum prediction:\", maximum)\n\nif minimum < 0:\n raise ValueError(\n \"āŒ Prediction below 0 detected\"\n )\n\nif maximum > 1:\n raise ValueError(\n \"āŒ Prediction above 1 detected\"\n )\n\nprint(\"āœ… All probabilities are within [0, 1]\")\n\n# ------------------------------------------------------------\n# 12. PREDICTION VARIATION\n# ------------------------------------------------------------\n\nprint(\"\\n[11] PREDICTION VARIATION\")\n\nunique_values = np.unique(values)\n\nprint(\"Unique prediction values:\", len(unique_values))\n\nif len(unique_values) <= 1:\n raise ValueError(\n \"āŒ All predictions are identical\"\n )\n\nprint(\"Per-target statistics:\")\n\nstats = pd.DataFrame({\n \"Target\": TARGET_COLS,\n \"Min\": values.min(axis=0),\n \"Max\": values.max(axis=0),\n \"Mean\": values.mean(axis=0),\n \"Std\": values.std(axis=0)\n})\n\ndisplay(stats)\n\nprint(\"āœ… Prediction variation confirmed\")\n\n# ------------------------------------------------------------\n# 13. SAMPLE SUBMISSION COMPARISON\n# ------------------------------------------------------------\n\nprint(\"\\n[12] SAMPLE SUBMISSION COMPARISON\")\n\nif submission.columns.tolist() != sample_df.columns.tolist():\n raise ValueError(\n \"āŒ Column order mismatch with sample submission\"\n )\n\nprint(\"Sample columns:\", len(sample_df.columns))\nprint(\"Submission columns:\", len(submission.columns))\n\nprint(\"āœ… Exact column structure confirmed\")\n\n# ------------------------------------------------------------\n# 14. SAVE FINAL AUDITED FILE\n# ------------------------------------------------------------\n\nsubmission.to_csv(\n SUBMISSION_PATH,\n index=False\n)\n\n# ------------------------------------------------------------\n# 15. RELOAD FINAL FILE\n# ------------------------------------------------------------\n\nprint(\"\\n[13] FINAL RELOAD CHECK\")\n\nfinal_submission = pd.read_csv(\n SUBMISSION_PATH\n)\n\nprint(\"Final file shape:\", final_submission.shape)\n\nassert final_submission.shape == submission.shape\nassert final_submission.columns.tolist() == expected_columns\nassert not final_submission.isna().any().any()\n\nfinal_values = final_submission[\n TARGET_COLS\n].to_numpy(dtype=np.float32)\n\nassert np.isfinite(final_values).all()\nassert final_values.min() >= 0\nassert final_values.max() <= 1\n\nprint(\"āœ… Final reload successful\")\nprint(\"āœ… Final CSV is valid\")\n\n# ------------------------------------------------------------\n# 16. PREVIEW\n# ------------------------------------------------------------\n\nprint(\"\\n[14] FINAL SUBMISSION PREVIEW\")\n\ndisplay(final_submission)\n\n# ------------------------------------------------------------\n# FINAL\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"STEP 24 COMPLETE\")\nprint(\"=\" * 70)\n\nprint(\"āœ… submission.csv exists\")\nprint(\"āœ… 3 test studies verified\")\nprint(\"āœ… 12 abnormalities verified\")\nprint(\"āœ… UID set verified\")\nprint(\"āœ… UID order verified\")\nprint(\"āœ… Column structure verified\")\nprint(\"āœ… Numeric values verified\")\nprint(\"āœ… NaN check passed\")\nprint(\"āœ… Inf check passed\")\nprint(\"āœ… Probability range [0,1] verified\")\nprint(\"āœ… Prediction variation verified\")\nprint(\"āœ… Sample submission structure matched\")\nprint()\nprint(\"FINAL FILE:\")\nprint(SUBMISSION_PATH)\n\nprint(\"\\nšŸš€ READY FOR FINAL KAGGLE SUBMISSION.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import FileLink\n\nFileLink('/kaggle/working/submission.csv')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\npath = \"/kaggle/working/submission.csv\"\n\nprint(\"Exists:\", os.path.exists(path))\nprint(\"Size:\", os.path.getsize(path), \"bytes\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\nshutil.make_archive(\n \"/kaggle/working/submission\",\n \"zip\",\n \"/kaggle/working\",\n \"submission.csv\"\n)\n\nprint(\"Created: /kaggle/working/submission.zip\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nimport shutil\n\nTEST_SERIES = \"/kaggle/input/competitions/rsna-knee-abnormality-detection/test_series\"\nOUTPUT = \"/kaggle/working/sample_knee_mri.dcm\"\n\ndicom_files = glob.glob(\n os.path.join(TEST_SERIES, \"**\", \"*.dcm\"),\n recursive=True\n)\n\nprint(\"DICOM files found:\", len(dicom_files))\n\nif len(dicom_files) == 0:\n raise RuntimeError(\"āŒ No DICOM files found\")\n\nprint(\"Selected DICOM:\")\nprint(dicom_files[0])\n\nshutil.copy2(dicom_files[0], OUTPUT)\n\nprint(\"\\nāœ… DICOM copied successfully\")\nprint(\"File:\", OUTPUT)\nprint(\"Exists:\", os.path.exists(OUTPUT))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}