{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nprint(os.listdir('/kaggle/input'))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:34.217171Z","iopub.execute_input":"2026-09-12T08:15:34.217568Z","iopub.status.idle":"2026-09-12T08:15:34.229169Z","shell.execute_reply.started":"2026-09-12T08:15:34.217522Z","shell.execute_reply":"2026-09-12T08:15:34.228018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for item in os.listdir('/kaggle/input'):\n print(item)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:34.231285Z","iopub.execute_input":"2026-09-12T08:15:34.231567Z","iopub.status.idle":"2026-09-12T08:15:34.255402Z","shell.execute_reply.started":"2026-09-12T08:15:34.23154Z","shell.execute_reply":"2026-09-12T08:15:34.254209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_PATH = '/kaggle/input/competitions/rsna-knee-abnormality-detection'\n\nprint(os.listdir(BASE_PATH))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:34.256725Z","iopub.execute_input":"2026-09-12T08:15:34.257238Z","iopub.status.idle":"2026-09-12T08:15:34.278548Z","shell.execute_reply.started":"2026-09-12T08:15:34.25721Z","shell.execute_reply":"2026-09-12T08:15:34.27729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\ntrain = pd.read_csv(os.path.join(BASE_PATH, \"train.csv\"))\n\nprint(\"Shape:\", train.shape)\nprint(\"\\nColumns:\")\nprint(train.columns.tolist())\n\ndisplay(train.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:34.280418Z","iopub.execute_input":"2026-09-12T08:15:34.280921Z","iopub.status.idle":"2026-09-12T08:15:36.233436Z","shell.execute_reply.started":"2026-09-12T08:15:34.280885Z","shell.execute_reply":"2026-09-12T08:15:36.231616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.236538Z","iopub.execute_input":"2026-09-12T08:15:36.236925Z","iopub.status.idle":"2026-09-12T08:15:36.270761Z","shell.execute_reply.started":"2026-09-12T08:15:36.236885Z","shell.execute_reply":"2026-09-12T08:15:36.269352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train[['ACL', 'MCL', 'Medial Meniscus', 'Lateral Meniscus',\n 'Medial OA', 'Lateral OA', 'PF OA', 'Effusion',\n 'Synovitis', \"Baker's\", 'Contusion', 'Fracture']].notna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.275007Z","iopub.execute_input":"2026-09-12T08:15:36.275497Z","iopub.status.idle":"2026-09-12T08:15:36.286906Z","shell.execute_reply.started":"2026-09-12T08:15:36.275469Z","shell.execute_reply":"2026-09-12T08:15:36.285554Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train[train['ACL'].notna()].head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.288669Z","iopub.execute_input":"2026-09-12T08:15:36.289205Z","iopub.status.idle":"2026-09-12T08:15:36.325423Z","shell.execute_reply.started":"2026-09-12T08:15:36.289174Z","shell.execute_reply":"2026-09-12T08:15:36.324281Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series = pd.read_csv(\n os.path.join(BASE_PATH, \"train_series.csv\")\n)\n\nprint(\"Shape:\", train_series.shape)\nprint(\"Columns:\", train_series.columns.tolist())\n\ndisplay(train_series.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.326808Z","iopub.execute_input":"2026-09-12T08:15:36.327271Z","iopub.status.idle":"2026-09-12T08:15:36.476951Z","shell.execute_reply.started":"2026-09-12T08:15:36.327227Z","shell.execute_reply":"2026-09-12T08:15:36.475768Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.478398Z","iopub.execute_input":"2026-09-12T08:15:36.478779Z","iopub.status.idle":"2026-09-12T08:15:36.500422Z","shell.execute_reply.started":"2026-09-12T08:15:36.478724Z","shell.execute_reply":"2026-09-12T08:15:36.499176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series['Anatomical_Plane'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.501838Z","iopub.execute_input":"2026-09-12T08:15:36.502231Z","iopub.status.idle":"2026-09-12T08:15:36.52298Z","shell.execute_reply.started":"2026-09-12T08:15:36.50219Z","shell.execute_reply":"2026-09-12T08:15:36.521857Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series['Fluid_Sensitive'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.524457Z","iopub.execute_input":"2026-09-12T08:15:36.525152Z","iopub.status.idle":"2026-09-12T08:15:36.548527Z","shell.execute_reply.started":"2026-09-12T08:15:36.525076Z","shell.execute_reply":"2026-09-12T08:15:36.547318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_series['Fat_Suppression'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.550177Z","iopub.execute_input":"2026-09-12T08:15:36.550553Z","iopub.status.idle":"2026-09-12T08:15:36.575316Z","shell.execute_reply.started":"2026-09-12T08:15:36.550505Z","shell.execute_reply":"2026-09-12T08:15:36.574142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_per_study = train_series.groupby('StudyInstanceUID').size()\n\nprint(series_per_study.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.580004Z","iopub.execute_input":"2026-09-12T08:15:36.580583Z","iopub.status.idle":"2026-09-12T08:15:36.613323Z","shell.execute_reply.started":"2026-09-12T08:15:36.580542Z","shell.execute_reply":"2026-09-12T08:15:36.612101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_study = train[train['ACL'].notna()].iloc[0]['StudyInstanceUID']\n\nprint(labeled_study)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.614626Z","iopub.execute_input":"2026-09-12T08:15:36.615111Z","iopub.status.idle":"2026-09-12T08:15:36.628501Z","shell.execute_reply.started":"2026-09-12T08:15:36.615065Z","shell.execute_reply":"2026-09-12T08:15:36.627441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_series = train_series[\n train_series['StudyInstanceUID'] == labeled_study\n]\n\nprint(study_series.shape)\ndisplay(study_series)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.629743Z","iopub.execute_input":"2026-09-12T08:15:36.630287Z","iopub.status.idle":"2026-09-12T08:15:36.659418Z","shell.execute_reply.started":"2026-09-12T08:15:36.630193Z","shell.execute_reply":"2026-09-12T08:15:36.658154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_uid = study_series.iloc[3]['SeriesInstanceUID']\n\nprint(series_uid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.660952Z","iopub.execute_input":"2026-09-12T08:15:36.661453Z","iopub.status.idle":"2026-09-12T08:15:36.683783Z","shell.execute_reply.started":"2026-09-12T08:15:36.661412Z","shell.execute_reply":"2026-09-12T08:15:36.682478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_uid = labeled_study\n\nprint(\"Study UID:\", study_uid)\nprint(\"Series UID:\", series_uid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.685163Z","iopub.execute_input":"2026-09-12T08:15:36.685481Z","iopub.status.idle":"2026-09-12T08:15:36.706266Z","shell.execute_reply.started":"2026-09-12T08:15:36.685455Z","shell.execute_reply":"2026-09-12T08:15:36.70496Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n study_uid,\n series_uid\n)\n\nprint(series_path)\nprint(os.path.exists(series_path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.707722Z","iopub.execute_input":"2026-09-12T08:15:36.708134Z","iopub.status.idle":"2026-09-12T08:15:36.733625Z","shell.execute_reply.started":"2026-09-12T08:15:36.708087Z","shell.execute_reply":"2026-09-12T08:15:36.732495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dicom_files = sorted(os.listdir(series_path))\n\nprint(\"Number of files:\", len(dicom_files))\nprint(\"First 5 files:\", dicom_files[:5])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.734865Z","iopub.execute_input":"2026-09-12T08:15:36.735292Z","iopub.status.idle":"2026-09-12T08:15:36.766255Z","shell.execute_reply.started":"2026-09-12T08:15:36.735263Z","shell.execute_reply":"2026-09-12T08:15:36.763436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pydicom\nimport os\n\nslices = []\n\nfor file in dicom_files:\n path = os.path.join(series_path, file)\n ds = pydicom.dcmread(path)\n slices.append(ds)\n\nprint(\"Number of slices:\", len(slices))\nprint(\"Image shape:\", slices[0].pixel_array.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:36.767595Z","iopub.execute_input":"2026-09-12T08:15:36.768065Z","iopub.status.idle":"2026-09-12T08:15:37.986463Z","shell.execute_reply.started":"2026-09-12T08:15:36.767993Z","shell.execute_reply":"2026-09-12T08:15:37.985292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nindices = [5, 10, 15, 20, 25, 30]\n\nfig, axes = plt.subplots(2, 3, figsize=(12, 8))\n\nfor ax, i in zip(axes.ravel(), indices):\n ax.imshow(slices[i].pixel_array, cmap='gray')\n ax.set_title(f\"Slice {i}\")\n ax.axis('off')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:37.987692Z","iopub.execute_input":"2026-09-12T08:15:37.988159Z","iopub.status.idle":"2026-09-12T08:15:39.101104Z","shell.execute_reply.started":"2026-09-12T08:15:37.988114Z","shell.execute_reply":"2026-09-12T08:15:39.100139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_info = train[train['StudyInstanceUID'] == study_uid]\n\ndisplay(\n study_info[\n ['StudyInstanceUID', 'Report', 'ACL', 'MCL',\n 'Medial Meniscus', 'Lateral Meniscus',\n 'Medial OA', 'Lateral OA', 'PF OA',\n 'Effusion', 'Synovitis', \"Baker's\",\n 'Contusion', 'Fracture']\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.102462Z","iopub.execute_input":"2026-09-12T08:15:39.1028Z","iopub.status.idle":"2026-09-12T08:15:39.123643Z","shell.execute_reply.started":"2026-09-12T08:15:39.102771Z","shell.execute_reply":"2026-09-12T08:15:39.122645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_cols = [\n 'ACL', 'MCL', 'Medial Meniscus', 'Lateral Meniscus',\n 'Medial OA', 'Lateral OA', 'PF OA',\n 'Effusion', 'Synovitis', \"Baker's\",\n 'Contusion', 'Fracture'\n]\n\nlabel_counts = train[label_cols].dropna().sum().sort_values(ascending=False)\n\nprint(\"Number of positive cases:\")\ndisplay(label_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.125265Z","iopub.execute_input":"2026-09-12T08:15:39.126629Z","iopub.status.idle":"2026-09-12T08:15:39.165166Z","shell.execute_reply.started":"2026-09-12T08:15:39.126585Z","shell.execute_reply":"2026-09-12T08:15:39.164395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(12, 5))\nlabel_counts.plot(kind='bar')\nplt.ylabel(\"Number of positive studies\")\nplt.xlabel(\"Abnormality\")\nplt.title(\"Distribution of Abnormalities in Labeled Studies\")\nplt.xticks(rotation=45, ha='right')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.166583Z","iopub.execute_input":"2026-09-12T08:15:39.166965Z","iopub.status.idle":"2026-09-12T08:15:39.507825Z","shell.execute_reply.started":"2026-09-12T08:15:39.166936Z","shell.execute_reply":"2026-09-12T08:15:39.506648Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Total studies:\", train['StudyInstanceUID'].nunique())\nprint(\"Labeled studies:\", train[train['ACL'].notna()]['StudyInstanceUID'].nunique())\nprint(\"Total series:\", train_series['SeriesInstanceUID'].nunique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.509103Z","iopub.execute_input":"2026-09-12T08:15:39.509507Z","iopub.status.idle":"2026-09-12T08:15:39.530133Z","shell.execute_reply.started":"2026-09-12T08:15:39.509472Z","shell.execute_reply":"2026-09-12T08:15:39.528988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Missing labels:\")\ndisplay(train[label_cols].isna().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.53276Z","iopub.execute_input":"2026-09-12T08:15:39.533743Z","iopub.status.idle":"2026-09-12T08:15:39.544378Z","shell.execute_reply.started":"2026-09-12T08:15:39.533707Z","shell.execute_reply":"2026-09-12T08:15:39.543365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pydicom\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.545665Z","iopub.execute_input":"2026-09-12T08:15:39.545923Z","iopub.status.idle":"2026-09-12T08:15:39.561618Z","shell.execute_reply.started":"2026-09-12T08:15:39.545898Z","shell.execute_reply":"2026-09-12T08:15:39.560628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_dicom_series(series_path):\n# Load all DICOM files from a series and sort them using DICOM spatial metadata.\n dicom_slices = []\n for filename in os.listdir(series_path):\n filepath = os.path.join(series_path, filename)\n try:\n ds = pydicom.dcmread(filepath)\n # Make sure the file contains image data\n if hasattr(ds, \"PixelData\"):\n dicom_slices.append(ds)\n except Exception as e:\n print(f\"Could not read {filename}: {e}\")\n # Sort using ImagePositionPatient when available\n if all(hasattr(ds, \"ImagePositionPatient\") for ds in dicom_slices):\n dicom_slices.sort(key=lambda ds: float(ds.ImagePositionPatient[2]))\n # Otherwise fall back to InstanceNumber\n elif all(hasattr(ds, \"InstanceNumber\") for ds in dicom_slices):\n dicom_slices.sort( key=lambda ds: int(ds.InstanceNumber))\n\n return dicom_slices","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.563004Z","iopub.execute_input":"2026-09-12T08:15:39.563423Z","iopub.status.idle":"2026-09-12T08:15:39.586337Z","shell.execute_reply.started":"2026-09-12T08:15:39.563371Z","shell.execute_reply":"2026-09-12T08:15:39.585018Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ordered_slices = load_dicom_series(series_path)\n\nprint(\"Number of slices:\", len(ordered_slices))\nprint(\"Image shape:\", ordered_slices[0].pixel_array.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.587928Z","iopub.execute_input":"2026-09-12T08:15:39.588404Z","iopub.status.idle":"2026-09-12T08:15:39.697076Z","shell.execute_reply.started":"2026-09-12T08:15:39.588357Z","shell.execute_reply":"2026-09-12T08:15:39.695895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"indices = np.linspace(\n 0,\n len(ordered_slices) - 1,\n 6,\n dtype=int\n)\n\nfig, axes = plt.subplots(2, 3, figsize=(12, 8))\n\nfor ax, i in zip(axes.ravel(), indices):\n ax.imshow(\n ordered_slices[i].pixel_array,\n cmap=\"gray\"\n )\n ax.set_title(f\"Slice {i}\")\n ax.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:39.698365Z","iopub.execute_input":"2026-09-12T08:15:39.69861Z","iopub.status.idle":"2026-09-12T08:15:40.712233Z","shell.execute_reply.started":"2026-09-12T08:15:39.698587Z","shell.execute_reply":"2026-09-12T08:15:40.710819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ds = ordered_slices[len(ordered_slices)//2]\n\nmetadata_fields = [\n \"PatientID\",\n \"StudyInstanceUID\",\n \"SeriesInstanceUID\",\n \"Modality\",\n \"Manufacturer\",\n \"MagneticFieldStrength\",\n \"SliceThickness\",\n \"SpacingBetweenSlices\",\n \"PixelSpacing\",\n \"ImageOrientationPatient\",\n \"ImagePositionPatient\",\n \"InstanceNumber\",\n \"Rows\",\n \"Columns\"\n]\n\nfor field in metadata_fields:\n if hasattr(ds, field):\n print(f\"{field}: {getattr(ds, field)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.713718Z","iopub.execute_input":"2026-09-12T08:15:40.714207Z","iopub.status.idle":"2026-09-12T08:15:40.723181Z","shell.execute_reply.started":"2026-09-12T08:15:40.714158Z","shell.execute_reply":"2026-09-12T08:15:40.722073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_pixel_array(dicom_slices):\n # Convert DICOM slices into a NumPy array.\n images = []\n for ds in dicom_slices:\n image = ds.pixel_array.astype(np.float32)\n images.append(image)\n return np.stack(images)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.724812Z","iopub.execute_input":"2026-09-12T08:15:40.725568Z","iopub.status.idle":"2026-09-12T08:15:40.746097Z","shell.execute_reply.started":"2026-09-12T08:15:40.725528Z","shell.execute_reply":"2026-09-12T08:15:40.744624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"volume = get_pixel_array(ordered_slices)\n\nprint(\"Volume shape:\", volume.shape)\nprint(\"Data type:\", volume.dtype)\nprint(\"Minimum:\", volume.min())\nprint(\"Maximum:\", volume.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.74779Z","iopub.execute_input":"2026-09-12T08:15:40.748202Z","iopub.status.idle":"2026-09-12T08:15:40.820329Z","shell.execute_reply.started":"2026-09-12T08:15:40.748159Z","shell.execute_reply":"2026-09-12T08:15:40.819233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_volume(volume): \n lower = np.percentile(volume, 1)\n upper = np.percentile(volume, 99)\n volume = np.clip(volume, lower, upper)\n volume = (volume - lower) / (upper - lower + 1e-8)\n return volume","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.821686Z","iopub.execute_input":"2026-09-12T08:15:40.822044Z","iopub.status.idle":"2026-09-12T08:15:40.828119Z","shell.execute_reply.started":"2026-09-12T08:15:40.821989Z","shell.execute_reply":"2026-09-12T08:15:40.826931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"normalized_volume = normalize_volume(volume)\n\nprint(\"Minimum:\", normalized_volume.min())\nprint(\"Maximum:\", normalized_volume.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.829486Z","iopub.execute_input":"2026-09-12T08:15:40.829846Z","iopub.status.idle":"2026-09-12T08:15:40.941446Z","shell.execute_reply.started":"2026-09-12T08:15:40.829807Z","shell.execute_reply":"2026-09-12T08:15:40.940219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"middle = len(normalized_volume) // 2\n\nplt.figure(figsize=(10, 4))\n\nplt.subplot(1, 2, 1)\nplt.imshow(\n volume[middle],\n cmap=\"gray\"\n)\nplt.title(\"Original MRI\")\nplt.axis(\"off\")\n\nplt.subplot(1, 2, 2)\nplt.imshow(\n normalized_volume[middle],\n cmap=\"gray\"\n)\nplt.title(\"Normalized MRI\")\nplt.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:40.942762Z","iopub.execute_input":"2026-09-12T08:15:40.943155Z","iopub.status.idle":"2026-09-12T08:15:41.238625Z","shell.execute_reply.started":"2026-09-12T08:15:40.943116Z","shell.execute_reply":"2026-09-12T08:15:41.237449Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_candidate_series(study_uid, train_series): \n candidates = train_series[\n (train_series[\"StudyInstanceUID\"] == study_uid) &\n (train_series[\"Fluid_Sensitive\"] == 1) &\n (train_series[\"Fat_Suppression\"] == 1)\n ].copy()\n \n return candidates","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.241626Z","iopub.execute_input":"2026-09-12T08:15:41.242021Z","iopub.status.idle":"2026-09-12T08:15:41.248559Z","shell.execute_reply.started":"2026-09-12T08:15:41.241977Z","shell.execute_reply":"2026-09-12T08:15:41.247667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"candidates = get_candidate_series(study_uid, train_series)\n\nprint(\"Study UID:\", study_uid)\nprint(\"Number of candidate series:\", len(candidates))\n\ndisplay(candidates)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.25021Z","iopub.execute_input":"2026-09-12T08:15:41.251753Z","iopub.status.idle":"2026-09-12T08:15:41.293173Z","shell.execute_reply.started":"2026-09-12T08:15:41.251709Z","shell.execute_reply":"2026-09-12T08:15:41.292222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(\n candidates[\n [\"SeriesInstanceUID\",\n \"Fluid_Sensitive\",\n \"Fat_Suppression\",\n \"Anatomical_Plane\"]\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.294359Z","iopub.execute_input":"2026-09-12T08:15:41.294677Z","iopub.status.idle":"2026-09-12T08:15:41.312756Z","shell.execute_reply.started":"2026-09-12T08:15:41.294644Z","shell.execute_reply":"2026-09-12T08:15:41.311158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_candidate_series(study_uid, train_series):\n candidates = train_series[\n (train_series[\"StudyInstanceUID\"] == study_uid) &\n (train_series[\"Fluid_Sensitive\"] == 1) &\n (train_series[\"Fat_Suppression\"] == 1)\n ].copy()\n\n return candidates","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.318957Z","iopub.execute_input":"2026-09-12T08:15:41.319463Z","iopub.status.idle":"2026-09-12T08:15:41.332616Z","shell.execute_reply.started":"2026-09-12T08:15:41.319434Z","shell.execute_reply":"2026-09-12T08:15:41.331647Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_columns = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nlabeled_studies = train.dropna(\n subset=label_columns\n).copy()\n\nprint(\"Number of labeled studies:\", len(labeled_studies))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.334Z","iopub.execute_input":"2026-09-12T08:15:41.334376Z","iopub.status.idle":"2026-09-12T08:15:41.358576Z","shell.execute_reply.started":"2026-09-12T08:15:41.334338Z","shell.execute_reply":"2026-09-12T08:15:41.357462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"candidate_series = train_series[\n (train_series[\"StudyInstanceUID\"].isin(\n labeled_studies[\"StudyInstanceUID\"]\n )) &\n (train_series[\"Fluid_Sensitive\"] == 1) &\n (train_series[\"Fat_Suppression\"] == 1)\n].copy()\n\nprint(\"Candidate series:\", len(candidate_series))\n\ndisplay(\n candidate_series[\n [\n \"StudyInstanceUID\",\n \"SeriesInstanceUID\",\n \"Fluid_Sensitive\",\n \"Fat_Suppression\",\n \"Anatomical_Plane\"\n ]\n ].head(20)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.359823Z","iopub.execute_input":"2026-09-12T08:15:41.360216Z","iopub.status.idle":"2026-09-12T08:15:41.413705Z","shell.execute_reply.started":"2026-09-12T08:15:41.360177Z","shell.execute_reply":"2026-09-12T08:15:41.412277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n candidate_series[\"Anatomical_Plane\"]\n .value_counts()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.414999Z","iopub.execute_input":"2026-09-12T08:15:41.418215Z","iopub.status.idle":"2026-09-12T08:15:41.426222Z","shell.execute_reply.started":"2026-09-12T08:15:41.418165Z","shell.execute_reply":"2026-09-12T08:15:41.425124Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show the abnormality labels for the first few labeled studies\n\ndisplay(\n labeled_studies[\n [\"StudyInstanceUID\"] + label_columns\n ].head(10)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.427651Z","iopub.execute_input":"2026-09-12T08:15:41.429483Z","iopub.status.idle":"2026-09-12T08:15:41.481683Z","shell.execute_reply.started":"2026-09-12T08:15:41.429434Z","shell.execute_reply":"2026-09-12T08:15:41.480771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check which anatomical planes are available for each labeled study\n\nplane_availability = (\n candidate_series\n .groupby(\"StudyInstanceUID\")[\"Anatomical_Plane\"]\n .apply(lambda x: \", \".join(sorted(x.unique())))\n .reset_index()\n)\n\ndisplay(plane_availability.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.483217Z","iopub.execute_input":"2026-09-12T08:15:41.484503Z","iopub.status.idle":"2026-09-12T08:15:41.508866Z","shell.execute_reply.started":"2026-09-12T08:15:41.484457Z","shell.execute_reply":"2026-09-12T08:15:41.50811Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count how many studies have each combination of planes\n\nplane_combination_counts = (\n plane_availability[\"Anatomical_Plane\"]\n .value_counts()\n)\n\nprint(plane_combination_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.510274Z","iopub.execute_input":"2026-09-12T08:15:41.510804Z","iopub.status.idle":"2026-09-12T08:15:41.52527Z","shell.execute_reply.started":"2026-09-12T08:15:41.510761Z","shell.execute_reply":"2026-09-12T08:15:41.52392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check number of candidate series per plane for each study\n\nseries_per_plane = (\n candidate_series\n .groupby([\"StudyInstanceUID\", \"Anatomical_Plane\"])\n .size()\n .reset_index(name=\"Number_of_Series\")\n)\n\ndisplay(series_per_plane.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.526726Z","iopub.execute_input":"2026-09-12T08:15:41.527098Z","iopub.status.idle":"2026-09-12T08:15:41.569347Z","shell.execute_reply.started":"2026-09-12T08:15:41.527055Z","shell.execute_reply":"2026-09-12T08:15:41.568306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n series_per_plane[\"Number_of_Series\"].value_counts()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.570356Z","iopub.execute_input":"2026-09-12T08:15:41.570964Z","iopub.status.idle":"2026-09-12T08:15:41.581854Z","shell.execute_reply.started":"2026-09-12T08:15:41.570923Z","shell.execute_reply":"2026-09-12T08:15:41.579518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find study-plane combinations with more than one candidate series\n\nmultiple_series = series_per_plane[\n series_per_plane[\"Number_of_Series\"] > 1\n].copy()\n\nprint(\"Study-plane combinations with multiple series:\",\n len(multiple_series))\n\ndisplay(multiple_series)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.584089Z","iopub.execute_input":"2026-09-12T08:15:41.584545Z","iopub.status.idle":"2026-09-12T08:15:41.616093Z","shell.execute_reply.started":"2026-09-12T08:15:41.584504Z","shell.execute_reply":"2026-09-12T08:15:41.615219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show the actual candidate series for the duplicated study-plane combinations\n\nduplicate_series = candidate_series.merge(\n multiple_series[\n [\"StudyInstanceUID\", \"Anatomical_Plane\"]\n ],\n on=[\"StudyInstanceUID\", \"Anatomical_Plane\"],\n how=\"inner\"\n)\n\ndisplay(\n duplicate_series[\n [\n \"StudyInstanceUID\",\n \"SeriesInstanceUID\",\n \"Fluid_Sensitive\",\n \"Fat_Suppression\",\n \"Anatomical_Plane\"\n ]\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.617341Z","iopub.execute_input":"2026-09-12T08:15:41.617697Z","iopub.status.idle":"2026-09-12T08:15:41.668942Z","shell.execute_reply.started":"2026-09-12T08:15:41.617662Z","shell.execute_reply":"2026-09-12T08:15:41.667438Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def count_dicom_slices(series_path):\n #Count DICOM files in a series directory.\n \n count = 0\n\n for filename in os.listdir(series_path):\n filepath = os.path.join(series_path, filename)\n\n try:\n ds = pydicom.dcmread(filepath)\n\n if hasattr(ds, \"PixelData\"):\n count += 1\n\n except:\n pass\n\n return count","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.670601Z","iopub.execute_input":"2026-09-12T08:15:41.671198Z","iopub.status.idle":"2026-09-12T08:15:41.694621Z","shell.execute_reply.started":"2026-09-12T08:15:41.671165Z","shell.execute_reply":"2026-09-12T08:15:41.693144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count slices for the duplicated candidate series\n\nslice_counts = []\n\nfor _, row in duplicate_series.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n count = count_dicom_slices(series_path)\n\n slice_counts.append(count)\n\nduplicate_series = duplicate_series.copy()\nduplicate_series[\"Number_of_Slices\"] = slice_counts\n\ndisplay(\n duplicate_series[\n [\n \"StudyInstanceUID\",\n \"SeriesInstanceUID\",\n \"Anatomical_Plane\",\n \"Number_of_Slices\"\n ]\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:41.695855Z","iopub.execute_input":"2026-09-12T08:15:41.697096Z","iopub.status.idle":"2026-09-12T08:15:56.420599Z","shell.execute_reply.started":"2026-09-12T08:15:41.697049Z","shell.execute_reply":"2026-09-12T08:15:56.419459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Take the first study with multiple series in the same plane\n\nexample_study = multiple_series.iloc[0][\"StudyInstanceUID\"]\nexample_plane = multiple_series.iloc[0][\"Anatomical_Plane\"]\n\nprint(\"Study:\", example_study)\nprint(\"Plane:\", example_plane)\n\ndisplay(\n duplicate_series[\n (duplicate_series[\"StudyInstanceUID\"] == example_study) &\n (duplicate_series[\"Anatomical_Plane\"] == example_plane)\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:56.422197Z","iopub.execute_input":"2026-09-12T08:15:56.422589Z","iopub.status.idle":"2026-09-12T08:15:56.43653Z","shell.execute_reply.started":"2026-09-12T08:15:56.422557Z","shell.execute_reply":"2026-09-12T08:15:56.435359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect metadata of the two candidate series\n\nexample_candidates = duplicate_series[\n (duplicate_series[\"StudyInstanceUID\"] == example_study) &\n (duplicate_series[\"Anatomical_Plane\"] == example_plane)\n]\n\nfor _, row in example_candidates.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n slices = load_dicom_series(series_path)\n ds = slices[len(slices) // 2]\n\n print(\"\\n-----------------------------\")\n print(\"Series UID:\", row[\"SeriesInstanceUID\"])\n print(\"Plane:\", row[\"Anatomical_Plane\"])\n print(\"Number of slices:\", len(slices))\n print(\"Image shape:\", ds.pixel_array.shape)\n\n for field in [\n \"SliceThickness\",\n \"SpacingBetweenSlices\",\n \"PixelSpacing\",\n \"MagneticFieldStrength\",\n \"ImageOrientationPatient\"\n ]:\n if hasattr(ds, field):\n print(f\"{field}: {getattr(ds, field)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:56.437915Z","iopub.execute_input":"2026-09-12T08:15:56.438312Z","iopub.status.idle":"2026-09-12T08:15:56.62786Z","shell.execute_reply.started":"2026-09-12T08:15:56.438232Z","shell.execute_reply":"2026-09-12T08:15:56.626442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compare additional DICOM information for the duplicate series\n\nfor _, row in example_candidates.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n slices = load_dicom_series(series_path)\n ds = slices[len(slices) // 2]\n\n print(\"\\n==============================\")\n print(\"Series UID:\", row[\"SeriesInstanceUID\"])\n print(\"Plane:\", row[\"Anatomical_Plane\"])\n\n for field in [\n \"SeriesDescription\",\n \"SequenceName\",\n \"ScanningSequence\",\n \"SequenceVariant\",\n \"MRAcquisitionType\",\n \"RepetitionTime\",\n \"EchoTime\",\n \"FlipAngle\"\n ]:\n if hasattr(ds, field):\n print(f\"{field}: {getattr(ds, field)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:56.635085Z","iopub.execute_input":"2026-09-12T08:15:56.6355Z","iopub.status.idle":"2026-09-12T08:15:56.78063Z","shell.execute_reply.started":"2026-09-12T08:15:56.63547Z","shell.execute_reply":"2026-09-12T08:15:56.779445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Compare the series descriptions of all duplicate candidate series\n\nfor _, row in duplicate_series.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n try:\n slices = load_dicom_series(series_path)\n ds = slices[len(slices) // 2]\n\n description = getattr(\n ds,\n \"SeriesDescription\",\n \"Not available\"\n )\n\n print(\n f\"{row['Anatomical_Plane']:8} | \"\n f\"{len(slices):3} slices | \"\n f\"{description}\"\n )\n\n except Exception as e:\n print(\"Error:\", e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:56.781949Z","iopub.execute_input":"2026-09-12T08:15:56.782648Z","iopub.status.idle":"2026-09-12T08:15:59.657492Z","shell.execute_reply.started":"2026-09-12T08:15:56.782579Z","shell.execute_reply":"2026-09-12T08:15:59.656578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the different series descriptions\n\nprint(\n duplicate_series.shape[0],\n \"duplicate candidate series found.\"\n)\n\nseries_descriptions = []\n\nfor _, row in duplicate_series.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n try:\n slices = load_dicom_series(series_path)\n\n if len(slices) > 0:\n ds = slices[len(slices) // 2]\n\n description = getattr(\n ds,\n \"SeriesDescription\",\n \"Unknown\"\n )\n\n series_descriptions.append(description)\n\n except Exception as e:\n print(\"Error:\", e)\n\nprint(\"\\nUnique Series Descriptions:\\n\")\n\nfor description in sorted(set(series_descriptions)):\n print(description)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:15:59.658853Z","iopub.execute_input":"2026-09-12T08:15:59.659258Z","iopub.status.idle":"2026-09-12T08:16:02.253396Z","shell.execute_reply.started":"2026-09-12T08:15:59.659217Z","shell.execute_reply":"2026-09-12T08:16:02.252277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Count the different series descriptions\n# among our candidate series\n\nfrom collections import Counter\n\ndescription_counts = Counter(series_descriptions)\n\nfor description, count in description_counts.most_common():\n print(f\"{count:2} | {description}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:16:02.255005Z","iopub.execute_input":"2026-09-12T08:16:02.255326Z","iopub.status.idle":"2026-09-12T08:16:02.262437Z","shell.execute_reply.started":"2026-09-12T08:16:02.255299Z","shell.execute_reply":"2026-09-12T08:16:02.261306Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Get SeriesDescription for all candidate series\n\nall_series_info = []\n\nfor _, row in candidate_series.iterrows():\n\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n try:\n slices = load_dicom_series(series_path)\n\n if len(slices) == 0:\n continue\n\n ds = slices[len(slices) // 2]\n\n description = getattr(\n ds,\n \"SeriesDescription\",\n \"Unknown\"\n )\n\n all_series_info.append({\n \"StudyInstanceUID\": row[\"StudyInstanceUID\"],\n \"SeriesInstanceUID\": row[\"SeriesInstanceUID\"],\n \"Anatomical_Plane\": row[\"Anatomical_Plane\"],\n \"Number_of_Slices\": len(slices),\n \"SeriesDescription\": description\n })\n\n except Exception as e:\n print(\"Error:\", e)\n\nall_series_info = pd.DataFrame(all_series_info)\n\nprint(\"Total series inspected:\", len(all_series_info))\n\ndisplay(all_series_info.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:16:02.263793Z","iopub.execute_input":"2026-09-12T08:16:02.264332Z","iopub.status.idle":"2026-09-12T08:17:10.23702Z","shell.execute_reply.started":"2026-09-12T08:16:02.264288Z","shell.execute_reply":"2026-09-12T08:17:10.236171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_description_counts = (\n all_series_info[\"SeriesDescription\"]\n .value_counts()\n)\n\ndisplay(all_description_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.237986Z","iopub.execute_input":"2026-09-12T08:17:10.238245Z","iopub.status.idle":"2026-09-12T08:17:10.248225Z","shell.execute_reply.started":"2026-09-12T08:17:10.238222Z","shell.execute_reply":"2026-09-12T08:17:10.247154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show the most common sequence descriptions\n# among all candidate series\n\nall_description_counts = (\n all_series_info[\"SeriesDescription\"]\n .value_counts()\n)\n\nprint(\"Number of unique descriptions:\",\n len(all_description_counts))\n\nprint(\"\\nTop 20 descriptions:\\n\")\n\ndisplay(all_description_counts.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.249931Z","iopub.execute_input":"2026-09-12T08:17:10.250541Z","iopub.status.idle":"2026-09-12T08:17:10.274623Z","shell.execute_reply.started":"2026-09-12T08:17:10.25051Z","shell.execute_reply":"2026-09-12T08:17:10.273342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Look at the candidate series distribution for each plane\n\nplane_summary = (\n all_series_info\n .groupby(\"Anatomical_Plane\")\n .agg(\n Number_of_Series=(\"SeriesInstanceUID\", \"count\"),\n Number_of_Studies=(\"StudyInstanceUID\", \"nunique\")\n )\n .sort_index()\n)\n\ndisplay(plane_summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.276204Z","iopub.execute_input":"2026-09-12T08:17:10.276915Z","iopub.status.idle":"2026-09-12T08:17:10.310494Z","shell.execute_reply.started":"2026-09-12T08:17:10.276883Z","shell.execute_reply":"2026-09-12T08:17:10.309367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Most common sequence descriptions for each anatomical plane\n\nfor plane in [\"Sagittal\", \"Coronal\", \"Axial\"]:\n\n print(\"\\n==============================\")\n print(f\"{plane} sequences\")\n print(\"==============================\")\n\n counts = (\n all_series_info[\n all_series_info[\"Anatomical_Plane\"] == plane\n ][\"SeriesDescription\"]\n .value_counts()\n .head(15)\n )\n\n display(counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.314229Z","iopub.execute_input":"2026-09-12T08:17:10.315021Z","iopub.status.idle":"2026-09-12T08:17:10.348858Z","shell.execute_reply.started":"2026-09-12T08:17:10.314985Z","shell.execute_reply":"2026-09-12T08:17:10.34781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def classify_sequence(description):\n \"\"\"\n Assign a broad MRI sequence family based on SeriesDescription.\n \"\"\"\n\n desc = str(description).lower()\n\n # Unknown / unusable descriptions\n if \"dummy\" in desc or desc == \"unknown\":\n return \"Unknown\"\n\n # STIR\n if \"stir\" in desc:\n return \"STIR\"\n\n # PD + fat suppression\n if (\n (\"pd\" in desc or \"pdw\" in desc)\n and (\n \"fs\" in desc\n or \"fat\" in desc\n or \"spir\" in desc\n or \"spair\" in desc\n )\n ):\n return \"PD_FS\"\n\n # T2 + fat suppression\n if (\n (\"t2\" in desc)\n and (\n \"fs\" in desc\n or \"fat\" in desc\n or \"spir\" in desc\n or \"spair\" in desc\n )\n ):\n return \"T2_FS\"\n\n return \"Other\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.350595Z","iopub.execute_input":"2026-09-12T08:17:10.350995Z","iopub.status.idle":"2026-09-12T08:17:10.364651Z","shell.execute_reply.started":"2026-09-12T08:17:10.350965Z","shell.execute_reply":"2026-09-12T08:17:10.363121Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"all_series_info[\"Sequence_Family\"] = (\n all_series_info[\"SeriesDescription\"]\n .apply(classify_sequence)\n)\n\ndisplay(\n all_series_info[\n [\n \"Anatomical_Plane\",\n \"SeriesDescription\",\n \"Sequence_Family\"\n ]\n ].head(30)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.366201Z","iopub.execute_input":"2026-09-12T08:17:10.366465Z","iopub.status.idle":"2026-09-12T08:17:10.394969Z","shell.execute_reply.started":"2026-09-12T08:17:10.366443Z","shell.execute_reply":"2026-09-12T08:17:10.393781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"family_counts = (\n all_series_info[\"Sequence_Family\"]\n .value_counts()\n)\n\ndisplay(family_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.396305Z","iopub.execute_input":"2026-09-12T08:17:10.396635Z","iopub.status.idle":"2026-09-12T08:17:10.419541Z","shell.execute_reply.started":"2026-09-12T08:17:10.396577Z","shell.execute_reply":"2026-09-12T08:17:10.418144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"family_by_plane = pd.crosstab(\n all_series_info[\"Anatomical_Plane\"],\n all_series_info[\"Sequence_Family\"]\n)\n\ndisplay(family_by_plane)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.420992Z","iopub.execute_input":"2026-09-12T08:17:10.421411Z","iopub.status.idle":"2026-09-12T08:17:10.46679Z","shell.execute_reply.started":"2026-09-12T08:17:10.42137Z","shell.execute_reply":"2026-09-12T08:17:10.465911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"study_family_coverage = (\n all_series_info\n .groupby([\"Sequence_Family\", \"Anatomical_Plane\"])\n [\"StudyInstanceUID\"]\n .nunique()\n .unstack(fill_value=0)\n)\n\ndisplay(study_family_coverage)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.468301Z","iopub.execute_input":"2026-09-12T08:17:10.468605Z","iopub.status.idle":"2026-09-12T08:17:10.481192Z","shell.execute_reply.started":"2026-09-12T08:17:10.468579Z","shell.execute_reply":"2026-09-12T08:17:10.480323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check how many studies have at least one usable sequence\n# in each anatomical plane.\n\nusable_families = [\"PD_FS\", \"T2_FS\", \"STIR\", \"Other\"]\n\nusable_series = all_series_info[\n all_series_info[\"Sequence_Family\"].isin(usable_families)\n]\n\nusable_plane_coverage = (\n usable_series\n .groupby(\"Anatomical_Plane\")[\"StudyInstanceUID\"]\n .nunique()\n)\n\nprint(\"Studies with at least one usable series:\")\nprint(usable_plane_coverage)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.482439Z","iopub.execute_input":"2026-09-12T08:17:10.482733Z","iopub.status.idle":"2026-09-12T08:17:10.506528Z","shell.execute_reply.started":"2026-09-12T08:17:10.482709Z","shell.execute_reply":"2026-09-12T08:17:10.505402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check which studies are missing a usable series in any plane\n\nusable_by_study_plane = (\n usable_series[\n [\"StudyInstanceUID\", \"Anatomical_Plane\"]\n ]\n .drop_duplicates()\n)\n\nexpected_planes = {\n \"Axial\",\n \"Coronal\",\n \"Sagittal\"\n}\n\nmissing_planes = []\n\nfor study_uid in labeled_studies[\"StudyInstanceUID\"]:\n\n available = set(\n usable_by_study_plane[\n usable_by_study_plane[\"StudyInstanceUID\"] == study_uid\n ][\"Anatomical_Plane\"]\n )\n\n missing = expected_planes - available\n\n if missing:\n missing_planes.append({\n \"StudyInstanceUID\": study_uid,\n \"Missing_Planes\": \", \".join(sorted(missing))\n })\n\nmissing_planes = pd.DataFrame(missing_planes)\n\nprint(\"Studies missing at least one usable plane:\",\n len(missing_planes))\n\ndisplay(missing_planes)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.508089Z","iopub.execute_input":"2026-09-12T08:17:10.508465Z","iopub.status.idle":"2026-09-12T08:17:10.557842Z","shell.execute_reply.started":"2026-09-12T08:17:10.508425Z","shell.execute_reply":"2026-09-12T08:17:10.556739Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect all candidate series belonging to studies\n# where we are currently missing usable planes\n\nproblem_studies = missing_planes[\"StudyInstanceUID\"].tolist()\n\nproblem_series = all_series_info[\n all_series_info[\"StudyInstanceUID\"].isin(problem_studies)\n].copy()\n\ndisplay(\n problem_series[\n [\n \"StudyInstanceUID\",\n \"Anatomical_Plane\",\n \"SeriesDescription\",\n \"Sequence_Family\",\n \"Number_of_Slices\"\n ]\n ].sort_values(\n [\"StudyInstanceUID\", \"Anatomical_Plane\"]\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.559026Z","iopub.execute_input":"2026-09-12T08:17:10.559493Z","iopub.status.idle":"2026-09-12T08:17:10.589819Z","shell.execute_reply.started":"2026-09-12T08:17:10.559453Z","shell.execute_reply":"2026-09-12T08:17:10.588829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_uid = study_series.iloc[3][\"SeriesInstanceUID\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.591165Z","iopub.execute_input":"2026-09-12T08:17:10.591523Z","iopub.status.idle":"2026-09-12T08:17:10.597638Z","shell.execute_reply.started":"2026-09-12T08:17:10.591497Z","shell.execute_reply":"2026-09-12T08:17:10.596662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def select_best_series(study_uid, plane):\n \"\"\"\n Select one candidate MRI series for a given study and anatomical plane.\n Priority:\n 1. PD + Fat Suppression\n 2. T2 + Fat Suppression\n 3. STIR\n 4. Other\n 5. Unknown / Dummy as last fallback\n \"\"\"\n\n # Get all candidate series for this study and plane\n candidates = all_series_info[\n (all_series_info[\"StudyInstanceUID\"] == study_uid) &\n (all_series_info[\"Anatomical_Plane\"] == plane)\n ].copy()\n\n if candidates.empty:\n return None\n\n # Sequence priority\n priority = {\n \"PD_FS\": 1,\n \"T2_FS\": 2,\n \"STIR\": 3,\n \"Other\": 4,\n \"Unknown\": 5\n }\n\n candidates[\"Priority\"] = (\n candidates[\"Sequence_Family\"]\n .map(priority)\n .fillna(5)\n )\n\n # Sort by sequence priority.\n # If multiple series have the same priority,\n # prefer the one with more slices.\n candidates = candidates.sort_values(\n [\"Priority\", \"Number_of_Slices\"],\n ascending=[True, False]\n )\n\n return candidates.iloc[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.598727Z","iopub.execute_input":"2026-09-12T08:17:10.599105Z","iopub.status.idle":"2026-09-12T08:17:10.621586Z","shell.execute_reply.started":"2026-09-12T08:17:10.599063Z","shell.execute_reply":"2026-09-12T08:17:10.620478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"example_study = labeled_studies.iloc[0][\"StudyInstanceUID\"]\n\nfor plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n\n selected = select_best_series(\n example_study,\n plane\n )\n\n print(\"\\n\", plane)\n\n if selected is None:\n print(\"No series available\")\n else:\n print(\"Series UID:\", selected[\"SeriesInstanceUID\"])\n print(\"Description:\", selected[\"SeriesDescription\"])\n print(\"Sequence family:\", selected[\"Sequence_Family\"])\n print(\"Number of slices:\", selected[\"Number_of_Slices\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.622782Z","iopub.execute_input":"2026-09-12T08:17:10.623084Z","iopub.status.idle":"2026-09-12T08:17:10.650454Z","shell.execute_reply.started":"2026-09-12T08:17:10.62301Z","shell.execute_reply":"2026-09-12T08:17:10.649352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selected_series = []\n\nfor study_uid in labeled_studies[\"StudyInstanceUID\"]:\n\n for plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n\n selected = select_best_series(\n study_uid,\n plane\n )\n\n if selected is not None:\n\n selected_series.append({\n \"StudyInstanceUID\": study_uid,\n \"Anatomical_Plane\": plane,\n \"SeriesInstanceUID\": selected[\"SeriesInstanceUID\"],\n \"SeriesDescription\": selected[\"SeriesDescription\"],\n \"Sequence_Family\": selected[\"Sequence_Family\"],\n \"Number_of_Slices\": selected[\"Number_of_Slices\"]\n })\n\nselected_series = pd.DataFrame(selected_series)\n\nprint(\"Selected series:\", len(selected_series))\n\ndisplay(selected_series.head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:10.651523Z","iopub.execute_input":"2026-09-12T08:17:10.651899Z","iopub.status.idle":"2026-09-12T08:17:11.066969Z","shell.execute_reply.started":"2026-09-12T08:17:10.651855Z","shell.execute_reply":"2026-09-12T08:17:11.066175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the selected MRI series\n\nselected_series_df = pd.DataFrame(selected_series)\n\nprint(\"Shape:\", selected_series_df.shape)\nprint(\"\\nColumns:\")\nprint(selected_series_df.columns.tolist())\n\ndisplay(selected_series_df.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.068096Z","iopub.execute_input":"2026-09-12T08:17:11.068349Z","iopub.status.idle":"2026-09-12T08:17:11.082117Z","shell.execute_reply.started":"2026-09-12T08:17:11.068325Z","shell.execute_reply":"2026-09-12T08:17:11.081114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check how many selected series each study has\n\nselected_count = (\n selected_series_df\n .groupby(\"StudyInstanceUID\")\n .size()\n)\n\nprint(\"Selected series per study:\")\nprint(selected_count.value_counts().sort_index())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.083151Z","iopub.execute_input":"2026-09-12T08:17:11.083444Z","iopub.status.idle":"2026-09-12T08:17:11.105702Z","shell.execute_reply.started":"2026-09-12T08:17:11.083419Z","shell.execute_reply":"2026-09-12T08:17:11.104722Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check whether each study has all 3 anatomical planes\n\nplane_check = (\n selected_series_df\n .groupby(\"StudyInstanceUID\")[\"Anatomical_Plane\"]\n .apply(set)\n)\n\ncomplete_studies = plane_check[\n plane_check.apply(\n lambda x: x == {\"Axial\", \"Coronal\", \"Sagittal\"}\n )\n]\n\nprint(\"Total labeled studies:\", len(labeled_studies))\nprint(\"Studies with all 3 planes:\", len(complete_studies))\nprint(\n \"Studies missing at least one plane:\",\n len(labeled_studies) - len(complete_studies)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.107213Z","iopub.execute_input":"2026-09-12T08:17:11.107732Z","iopub.status.idle":"2026-09-12T08:17:11.127678Z","shell.execute_reply.started":"2026-09-12T08:17:11.107685Z","shell.execute_reply":"2026-09-12T08:17:11.126582Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(all_series_info.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.129134Z","iopub.execute_input":"2026-09-12T08:17:11.129493Z","iopub.status.idle":"2026-09-12T08:17:11.148773Z","shell.execute_reply.started":"2026-09-12T08:17:11.129466Z","shell.execute_reply":"2026-09-12T08:17:11.147556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show variables that may contain dataset paths / DICOM information\nfor name in [\n \"DATA_DIR\",\n \"TRAIN_DIR\",\n \"train_dir\",\n \"data_dir\",\n \"ROOT_DIR\",\n \"root_dir\"\n]:\n if name in globals():\n print(name, \"=\", globals()[name])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.149938Z","iopub.execute_input":"2026-09-12T08:17:11.150335Z","iopub.status.idle":"2026-09-12T08:17:11.172667Z","shell.execute_reply.started":"2026-09-12T08:17:11.150307Z","shell.execute_reply":"2026-09-12T08:17:11.171474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for name, obj in list(globals().items()):\n if isinstance(obj, pd.DataFrame):\n print(name, \"->\", obj.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.17381Z","iopub.execute_input":"2026-09-12T08:17:11.174147Z","iopub.status.idle":"2026-09-12T08:17:11.201554Z","shell.execute_reply.started":"2026-09-12T08:17:11.17412Z","shell.execute_reply":"2026-09-12T08:17:11.200292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show variables containing anything related to paths/directories/data\nfor name in list(globals().keys()):\n if any(word in name.lower() for word in [\"path\", \"dir\", \"root\", \"input\", \"data\"]):\n print(name, \"=\", globals()[name])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.203005Z","iopub.execute_input":"2026-09-12T08:17:11.203421Z","iopub.status.idle":"2026-09-12T08:17:11.223324Z","shell.execute_reply.started":"2026-09-12T08:17:11.203379Z","shell.execute_reply":"2026-09-12T08:17:11.222262Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"display(train_series.head(1))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.22471Z","iopub.execute_input":"2026-09-12T08:17:11.225713Z","iopub.status.idle":"2026-09-12T08:17:11.251278Z","shell.execute_reply.started":"2026-09-12T08:17:11.225668Z","shell.execute_reply":"2026-09-12T08:17:11.2503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show the first selected series\ndisplay(selected_series.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.25259Z","iopub.execute_input":"2026-09-12T08:17:11.253207Z","iopub.status.idle":"2026-09-12T08:17:11.281547Z","shell.execute_reply.started":"2026-09-12T08:17:11.253155Z","shell.execute_reply":"2026-09-12T08:17:11.280093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pydicom\n\n\ndef load_series_volume(study_uid, series_uid):\n \"\"\"\n Load a DICOM series and return a 3D NumPy volume.\n Slices are ordered using DICOM spatial metadata.\n \"\"\"\n\n # Path to this specific series\n series_path = os.path.join(\n BASE_PATH,\n \"train_series\",\n str(study_uid),\n str(series_uid)\n )\n\n if not os.path.exists(series_path):\n print(f\"Series directory not found: {series_path}\")\n return None\n\n # Find all DICOM files\n dicom_files = [\n os.path.join(series_path, f)\n for f in os.listdir(series_path)\n if f.lower().endswith(\".dcm\")\n ]\n\n if not dicom_files:\n print(\"No DICOM files found.\")\n return None\n\n # Read DICOM slices\n slices = []\n\n for file_path in dicom_files:\n try:\n ds = pydicom.dcmread(file_path)\n\n if hasattr(ds, \"PixelData\"):\n slices.append(ds)\n\n except Exception as e:\n print(f\"Could not read {file_path}: {e}\")\n\n if not slices:\n print(\"No readable DICOM slices found.\")\n return None\n # Sort slices using DICOM spatial metadata\n\n if all(\n hasattr(ds, \"ImageOrientationPatient\") and\n hasattr(ds, \"ImagePositionPatient\")\n for ds in slices\n ):\n\n orientation = np.array(\n slices[0].ImageOrientationPatient,\n dtype=np.float64\n )\n\n row_direction = orientation[:3]\n column_direction = orientation[3:]\n\n # Direction perpendicular to the image plane\n slice_direction = np.cross(\n row_direction,\n column_direction\n )\n\n # Sort according to position along slice direction\n slices.sort(\n key=lambda ds: np.dot(\n np.array(\n ds.ImagePositionPatient,\n dtype=np.float64\n ),\n slice_direction\n )\n )\n\n # Fallback if spatial information is unavailable\n elif all(hasattr(ds, \"InstanceNumber\") for ds in slices):\n\n slices.sort(\n key=lambda ds: int(ds.InstanceNumber)\n )\n\n # Convert slices into a 3D NumPy volume\n volume = np.stack(\n [ds.pixel_array for ds in slices],\n axis=0\n )\n\n return volume","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.291006Z","iopub.execute_input":"2026-09-12T08:17:11.292094Z","iopub.status.idle":"2026-09-12T08:17:11.306928Z","shell.execute_reply.started":"2026-09-12T08:17:11.291989Z","shell.execute_reply":"2026-09-12T08:17:11.305829Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_row = selected_series_df.iloc[0]\n\ntest_study = test_row[\"StudyInstanceUID\"]\ntest_series = test_row[\"SeriesInstanceUID\"]\ntest_plane = test_row[\"Anatomical_Plane\"]\n\nvolume = load_series_volume(\n test_study,\n test_series\n)\n\nprint(\"Plane:\", test_plane)\nprint(\"Volume shape:\", volume.shape)\nprint(\"Data type:\", volume.dtype)\nprint(\"Min:\", volume.min())\nprint(\"Max:\", volume.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.30824Z","iopub.execute_input":"2026-09-12T08:17:11.308751Z","iopub.status.idle":"2026-09-12T08:17:11.427795Z","shell.execute_reply.started":"2026-09-12T08:17:11.308699Z","shell.execute_reply":"2026-09-12T08:17:11.42686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nindices = [\n 0,\n volume.shape[0] // 2,\n volume.shape[0] - 1\n]\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\nfor ax, idx in zip(axes, indices):\n ax.imshow(volume[idx], cmap=\"gray\")\n ax.set_title(f\"Slice {idx}\")\n ax.axis(\"off\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.429208Z","iopub.execute_input":"2026-09-12T08:17:11.429584Z","iopub.status.idle":"2026-09-12T08:17:11.968464Z","shell.execute_reply.started":"2026-09-12T08:17:11.429552Z","shell.execute_reply":"2026-09-12T08:17:11.967076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def normalize_volume(volume):\n#Normalize a 3D MRI volume to [0, 1]\n volume = volume.astype(np.float32)\n\n min_val = volume.min()\n max_val = volume.max()\n\n if max_val > min_val:\n volume = (volume - min_val) / (max_val - min_val)\n else:\n volume = np.zeros_like(volume, dtype=np.float32)\n\n return volume","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.969911Z","iopub.execute_input":"2026-09-12T08:17:11.970289Z","iopub.status.idle":"2026-09-12T08:17:11.977273Z","shell.execute_reply.started":"2026-09-12T08:17:11.97026Z","shell.execute_reply":"2026-09-12T08:17:11.976321Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"normalized_volume = normalize_volume(volume)\n\nprint(\"Shape:\", normalized_volume.shape)\nprint(\"Data type:\", normalized_volume.dtype)\nprint(\"Min:\", normalized_volume.min())\nprint(\"Max:\", normalized_volume.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:11.978277Z","iopub.execute_input":"2026-09-12T08:17:11.978531Z","iopub.status.idle":"2026-09-12T08:17:12.01908Z","shell.execute_reply.started":"2026-09-12T08:17:11.978507Z","shell.execute_reply":"2026-09-12T08:17:12.018102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Testing one selected series from each anatomical plane:\\n\")\n\nfor plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n\n plane_rows = selected_series_df[\n selected_series_df[\"Anatomical_Plane\"] == plane\n ]\n\n if len(plane_rows) == 0:\n print(f\"{plane}: No series found\")\n continue\n\n row = plane_rows.iloc[0]\n\n vol = load_series_volume(\n row[\"StudyInstanceUID\"],\n row[\"SeriesInstanceUID\"]\n )\n\n print(\n f\"{plane:10} | \"\n f\"shape = {vol.shape} | \"\n f\"dtype = {vol.dtype} | \"\n f\"range = ({vol.min()}, {vol.max()})\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.020476Z","iopub.execute_input":"2026-09-12T08:17:12.020836Z","iopub.status.idle":"2026-09-12T08:17:12.273469Z","shell.execute_reply.started":"2026-09-12T08:17:12.020798Z","shell.execute_reply":"2026-09-12T08:17:12.272309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_columns = [\n \"acl\",\n \"mcl\",\n \"meniscus\",\n \"acl_tear\",\n \"mcl_tear\",\n \"meniscus_tear\",\n \"bone_contusion\",\n \"cartilage\",\n \"fracture\",\n \"effusion\",\n \"synovitis\",\n \"osteoarthritis\"\n]\n\nprint(\"Number of targets:\", len(target_columns))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.274835Z","iopub.execute_input":"2026-09-12T08:17:12.275203Z","iopub.status.idle":"2026-09-12T08:17:12.281311Z","shell.execute_reply.started":"2026-09-12T08:17:12.275175Z","shell.execute_reply":"2026-09-12T08:17:12.280193Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.282539Z","iopub.execute_input":"2026-09-12T08:17:12.282992Z","iopub.status.idle":"2026-09-12T08:17:12.30313Z","shell.execute_reply.started":"2026-09-12T08:17:12.282964Z","shell.execute_reply":"2026-09-12T08:17:12.301936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_columns = [\n \"ACL\",\n \"MCL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"Medial OA\",\n \"Lateral OA\",\n \"PF OA\",\n \"Effusion\",\n \"Synovitis\",\n \"Baker's\",\n \"Contusion\",\n \"Fracture\"\n]\n\nstudy_labels = train[\n [\"StudyInstanceUID\"] + target_columns\n].copy()\n\nprint(\"Study-level label table shape:\", study_labels.shape)\nprint(\"\\nColumns:\")\nprint(study_labels.columns.tolist())\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(study_labels.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.304614Z","iopub.execute_input":"2026-09-12T08:17:12.304993Z","iopub.status.idle":"2026-09-12T08:17:12.338237Z","shell.execute_reply.started":"2026-09-12T08:17:12.304964Z","shell.execute_reply":"2026-09-12T08:17:12.336755Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Missing values per label:\\n\")\n\nprint(\n study_labels[target_columns]\n .isna()\n .sum()\n)\n\nprint(\"\\nUnique values per label:\\n\")\n\nfor col in target_columns:\n print(f\"{col:20}:\", study_labels[col].dropna().unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.339391Z","iopub.execute_input":"2026-09-12T08:17:12.33966Z","iopub.status.idle":"2026-09-12T08:17:12.364492Z","shell.execute_reply.started":"2026-09-12T08:17:12.339636Z","shell.execute_reply":"2026-09-12T08:17:12.363481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_selected_series = selected_series_df[\n selected_series_df[\"StudyInstanceUID\"].isin(\n labeled_studies[\"StudyInstanceUID\"]\n )\n].copy()\n\nprint(\"Selected series from labeled studies:\")\nprint(\"Shape:\", labeled_selected_series.shape)\n\nprint(\"\\nSeries per anatomical plane:\")\nprint(\n labeled_selected_series[\"Anatomical_Plane\"]\n .value_counts()\n)\n\nprint(\"\\nStudies represented:\")\nprint(\n labeled_selected_series[\"StudyInstanceUID\"].nunique()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.365671Z","iopub.execute_input":"2026-09-12T08:17:12.365961Z","iopub.status.idle":"2026-09-12T08:17:12.381216Z","shell.execute_reply.started":"2026-09-12T08:17:12.365897Z","shell.execute_reply":"2026-09-12T08:17:12.380128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Selected series by plane and sequence family:\\n\")\n\ndisplay(\n labeled_selected_series\n .groupby([\"Anatomical_Plane\", \"Sequence_Family\"])\n .size()\n .reset_index(name=\"Count\")\n .sort_values([\"Anatomical_Plane\", \"Count\"], ascending=[True, False])\n)\n\nprint(\"\\nSlice-count statistics:\\n\")\n\ndisplay(\n labeled_selected_series\n .groupby(\"Anatomical_Plane\")[\"Number_of_Slices\"]\n .describe()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.382749Z","iopub.execute_input":"2026-09-12T08:17:12.38324Z","iopub.status.idle":"2026-09-12T08:17:12.432121Z","shell.execute_reply.started":"2026-09-12T08:17:12.383203Z","shell.execute_reply":"2026-09-12T08:17:12.431185Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_count_per_study = (\n labeled_selected_series\n .groupby(\"StudyInstanceUID\")\n .size()\n .reset_index(name=\"Series_Count\")\n)\n\nprint(\"Series count per study:\")\nprint(\n series_count_per_study[\"Series_Count\"]\n .value_counts()\n .sort_index()\n)\n\nprint(\"\\nStatistics:\")\nprint(\n series_count_per_study[\"Series_Count\"].describe()\n)\n\nprint(\"\\nStudies with multiple selected series:\")\ndisplay(\n series_count_per_study[\n series_count_per_study[\"Series_Count\"] > 3\n ].sort_values(\"Series_Count\", ascending=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.433468Z","iopub.execute_input":"2026-09-12T08:17:12.433829Z","iopub.status.idle":"2026-09-12T08:17:12.450476Z","shell.execute_reply.started":"2026-09-12T08:17:12.433792Z","shell.execute_reply":"2026-09-12T08:17:12.449561Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Study-level model index\n\nstudy_index = labeled_studies[\n [\"StudyInstanceUID\"] + target_columns\n].copy()\n\nfor plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n\n plane_df = (\n labeled_selected_series[\n labeled_selected_series[\"Anatomical_Plane\"] == plane\n ][[\"StudyInstanceUID\", \"SeriesInstanceUID\"]]\n .rename(\n columns={\n \"SeriesInstanceUID\": f\"{plane}_SeriesUID\"\n }\n )\n )\n\n study_index = study_index.merge(\n plane_df,\n on=\"StudyInstanceUID\",\n how=\"left\"\n )\n\nprint(\"Study-level model index shape:\", study_index.shape)\n\nprint(\"\\nMissing series:\")\nprint(\n study_index[\n [\"Axial_SeriesUID\", \"Coronal_SeriesUID\", \"Sagittal_SeriesUID\"]\n ].isna().sum()\n)\n\ndisplay(study_index.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.451791Z","iopub.execute_input":"2026-09-12T08:17:12.452192Z","iopub.status.idle":"2026-09-12T08:17:12.507525Z","shell.execute_reply.started":"2026-09-12T08:17:12.452164Z","shell.execute_reply":"2026-09-12T08:17:12.50618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\n\nprint(\"PyTorch version:\", torch.__version__)\nprint(\"CUDA available:\", torch.cuda.is_available())\n\nif torch.cuda.is_available():\n print(\"GPU:\", torch.cuda.get_device_name(0))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:12.508802Z","iopub.execute_input":"2026-09-12T08:17:12.509182Z","iopub.status.idle":"2026-09-12T08:17:16.650682Z","shell.execute_reply.started":"2026-09-12T08:17:12.509143Z","shell.execute_reply":"2026-09-12T08:17:16.649495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prepare_volume(volume, target_shape=(32, 128, 128)):\n #Normalize and resize a 3D MRI volume to a fixed shape.\n\n #Input: volume: NumPy array of shape (slices, height, width)\n\n #Output: torch.Tensor of shape (1, D, H, W)\n # Normalize intensity\n volume = normalize_volume(volume)\n\n # Convert to tensor\n volume = torch.from_numpy(volume).float()\n\n # Add batch and channel dimensions\n volume = volume.unsqueeze(0).unsqueeze(0)\n\n # Resize the 3D volume\n volume = torch.nn.functional.interpolate(\n volume,\n size=target_shape,\n mode=\"trilinear\",\n align_corners=False\n )\n\n # Remove batch dimension\n volume = volume.squeeze(0)\n\n return volume","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:16.651963Z","iopub.execute_input":"2026-09-12T08:17:16.652537Z","iopub.status.idle":"2026-09-12T08:17:16.659569Z","shell.execute_reply.started":"2026-09-12T08:17:16.652504Z","shell.execute_reply":"2026-09-12T08:17:16.658444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RSNAKneeDataset(Dataset):\n\n def __init__(self, dataframe, target_shape=(32, 128, 128)):\n self.dataframe = dataframe.reset_index(drop=True)\n self.target_shape = target_shape\n\n def __len__(self):\n return len(self.dataframe)\n\n def _load_plane(self, study_uid, series_uid):\n # Missing plane\n if pd.isna(series_uid):\n return torch.zeros(\n (1, *self.target_shape),\n dtype=torch.float32\n )\n\n volume = load_series_volume(\n study_uid,\n series_uid\n )\n\n if volume is None:\n return torch.zeros(\n (1, *self.target_shape),\n dtype=torch.float32\n )\n\n return prepare_volume(\n volume,\n target_shape=self.target_shape\n )\n\n def __getitem__(self, idx):\n\n row = self.dataframe.iloc[idx]\n\n study_uid = row[\"StudyInstanceUID\"]\n\n # Load the three anatomical planes\n axial = self._load_plane(\n study_uid,\n row[\"Axial_SeriesUID\"]\n )\n\n coronal = self._load_plane(\n study_uid,\n row[\"Coronal_SeriesUID\"]\n )\n\n sagittal = self._load_plane(\n study_uid,\n row[\"Sagittal_SeriesUID\"]\n )\n\n # Stack planes as separate input channels\n image = torch.cat(\n [axial, coronal, sagittal],\n dim=0\n )\n\n # 12 abnormality labels\n labels = torch.tensor(\n row[target_columns].values.astype(np.float32)\n )\n\n return image, labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:16.661356Z","iopub.execute_input":"2026-09-12T08:17:16.661708Z","iopub.status.idle":"2026-09-12T08:17:16.687828Z","shell.execute_reply.started":"2026-09-12T08:17:16.66168Z","shell.execute_reply":"2026-09-12T08:17:16.686348Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_dataset = RSNAKneeDataset(\n study_index,\n target_shape=(32, 128, 128)\n)\n\nimage, labels = test_dataset[0]\n\nprint(\"Image shape:\", image.shape)\nprint(\"Image dtype:\", image.dtype)\nprint(\"Image range:\", (image.min().item(), image.max().item()))\n\nprint(\"\\nLabels shape:\", labels.shape)\nprint(\"Labels:\", labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:16.689088Z","iopub.execute_input":"2026-09-12T08:17:16.689362Z","iopub.status.idle":"2026-09-12T08:17:17.184258Z","shell.execute_reply.started":"2026-09-12T08:17:16.689338Z","shell.execute_reply":"2026-09-12T08:17:17.183387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(\n test_dataset,\n batch_size=2,\n shuffle=True,\n num_workers=0\n)\n\nimages, labels = next(iter(train_loader))\n\nprint(\"Batch image shape:\", images.shape)\nprint(\"Batch label shape:\", labels.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:17.185523Z","iopub.execute_input":"2026-09-12T08:17:17.185899Z","iopub.status.idle":"2026-09-12T08:17:17.943779Z","shell.execute_reply.started":"2026-09-12T08:17:17.185861Z","shell.execute_reply":"2026-09-12T08:17:17.942798Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RSNAKnee3DCNN(nn.Module):\n\n def __init__(self, num_classes=12):\n super().__init__()\n\n self.features = nn.Sequential(\n nn.Conv3d(3, 16, kernel_size=3, padding=1),\n nn.BatchNorm3d(16),\n nn.ReLU(),\n nn.MaxPool3d(2),\n\n nn.Conv3d(16, 32, kernel_size=3, padding=1),\n nn.BatchNorm3d(32),\n nn.ReLU(),\n nn.MaxPool3d(2),\n\n nn.Conv3d(32, 64, kernel_size=3, padding=1),\n nn.BatchNorm3d(64),\n nn.ReLU(),\n\n nn.AdaptiveAvgPool3d(1)\n )\n\n self.classifier = nn.Sequential(\n nn.Flatten(),\n nn.Linear(64, 32),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(32, num_classes)\n )\n\n def forward(self, x):\n x = self.features(x)\n x = self.classifier(x)\n return x\n\n\nmodel = RSNAKnee3DCNN(num_classes=len(target_columns))\n\nprint(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:17.94484Z","iopub.execute_input":"2026-09-12T08:17:17.945114Z","iopub.status.idle":"2026-09-12T08:17:17.964236Z","shell.execute_reply.started":"2026-09-12T08:17:17.94509Z","shell.execute_reply":"2026-09-12T08:17:17.962808Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.eval()\n\nwith torch.no_grad():\n outputs = model(images)\n\nprint(\"Input shape:\", images.shape)\nprint(\"Output shape:\", outputs.shape)\nprint(\"Output values:\", outputs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:17.965593Z","iopub.execute_input":"2026-09-12T08:17:17.96598Z","iopub.status.idle":"2026-09-12T08:17:18.340395Z","shell.execute_reply.started":"2026-09-12T08:17:17.965942Z","shell.execute_reply":"2026-09-12T08:17:18.339224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(\n study_index,\n test_size=0.20,\n random_state=42\n)\n\nprint(\"Training studies:\", len(train_df))\nprint(\"Validation studies:\", len(val_df))\n\nprint(\"\\nTraining label counts:\")\nprint(train_df[target_columns].sum().sort_values(ascending=False))\n\nprint(\"\\nValidation label counts:\")\nprint(val_df[target_columns].sum().sort_values(ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:18.341678Z","iopub.execute_input":"2026-09-12T08:17:18.342907Z","iopub.status.idle":"2026-09-12T08:17:19.474908Z","shell.execute_reply.started":"2026-09-12T08:17:18.342864Z","shell.execute_reply":"2026-09-12T08:17:19.473541Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"try:\n from iterstrat.ml_stratifiers import MultilabelStratifiedShuffleSplit\n print(\"Multilabel stratification: AVAILABLE\")\nexcept ImportError:\n print(\"Multilabel stratification: NOT AVAILABLE\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.476205Z","iopub.execute_input":"2026-09-12T08:17:19.477263Z","iopub.status.idle":"2026-09-12T08:17:19.483759Z","shell.execute_reply.started":"2026-09-12T08:17:19.477231Z","shell.execute_reply":"2026-09-12T08:17:19.482551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def multilabel_split(df, test_size=0.20, random_state=42):\n\n rng = np.random.RandomState(random_state)\n\n labels = df[target_columns].values.astype(int)\n n_samples = len(df)\n n_val = int(round(n_samples * test_size))\n\n # Start with the rarest labels\n label_counts = labels.sum(axis=0)\n label_order = np.argsort(label_counts)\n\n val_indices = set()\n remaining = set(range(n_samples))\n\n # Target number of positives for each label in validation\n target_val_counts = np.round(label_counts * test_size).astype(int)\n\n for label_idx in label_order:\n\n candidates = [\n i for i in remaining\n if labels[i, label_idx] == 1\n ]\n\n rng.shuffle(candidates)\n\n needed = target_val_counts[label_idx]\n\n selected = candidates[:needed]\n\n for i in selected:\n val_indices.add(i)\n remaining.discard(i)\n\n # Fill remaining validation slots\n remaining_candidates = list(remaining)\n rng.shuffle(remaining_candidates)\n\n while len(val_indices) < n_val:\n val_indices.add(remaining_candidates.pop())\n\n val_indices = sorted(val_indices)\n train_indices = sorted(set(range(n_samples)) - set(val_indices))\n\n train_df = df.iloc[train_indices].copy()\n val_df = df.iloc[val_indices].copy()\n\n return train_df, val_df\n\n\ntrain_df, val_df = multilabel_split(\n study_index,\n test_size=0.20,\n random_state=42\n)\n\nprint(\"Training studies:\", len(train_df))\nprint(\"Validation studies:\", len(val_df))\n\nprint(\"\\nTraining label counts:\")\nprint(train_df[target_columns].sum().sort_values(ascending=False))\n\nprint(\"\\nValidation label counts:\")\nprint(val_df[target_columns].sum().sort_values(ascending=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.485411Z","iopub.execute_input":"2026-09-12T08:17:19.48586Z","iopub.status.idle":"2026-09-12T08:17:19.516093Z","shell.execute_reply.started":"2026-09-12T08:17:19.48583Z","shell.execute_reply":"2026-09-12T08:17:19.515079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, val_df = train_test_split(\n study_index,\n test_size=12,\n random_state=42,\n shuffle=True\n)\n\nprint(\"Training studies:\", len(train_df))\nprint(\"Validation studies:\", len(val_df))\n\nprint(\"\\nValidation positive counts:\")\nprint(val_df[target_columns].sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.51737Z","iopub.execute_input":"2026-09-12T08:17:19.517676Z","iopub.status.idle":"2026-09-12T08:17:19.542684Z","shell.execute_reply.started":"2026-09-12T08:17:19.517632Z","shell.execute_reply":"2026-09-12T08:17:19.541489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataset = RSNAKneeDataset(train_df)\nval_dataset = RSNAKneeDataset(val_df)\n\nprint(\"Training dataset size:\", len(train_dataset))\nprint(\"Validation dataset size:\", len(val_dataset))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.544078Z","iopub.execute_input":"2026-09-12T08:17:19.54446Z","iopub.status.idle":"2026-09-12T08:17:19.566539Z","shell.execute_reply.started":"2026-09-12T08:17:19.5444Z","shell.execute_reply":"2026-09-12T08:17:19.565773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_loader = DataLoader(\n train_dataset,\n batch_size=2,\n shuffle=True,\n num_workers=0\n)\n\nval_loader = DataLoader(\n val_dataset,\n batch_size=2,\n shuffle=False,\n num_workers=0\n)\n\nprint(\"Training batches:\", len(train_loader))\nprint(\"Validation batches:\", len(val_loader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.568152Z","iopub.execute_input":"2026-09-12T08:17:19.56893Z","iopub.status.idle":"2026-09-12T08:17:19.597251Z","shell.execute_reply.started":"2026-09-12T08:17:19.568881Z","shell.execute_reply":"2026-09-12T08:17:19.596133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"positive_counts = train_df[target_columns].sum().values\nnegative_counts = len(train_df) - positive_counts\n\npos_weight = negative_counts / positive_counts\n\npos_weight = torch.tensor(\n pos_weight,\n dtype=torch.float32\n)\n\nprint(\"Class weights:\")\nfor label, weight in zip(target_columns, pos_weight):\n print(f\"{label:20s}: {weight.item():.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.598401Z","iopub.execute_input":"2026-09-12T08:17:19.598759Z","iopub.status.idle":"2026-09-12T08:17:19.620898Z","shell.execute_reply.started":"2026-09-12T08:17:19.598702Z","shell.execute_reply":"2026-09-12T08:17:19.6198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion = nn.BCEWithLogitsLoss(\n pos_weight=pos_weight\n)\n\noptimizer = torch.optim.Adam(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nprint(\"Loss function:\", criterion)\nprint(\"Optimizer:\", optimizer)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:19.622345Z","iopub.execute_input":"2026-09-12T08:17:19.62276Z","iopub.status.idle":"2026-09-12T08:17:22.983706Z","shell.execute_reply.started":"2026-09-12T08:17:19.622711Z","shell.execute_reply":"2026-09-12T08:17:22.982474Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.train()\n\nimages, labels = next(iter(train_loader))\n\noptimizer.zero_grad()\n\noutputs = model(images)\n\nloss = criterion(outputs, labels)\n\nloss.backward()\n\noptimizer.step()\n\nprint(\"Batch image shape:\", images.shape)\nprint(\"Batch label shape:\", labels.shape)\nprint(\"Output shape:\", outputs.shape)\nprint(\"Loss:\", loss.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:22.985103Z","iopub.execute_input":"2026-09-12T08:17:22.985676Z","iopub.status.idle":"2026-09-12T08:17:24.729403Z","shell.execute_reply.started":"2026-09-12T08:17:22.985645Z","shell.execute_reply":"2026-09-12T08:17:24.728198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = RSNAKnee3DCNN(num_classes=len(target_columns))\n\ncriterion = nn.BCEWithLogitsLoss(\n pos_weight=pos_weight\n)\n\noptimizer = torch.optim.Adam(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nnum_epochs = 10\n\nprint(\"Model reset.\")\nprint(\"Epochs:\", num_epochs)\nprint(\"Learning rate:\", 1e-4)\nprint(\"Batch size:\", train_loader.batch_size)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:24.730896Z","iopub.execute_input":"2026-09-12T08:17:24.731366Z","iopub.status.idle":"2026-09-12T08:17:24.74423Z","shell.execute_reply.started":"2026-09-12T08:17:24.73132Z","shell.execute_reply":"2026-09-12T08:17:24.74302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_losses = []\nval_losses = []\n\nfor epoch in range(num_epochs):\n model.train()\n running_train_loss = 0.0\n for images, labels in train_loader:\n optimizer.zero_grad()\n outputs = model(images)\n loss = criterion(outputs, labels)\n loss.backward()\n optimizer.step()\n running_train_loss += loss.item()\n\n epoch_train_loss = running_train_loss / len(train_loader)\n train_losses.append(epoch_train_loss)\n \n model.eval()\n running_val_loss = 0.0\n\n with torch.no_grad():\n for images, labels in val_loader:\n outputs = model(images)\n loss = criterion(outputs, labels)\n running_val_loss += loss.item()\n\n epoch_val_loss = running_val_loss / len(val_loader)\n val_losses.append(epoch_val_loss)\n print(\n f\"Epoch [{epoch + 1}/{num_epochs}] \"\n f\"Train Loss: {epoch_train_loss:.4f} | \"\n f\"Val Loss: {epoch_val_loss:.4f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:17:24.745825Z","iopub.execute_input":"2026-09-12T08:17:24.746649Z","iopub.status.idle":"2026-09-12T08:24:57.977253Z","shell.execute_reply.started":"2026-09-12T08:17:24.746618Z","shell.execute_reply":"2026-09-12T08:24:57.97631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nmodel.eval()\n\nall_labels = []\nall_probs = []\n\nwith torch.no_grad():\n\n for images, labels in val_loader:\n\n outputs = model(images)\n probabilities = torch.sigmoid(outputs)\n\n all_labels.append(labels.numpy())\n all_probs.append(probabilities.numpy())\n\nall_labels = np.concatenate(all_labels, axis=0)\nall_probs = np.concatenate(all_probs, axis=0)\n\nprint(\"Validation labels shape:\", all_labels.shape)\nprint(\"Validation predictions shape:\", all_probs.shape)\n\nprint(\"\\nPer-abnormality ROC-AUC:\")\n\nauc_scores = {}\n\nfor i, label in enumerate(target_columns):\n\n unique_classes = np.unique(all_labels[:, i])\n\n if len(unique_classes) < 2:\n auc_scores[label] = np.nan\n print(f\"{label:20s}: N/A (only one class in validation)\")\n else:\n auc = roc_auc_score(\n all_labels[:, i],\n all_probs[:, i]\n )\n auc_scores[label] = auc\n print(f\"{label:20s}: {auc:.4f}\")\n\nvalid_auc = [\n score for score in auc_scores.values()\n if not np.isnan(score)\n]\n\nprint(\n f\"\\nMean ROC-AUC (valid labels): \"\n f\"{np.mean(valid_auc):.4f}\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:24:57.978795Z","iopub.execute_input":"2026-09-12T08:24:57.979331Z","iopub.status.idle":"2026-09-12T08:25:05.317895Z","shell.execute_reply.started":"2026-09-12T08:24:57.979298Z","shell.execute_reply":"2026-09-12T08:25:05.316705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PlaneEncoder3D(nn.Module):\n def __init__(self):\n super().__init__()\n\n self.features = nn.Sequential(\n nn.Conv3d(1, 16, kernel_size=3, padding=1),\n nn.BatchNorm3d(16),\n nn.ReLU(),\n nn.MaxPool3d(2),\n\n nn.Conv3d(16, 32, kernel_size=3, padding=1),\n nn.BatchNorm3d(32),\n nn.ReLU(),\n nn.MaxPool3d(2),\n\n nn.Conv3d(32, 64, kernel_size=3, padding=1),\n nn.BatchNorm3d(64),\n nn.ReLU(),\n\n nn.AdaptiveAvgPool3d(1)\n )\n\n def forward(self, x):\n x = self.features(x)\n return x.flatten(1)\n\n\nclass RSNAKneeMultiPlane3DCNN(nn.Module):\n def __init__(self, num_classes=12):\n super().__init__()\n\n # One encoder for each anatomical plane\n self.axial_encoder = PlaneEncoder3D()\n self.coronal_encoder = PlaneEncoder3D()\n self.sagittal_encoder = PlaneEncoder3D()\n\n # Fuse the learned representations\n self.classifier = nn.Sequential(\n nn.Linear(64 * 3, 64),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(64, num_classes)\n )\n\n def forward(self, x):\n # x shape: [batch, 3, depth, height, width]\n\n axial = x[:, 0:1, :, :, :]\n coronal = x[:, 1:2, :, :, :]\n sagittal = x[:, 2:3, :, :, :]\n\n axial_features = self.axial_encoder(axial)\n coronal_features = self.coronal_encoder(coronal)\n sagittal_features = self.sagittal_encoder(sagittal)\n\n fused_features = torch.cat(\n [axial_features, coronal_features, sagittal_features],\n dim=1\n )\n\n return self.classifier(fused_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:25:05.31965Z","iopub.execute_input":"2026-09-12T08:25:05.320715Z","iopub.status.idle":"2026-09-12T08:25:05.333373Z","shell.execute_reply.started":"2026-09-12T08:25:05.320676Z","shell.execute_reply":"2026-09-12T08:25:05.33206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create the new multi-plane model\nmulti_plane_model = RSNAKneeMultiPlane3DCNN(\n num_classes=len(target_columns)\n)\nimages, labels = next(iter(train_loader))\n\nwith torch.no_grad():\n outputs = multi_plane_model(images)\n\nprint(\"Input shape :\", images.shape)\nprint(\"Label shape :\", labels.shape)\nprint(\"Output shape:\", outputs.shape)\nprint(\"Output dtype:\", outputs.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:25:05.334927Z","iopub.execute_input":"2026-09-12T08:25:05.335475Z","iopub.status.idle":"2026-09-12T08:25:08.4817Z","shell.execute_reply.started":"2026-09-12T08:25:05.335433Z","shell.execute_reply":"2026-09-12T08:25:08.4806Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"multi_plane_model = RSNAKneeMultiPlane3DCNN(\n num_classes=len(target_columns)\n)\n\n# Use the same class-imbalance handling as the baseline\nmulti_plane_criterion = nn.BCEWithLogitsLoss(\n pos_weight=pos_weight\n)\n\nmulti_plane_optimizer = torch.optim.Adam(\n multi_plane_model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nprint(\"Model, loss function, and optimizer initialized.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:25:08.483113Z","iopub.execute_input":"2026-09-12T08:25:08.483542Z","iopub.status.idle":"2026-09-12T08:25:08.49877Z","shell.execute_reply.started":"2026-09-12T08:25:08.483511Z","shell.execute_reply":"2026-09-12T08:25:08.497343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images, labels = next(iter(train_loader))\nmulti_plane_model.train()\noutputs = multi_plane_model(images)\nloss = multi_plane_criterion(outputs, labels)\nmulti_plane_optimizer.zero_grad()\nloss.backward()\nmulti_plane_optimizer.step()\n\nprint(\"Batch image shape :\", images.shape)\nprint(\"Batch label shape :\", labels.shape)\nprint(\"Output shape :\", outputs.shape)\nprint(\"Training loss :\", loss.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:25:08.500563Z","iopub.execute_input":"2026-09-12T08:25:08.501073Z","iopub.status.idle":"2026-09-12T08:25:11.116191Z","shell.execute_reply.started":"2026-09-12T08:25:08.500989Z","shell.execute_reply":"2026-09-12T08:25:11.114918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_epochs = 10\n\nmulti_plane_train_losses = []\nmulti_plane_val_losses = []\n\nfor epoch in range(num_epochs):\n multi_plane_model.train()\n running_train_loss = 0.0\n\n for images, labels in train_loader:\n multi_plane_optimizer.zero_grad()\n\n outputs = multi_plane_model(images)\n loss = multi_plane_criterion(outputs, labels)\n\n loss.backward()\n multi_plane_optimizer.step()\n\n running_train_loss += loss.item()\n\n avg_train_loss = running_train_loss / len(train_loader)\n multi_plane_model.eval()\n running_val_loss = 0.0\n\n with torch.no_grad():\n for images, labels in val_loader:\n outputs = multi_plane_model(images)\n loss = multi_plane_criterion(outputs, labels)\n\n running_val_loss += loss.item()\n\n avg_val_loss = running_val_loss / len(val_loader)\n\n multi_plane_train_losses.append(avg_train_loss)\n multi_plane_val_losses.append(avg_val_loss)\n\n print(\n f\"Epoch {epoch + 1:02d}/{num_epochs} | \"\n f\"Train Loss: {avg_train_loss:.4f} | \"\n f\"Val Loss: {avg_val_loss:.4f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:25:11.117527Z","iopub.execute_input":"2026-09-12T08:25:11.117909Z","iopub.status.idle":"2026-09-12T08:37:26.452826Z","shell.execute_reply.started":"2026-09-12T08:25:11.117868Z","shell.execute_reply":"2026-09-12T08:37:26.451702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nmulti_plane_model.eval()\n\nall_labels_mp = []\nall_probs_mp = []\n\nwith torch.no_grad():\n for images, labels in val_loader:\n outputs = multi_plane_model(images)\n probabilities = torch.sigmoid(outputs)\n\n all_labels_mp.append(labels.numpy())\n all_probs_mp.append(probabilities.numpy())\n\nall_labels_mp = np.concatenate(all_labels_mp, axis=0)\nall_probs_mp = np.concatenate(all_probs_mp, axis=0)\n\nprint(\"Validation labels shape:\", all_labels_mp.shape)\nprint(\"Validation predictions shape:\", all_probs_mp.shape)\n\nprint(\"\\nMulti-plane model ROC-AUC:\")\n\nmp_auc_scores = {}\n\nfor i, label in enumerate(target_columns):\n\n unique_classes = np.unique(all_labels_mp[:, i])\n\n if len(unique_classes) < 2:\n mp_auc_scores[label] = np.nan\n print(f\"{label:20s}: N/A (only one class in validation)\")\n else:\n auc = roc_auc_score(\n all_labels_mp[:, i],\n all_probs_mp[:, i]\n )\n mp_auc_scores[label] = auc\n print(f\"{label:20s}: {auc:.4f}\")\n\nvalid_mp_auc = [\n score for score in mp_auc_scores.values()\n if not np.isnan(score)\n]\n\nprint(f\"\\nMean ROC-AUC (valid labels): {np.mean(valid_mp_auc):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:26.4543Z","iopub.execute_input":"2026-09-12T08:37:26.454723Z","iopub.status.idle":"2026-09-12T08:37:34.539496Z","shell.execute_reply.started":"2026-09-12T08:37:26.454694Z","shell.execute_reply":"2026-09-12T08:37:34.538252Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(\"Files/folders in competition directory:\\n\")\n\nfor item in os.listdir(BASE_PATH):\n path = os.path.join(BASE_PATH, item)\n if os.path.isdir(path):\n print(f\"[DIR ] {item}\")\n else:\n print(f\"[FILE] {item}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:34.540679Z","iopub.execute_input":"2026-09-12T08:37:34.540939Z","iopub.status.idle":"2026-09-12T08:37:34.552783Z","shell.execute_reply.started":"2026-09-12T08:37:34.540915Z","shell.execute_reply":"2026-09-12T08:37:34.551452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = RSNAKnee3DCNN(\n num_classes=len(target_columns)\n)\ncriterion = nn.BCEWithLogitsLoss(\n pos_weight=pos_weight\n)\n\noptimizer = torch.optim.Adam(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\nprint(\"Baseline model restored.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:34.554248Z","iopub.execute_input":"2026-09-12T08:37:34.554707Z","iopub.status.idle":"2026-09-12T08:37:34.572431Z","shell.execute_reply.started":"2026-09-12T08:37:34.554643Z","shell.execute_reply":"2026-09-12T08:37:34.571422Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Total studies:\", len(train))\n\nprint(\"\\nReports available:\")\nprint(train[\"Report\"].notna().sum())\n\nprint(\"\\nReports missing:\")\nprint(train[\"Report\"].isna().sum())\n\nprint(\"\\nStudies with at least one official label:\")\nprint(train[target_columns].notna().any(axis=1).sum())\n\nprint(\"\\nStudies with all 12 official labels:\")\nprint(train[target_columns].notna().all(axis=1).sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:34.573875Z","iopub.execute_input":"2026-09-12T08:37:34.574283Z","iopub.status.idle":"2026-09-12T08:37:34.600892Z","shell.execute_reply.started":"2026-09-12T08:37:34.574241Z","shell.execute_reply":"2026-09-12T08:37:34.599615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"reports = train[\"Report\"].dropna().astype(str)\n\nprint(\"Number of reports:\", len(reports))\n\nprint(\"\\nReport length statistics:\")\nprint(reports.str.len().describe())\n\nprint(\"\\nSample reports:\\n\")\nfor i, report in enumerate(reports.sample(10, random_state=42)):\n print(f\"\\n--- Report {i+1} ---\")\n print(report[:1000])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:34.602299Z","iopub.execute_input":"2026-09-12T08:37:34.602752Z","iopub.status.idle":"2026-09-12T08:37:34.64388Z","shell.execute_reply.started":"2026-09-12T08:37:34.602701Z","shell.execute_reply":"2026-09-12T08:37:34.642599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"language_keywords = {\n \"English\": [\"MRI\", \"knee\", \"ACL\", \"meniscus\", \"tear\", \"effusion\"],\n \"German\": [\"Knie\", \"Meniskus\", \"Kreuzband\", \"Gelenkerguss\", \"Knorpel\"],\n \"Spanish\": [\"rodilla\", \"menisco\", \"ligamento\", \"derrame\", \"rotura\"],\n \"Turkish\": [\"diz\", \"menisküs\", \"bağ\", \"yırtık\", \"eklem\"],\n \"Greek\": [\"γόνατο\", \"μηνίσκου\", \"ρήξη\", \"σύνδεσμο\", \"ύδραρθρο\"],\n \"Croatian/Bosnian/Serbian\": [\"koljena\", \"meniska\", \"ruptura\", \"ligament\", \"izljev\"],\n \"French\": [\"genou\", \"ménisque\", \"rupture\", \"ligament\", \"épanchement\"],\n}\n\nfor language, keywords in language_keywords.items():\n mask = train[\"Report\"].str.lower().apply(\n lambda x: any(keyword.lower() in x for keyword in keywords)\n )\n print(f\"{language:25s}: {mask.sum()} reports\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:34.645598Z","iopub.execute_input":"2026-09-12T08:37:34.646186Z","iopub.status.idle":"2026-09-12T08:37:35.052846Z","shell.execute_reply.started":"2026-09-12T08:37:34.646155Z","shell.execute_reply":"2026-09-12T08:37:35.051716Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports = train[\n train[target_columns].notna().all(axis=1)\n].copy()\n\nprint(\"Officially labeled studies:\", len(labeled_reports))\n\nfor _, row in labeled_reports.head(10).iterrows():\n print(\"\\n\" + \"=\" * 80)\n print(\"Study:\", row[\"StudyInstanceUID\"])\n print(\"\\nOfficial labels:\")\n \n for target in target_columns:\n print(f\" {target:20s}: {int(row[target])}\")\n \n print(\"\\nReport:\")\n print(row[\"Report\"][:1500])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.054265Z","iopub.execute_input":"2026-09-12T08:37:35.054686Z","iopub.status.idle":"2026-09-12T08:37:35.068849Z","shell.execute_reply.started":"2026-09-12T08:37:35.054643Z","shell.execute_reply":"2026-09-12T08:37:35.067868Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_acl(report):\n \n text = str(report).lower()\n\n # Strong negative patterns\n negative_patterns = [\n r\"\\bacl\\b.*\\bintact\\b\",\n r\"\\bacl\\b.*\\bnormal\\b\",\n r\"\\bacl\\b.*\\bpreserved\\b\",\n r\"\\bacl\\b.*\\bwithin normal\\b\",\n r\"\\bno (?:evidence of )?(?:acl )?(?:tear|rupture)\\b\",\n r\"acl.*(?:ohne.*ruptur|intakt)\",\n r\"(?:lca).*?(?:sin alteraciones|intacto)\",\n r\"(?:lca).*?(?:sin signos|sin caracteres).*?(?:rotura|ruptura)\",\n ]\n\n # Strong positive patterns\n positive_patterns = [\n r\"\\bacl\\b.*\\btear\\b\",\n r\"\\bacl\\b.*\\brupture\\b\",\n r\"\\bacl\\b.*\\binjury\\b\",\n r\"\\bacl\\b.*\\bsprain\\b\",\n r\"\\bcomplete tear of the anterior cruciate ligament\\b\",\n r\"\\bpartial tear of the anterior cruciate ligament\\b\",\n r\"\\binterstitial tear of the anterior cruciate ligament\\b\",\n \n # Common non-English terminology\n r\"\\blca\\b.*\\brotura\\b\",\n r\"\\blca\\b.*\\bruptura\\b\",\n r\"\\blca\\b.*\\blesi[oó]n\\b\",\n r\"\\bkreuzband\\b.*\\briss\\b\",\n r\"\\bkreuzband\\b.*\\bruptur\\b\",\n r\"\\bmenisküs\\b\", # deliberately NOT sufficient; placeholder avoided below\n ]\n\n # Remove the deliberately unsafe generic pattern above\n positive_patterns = positive_patterns[:-1]\n\n # Check negatives first\n for pattern in negative_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 0\n\n # Then positives\n for pattern in positive_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.070559Z","iopub.execute_input":"2026-09-12T08:37:35.070955Z","iopub.status.idle":"2026-09-12T08:37:35.094132Z","shell.execute_reply.started":"2026-09-12T08:37:35.070915Z","shell.execute_reply":"2026-09-12T08:37:35.092945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports[\"ACL_extracted\"] = labeled_reports[\"Report\"].apply(extract_acl)\n\ncomparison = labeled_reports[\n [\"ACL\", \"ACL_extracted\"]\n].copy()\n\nprint(comparison.to_string(index=False))\n\nprint(\"\\nAgreement:\")\nprint(\n (comparison[\"ACL\"] == comparison[\"ACL_extracted\"])\n .sum(),\n \"out of\",\n comparison[\"ACL\"].notna().sum()\n)\n\nprint(\"\\nConfusion table:\")\nprint(\n pd.crosstab(\n comparison[\"ACL\"],\n comparison[\"ACL_extracted\"],\n dropna=False\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.096189Z","iopub.execute_input":"2026-09-12T08:37:35.096788Z","iopub.status.idle":"2026-09-12T08:37:35.171841Z","shell.execute_reply.started":"2026-09-12T08:37:35.096755Z","shell.execute_reply":"2026-09-12T08:37:35.170286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\n\ndef get_acl_sentences(report):\n text = str(report)\n\n # Split reports into reasonably sized sentences/lines\n sentences = re.split(r'(?<=[.!?])\\s+|\\n+', text)\n\n acl_terms = [\n \"ACL\",\n \"LCA\",\n \"anterior cruciate\",\n \"cruciate ligament\",\n \"vorderes kreuzband\",\n \"kreuzband\",\n \"ön çapraz\",\n \"ön çapraz bağ\",\n \"πρόσθιου χιαστού\",\n \"πρόσθιος χιαστός\",\n \"prednjeg križnog\",\n \"prednjeg ukrštenog\",\n \"ligamento cruzado anterior\",\n \"ligamentum cruciatum anterius\",\n ]\n\n matched = []\n\n for sentence in sentences:\n sentence_lower = sentence.lower()\n\n if any(term.lower() in sentence_lower for term in acl_terms):\n matched.append(sentence.strip())\n\n return matched\n\n\n# Show ACL-related sentences from the 58 official studies\nfor _, row in labeled_reports.head(10).iterrows():\n print(\"\\n\" + \"=\" * 80)\n print(\"Official ACL:\", int(row[\"ACL\"]))\n\n for sentence in get_acl_sentences(row[\"Report\"]):\n print(\"→\", sentence)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.173116Z","iopub.execute_input":"2026-09-12T08:37:35.173933Z","iopub.status.idle":"2026-09-12T08:37:35.186334Z","shell.execute_reply.started":"2026-09-12T08:37:35.173903Z","shell.execute_reply":"2026-09-12T08:37:35.185222Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for _, row in labeled_reports.iterrows():\n acl_sentences = get_acl_sentences(row[\"Report\"])\n\n print(\"\\n\" + \"=\" * 100)\n print(\"Official ACL:\", int(row[\"ACL\"]))\n\n for sentence in acl_sentences:\n print(\"→\", sentence)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.187369Z","iopub.execute_input":"2026-09-12T08:37:35.187774Z","iopub.status.idle":"2026-09-12T08:37:35.225586Z","shell.execute_reply.started":"2026-09-12T08:37:35.187746Z","shell.execute_reply":"2026-09-12T08:37:35.22459Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_acl_v2(report):\n text = str(report).lower()\n # 1. HIGH-CONFIDENCE NEGATIVE\n negative_patterns = [\n # English\n r\"\\bacl\\b[^.\\n]{0,120}\\bintact\\b\",\n r\"\\bacl\\b[^.\\n]{0,120}\\bnormal\\b\",\n r\"\\bacl\\b[^.\\n]{0,120}\\bunremarkable\\b\",\n r\"\\banterior cruciate ligament\\b[^.\\n]{0,120}\\bintact\\b\",\n r\"\\banterior cruciate ligament\\b[^.\\n]{0,120}\\bnormal\\b\",\n r\"\\banterior cruciate ligament\\b[^.\\n]{0,120}\\bno tear\\b\",\n r\"\\bno tear\\b[^.\\n]{0,120}\\b(?:acl|anterior cruciate ligament)\\b\",\n\n # Spanish\n r\"\\blca\\b[^.\\n]{0,120}\\bsin\\b[^.\\n]{0,40}(?:rotura|ruptura)\",\n r\"\\blca\\b[^.\\n]{0,120}\\bnormal\\b\",\n r\"\\blca\\b[^.\\n]{0,120}\\bintacto\\b\",\n\n # Turkish\n r\"\\bön çapraz bağ\\b[^.\\n]{0,120}\\bnormal\\b\",\n r\"\\bön çapraz bağ\\b[^.\\n]{0,120}\\bkorun\",\n \n # Croatian / Bosnian / Serbian\n r\"\\bprednjeg križnog ligamenta\\b[^.\\n]{0,120}\\buredan\\b\",\n r\"\\bprednjeg križnog ligamenta\\b[^.\\n]{0,120}\\bočuvan\\b\",\n ]\n # 2. HIGH-CONFIDENCE POSITIVE\n positive_patterns = [\n # English\n r\"\\bcomplete tear\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\bcomplete rupture\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bcomplete tear\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bcomplete rupture\\b\",\n\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\btear\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\brupture\\b\",\n\n # High-grade partial tear\n r\"\\bhigh[- ]grade partial[- ]thickness tear\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\bhigh[- ]grade partial[- ]thickness tear\\b[^.\\n]{0,100}\\b\",\n\n # Spanish\n r\"\\brotura\\b[^.\\n]{0,100}\\blca\\b\",\n r\"\\bruptura\\b[^.\\n]{0,100}\\blca\\b\",\n r\"\\blca\\b[^.\\n]{0,100}\\brotura\\b\",\n r\"\\blca\\b[^.\\n]{0,100}\\bruptura\\b\",\n\n # Turkish\n r\"\\bön çapraz bağ\\b[^.\\n]{0,120}\\b(?:tam kat|komplet|tam)\\b[^.\\n]{0,60}(?:yırtık|rüptür)\",\n r\"\\bön çapraz bağ\\b[^.\\n]{0,120}\\b(?:yırtık|rüptür)\\b\",\n\n # Greek\n r\"\\bπρόσθι(?:ου|ος)\\s+χιαστ(?:ού|ός)\\s+συνδέσμου\\b[^.\\n]{0,120}\\bρήξη\\b\",\n r\"\\bρήξη\\b[^.\\n]{0,120}\\bπρόσθι(?:ου|ος)\\s+χιαστ\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bpotpuna ruptura\\b[^.\\n]{0,100}\\bprednjeg križnog ligamenta\\b\",\n r\"\\bruptura\\b[^.\\n]{0,100}\\bprednjeg križnog ligamenta\\b\",\n r\"\\bprednjeg križnog ligamenta\\b[^.\\n]{0,100}\\bruptura\\b\",\n ]\n # 3. IMPORTANT: Ignore low-grade / equivocal findings.\n uncertain_patterns = [\n r\"\\bgrade\\s*[12]\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bgrade\\s*[12]\\b\",\n r\"\\blow[- ]grade\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\binterstitial\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\binterstitial\\b\",\n r\"\\bsprain\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bsprain\\b\",\n r\"\\bsignal\\b[^.\\n]{0,100}\\b(?:acl|anterior cruciate ligament)\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bmyxoid\\b\",\n r\"\\b(?:acl|anterior cruciate ligament)\\b[^.\\n]{0,100}\\bmucoid\\b\",\n ]\n\n # Negative gets priority only when the same local statement\n # clearly says the ACL is normal.\n for pattern in negative_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 0\n\n # Strong positives next\n for pattern in positive_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 1\n\n # Explicitly uncertain findings\n for pattern in uncertain_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return np.nan\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.227609Z","iopub.execute_input":"2026-09-12T08:37:35.228078Z","iopub.status.idle":"2026-09-12T08:37:35.245415Z","shell.execute_reply.started":"2026-09-12T08:37:35.228007Z","shell.execute_reply":"2026-09-12T08:37:35.24424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports[\"ACL_extracted_v2\"] = labeled_reports[\"Report\"].apply(\n extract_acl_v2\n)\n\ncomparison = labeled_reports[\n [\"ACL\", \"ACL_extracted_v2\"]\n].copy()\n\nprint(comparison.to_string(index=False))\n\nprint(\"\\nValue counts:\")\nprint(comparison[\"ACL_extracted_v2\"].value_counts(dropna=False))\n\nprint(\"\\nConfusion table:\")\nprint(\n pd.crosstab(\n comparison[\"ACL\"],\n comparison[\"ACL_extracted_v2\"],\n dropna=False\n )\n)\n\nknown = comparison[\"ACL_extracted_v2\"].notna()\n\nprint(\"\\nCoverage:\",\n f\"{known.sum()}/{len(comparison)} \"\n f\"({known.mean()*100:.1f}%)\")\n\nif known.any():\n print(\"Agreement among extracted cases:\",\n f\"{(comparison.loc[known, 'ACL'] == comparison.loc[known, 'ACL_extracted_v2']).mean()*100:.1f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.246664Z","iopub.execute_input":"2026-09-12T08:37:35.247525Z","iopub.status.idle":"2026-09-12T08:37:35.341654Z","shell.execute_reply.started":"2026-09-12T08:37:35.247492Z","shell.execute_reply":"2026-09-12T08:37:35.340616Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_mcl_sentences(report):\n text = str(report)\n\n sentences = re.split(r'(?<=[.!?])\\s+|\\n+', text)\n\n mcl_terms = [\n \"MCL\",\n \"medial collateral ligament\",\n \"ligamento colateral medial\",\n \"ligamento colateral interno\",\n \"ligamentum collaterale mediale\",\n \"medial collateral\",\n \"medijalnog kolateralnog\",\n \"medijalni kolateralni\",\n \"medialnog kolateralnog\",\n \"medial collateral\",\n \"medial collateral ligament\",\n \"medial collateral ligamen\",\n \"medial collateral ligament\",\n \"medial collateral\",\n \"medial collateral\",\n \"medial collateral\",\n \"medial collateral\",\n \"medial collateral\",\n ]\n\n matched = []\n\n for sentence in sentences:\n s = sentence.lower()\n\n if any(term.lower() in s for term in mcl_terms):\n matched.append(sentence.strip())\n\n return matched\n\n\nfor _, row in labeled_reports.iterrows():\n sentences = get_mcl_sentences(row[\"Report\"])\n\n if sentences:\n print(\"\\n\" + \"=\" * 100)\n print(\"Official MCL:\", int(row[\"MCL\"]))\n\n for sentence in sentences:\n print(\"→\", sentence)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.342999Z","iopub.execute_input":"2026-09-12T08:37:35.343347Z","iopub.status.idle":"2026-09-12T08:37:35.370672Z","shell.execute_reply.started":"2026-09-12T08:37:35.343321Z","shell.execute_reply":"2026-09-12T08:37:35.369415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_mcl_v2(report):\n text = str(report).lower()\n negative_patterns = [\n r\"\\bmcl\\b[^.\\n]{0,100}\\bintact\\b\",\n r\"\\bmcl\\b[^.\\n]{0,100}\\bnormal\\b\",\n r\"\\bmedial collateral ligament\\b[^.\\n]{0,100}\\bintact\\b\",\n r\"\\bmedial collateral ligament\\b[^.\\n]{0,100}\\bnormal\\b\",\n r\"\\bno tear\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n\n # Low-grade findings were officially negative\n r\"\\blow[- ]grade sprain\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n r\"\\blow[- ]grade partial[- ](?:thickness )?tear\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n r\"\\blow[- ]grade partial tear\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n ]\n positive_patterns = [\n # Complete injury\n r\"\\bcomplete rupture\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n r\"\\bcomplete tear\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n\n r\"\\b(?:mcl|medial collateral ligament)\\b[^.\\n]{0,100}\\bcomplete rupture\\b\",\n r\"\\b(?:mcl|medial collateral ligament)\\b[^.\\n]{0,100}\\bcomplete tear\\b\",\n\n # High-grade tear\n r\"\\bhigh[- ]grade tear\\b[^.\\n]{0,100}\\b(?:mcl|medial collateral ligament)\\b\",\n r\"\\b(?:mcl|medial collateral ligament)\\b[^.\\n]{0,100}\\bhigh[- ]grade tear\\b\",\n\n # Spanish pattern observed in an official-positive case\n r\"\\besguince grado ii\\b[^.\\n]{0,120}\\bligamento colateral medial\\b\",\n r\"\\besguince grado 2\\b[^.\\n]{0,120}\\bligamento colateral medial\\b\",\n\n # Explicit Spanish rupture\n r\"\\brotura\\b[^.\\n]{0,100}\\bligamento colateral medial\\b\",\n r\"\\bruptura\\b[^.\\n]{0,100}\\bligamento colateral medial\\b\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 0\n\n for pattern in positive_patterns:\n if re.search(pattern, text, flags=re.IGNORECASE):\n return 1\n\n # Everything else remains unknown\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.372215Z","iopub.execute_input":"2026-09-12T08:37:35.372731Z","iopub.status.idle":"2026-09-12T08:37:35.399655Z","shell.execute_reply.started":"2026-09-12T08:37:35.372641Z","shell.execute_reply":"2026-09-12T08:37:35.398518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports[\"MCL_extracted_v2\"] = labeled_reports[\"Report\"].apply(\n extract_mcl_v2\n)\n\ncomparison = labeled_reports[\n [\"MCL\", \"MCL_extracted_v2\"]\n].copy()\n\nprint(comparison.to_string(index=False))\n\nprint(\"\\nValue counts:\")\nprint(comparison[\"MCL_extracted_v2\"].value_counts(dropna=False))\n\nprint(\"\\nConfusion table:\")\nprint(\n pd.crosstab(\n comparison[\"MCL\"],\n comparison[\"MCL_extracted_v2\"],\n dropna=False\n )\n)\n\nknown = comparison[\"MCL_extracted_v2\"].notna()\n\nprint(\"\\nCoverage:\",\n f\"{known.sum()}/{len(comparison)} \"\n f\"({known.mean()*100:.1f}%)\")\n\nif known.any():\n print(\n \"Agreement among extracted cases:\",\n f\"{(comparison.loc[known, 'MCL'] == comparison.loc[known, 'MCL_extracted_v2']).mean()*100:.1f}%\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.40129Z","iopub.execute_input":"2026-09-12T08:37:35.401678Z","iopub.status.idle":"2026-09-12T08:37:35.471922Z","shell.execute_reply.started":"2026-09-12T08:37:35.40164Z","shell.execute_reply":"2026-09-12T08:37:35.470953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport pandas as pd\n\ndef get_medial_meniscus_sentences(row):\n report = str(row[\"Report\"])\n \n sentences = re.split(r'(?<=[.!?])\\s+', report)\n \n keywords = [\n \"medial meniscus\",\n \"medialis meniscus\",\n \"meniscus medialis\",\n \"medial menisc\",\n \"menisco medial\",\n \"menisco interno\",\n \"medial menisk\",\n \"meniskus medialis\",\n \"meniskus medial\",\n \"meniskus medialis\",\n \"medialni menisk\",\n \"medijalni menisk\",\n \"medial menisküs\",\n \"medialni meniskus\",\n \"medialni menisk\"\n ]\n \n matched = []\n \n for sentence in sentences:\n sentence_lower = sentence.lower()\n \n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n \n return \" \".join(matched)\n\n\nmedial_meniscus_cases = labeled_reports[\n labeled_reports[\"Medial Meniscus\"].notna()\n].copy()\n\nmedial_meniscus_cases[\"Medial_Meniscus_sentences\"] = (\n medial_meniscus_cases.apply(\n get_medial_meniscus_sentences,\n axis=1\n )\n)\n\nprint(\n medial_meniscus_cases[\n [\"Medial Meniscus\", \"Medial_Meniscus_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.473201Z","iopub.execute_input":"2026-09-12T08:37:35.473553Z","iopub.status.idle":"2026-09-12T08:37:35.495762Z","shell.execute_reply.started":"2026-09-12T08:37:35.473526Z","shell.execute_reply":"2026-09-12T08:37:35.494636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_medial_meniscus_v2(report):\n text = str(report).lower()\n\n negative_patterns = [\n # English\n r\"\\bmedial meniscus\\b.*\\b(?:intact|normal)\\b\",\n r\"\\bmedial meniscus\\b.*\\bno tear\\b\",\n r\"\\bmedial meniscus\\b.*\\bnot torn\\b\",\n r\"\\bmedial meniscus\\b.*\\bwithout.*tear\\b\",\n\n # Spanish\n r\"menisco (?:medial|interno).*sin.*(?:rotura|desgarro)\",\n r\"sin signos de (?:rotura|desgarro).*menisco\",\n\n # Turkish\n r\"medial menisküs.*(?:normal|intakt)\",\n r\"medial menisküs.*yırtık.*(?:yok|izlenmedi)\",\n\n # Croatian / Bosnian / Serbian\n r\"medijalni menisk.*bez.*ruptur\",\n r\"medijalni menisk.*bez.*rascjep\",\n r\"medijalni meniskus.*bez.*ruptur\",\n\n # Explicit degeneration without tear\n r\"degeneration.*without.*tear\",\n r\"degeneración.*sin.*rotura\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n uncertain_patterns = [\n r\"\\bsuspect\\b.*\\b(?:tear|lesion)\\b\",\n r\"\\bsuspicious for\\b.*\\btear\\b\",\n r\"\\br/o\\b.*\\b(?:tear|lesion)\\b\",\n r\"\\bcould represent\\b.*\\blesion\\b\",\n r\"\\bpossible\\b.*\\btear\\b\",\n\n # Degeneration by itself is NOT a tear\n r\"\\bmucoid degeneration\\b\",\n r\"\\bintrasubstance degeneration\\b\",\n r\"\\bgrade [i1-ii2]+\\s+degeneration\\b\",\n r\"degeneración\",\n r\"dejenerasyon\",\n ]\n\n\n sentences = re.split(r'(?<=[.!?])\\s+|\\n+', text)\n\n medial_terms = [\n \"medial meniscus\",\n \"meniscus medialis\",\n \"medialis meniscus\",\n \"medial menisk\",\n \"menisco medial\",\n \"menisco interno\",\n \"meniskus medialis\",\n \"medial menisküs\",\n \"medijalni menisk\",\n \"medijalni meniskus\",\n ]\n\n strong_tear_terms = [\n # English\n \"complete tear\",\n \"complete tearing\",\n \"radial tear\",\n \"root tear\",\n \"bucket-handle tear\",\n \"bucket handle tear\",\n \"complex tear\",\n \"vertical tear\",\n \"horizontal tear\",\n \"tear of the medial meniscus\",\n \"medial meniscus tear\",\n\n # Spanish\n \"rotura del menisco medial\",\n \"rotura del menisco interno\",\n \"rotura de menisco interno\",\n \"desgarro del menisco medial\",\n \"desgarro del menisco interno\",\n\n # Turkish\n \"radial yırtık\",\n \"horizontal yırtık\",\n \"vertikal yırtık\",\n \"kova sapı yırtığı\",\n \"yırtık ile uyumlu\",\n\n # Croatian/Bosnian/Serbian\n \"ruptura medijalnog menisk\",\n \"ruptura medijalni menisk\",\n \"rascjep medijalnog menisk\",\n ]\n\n for sentence in sentences:\n has_medial = any(term in sentence for term in medial_terms)\n has_tear = any(term in sentence for term in strong_tear_terms)\n\n if has_medial and has_tear:\n\n # Avoid uncertain formulations\n if any(\n term in sentence\n for term in [\n \"suspect\",\n \"suspicious\",\n \"r/o\",\n \"could represent\",\n \"possible\",\n \"podozren\",\n \"može predstavljati\",\n ]\n ):\n continue\n\n return 1\n\n for sentence in sentences:\n has_medial = any(term in sentence for term in medial_terms)\n\n if has_medial and any(\n term in sentence\n for term in [\n \"no tear\",\n \"not torn\",\n \"without tear\",\n \"sin signos de rotura\",\n \"sin criterios\",\n \"sin desgarro\",\n \"sin rotura\",\n \"yırtık yok\",\n \"normal\",\n \"intact\",\n \"intacto\",\n \"intacta\",\n ]\n ):\n return 0\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.497278Z","iopub.execute_input":"2026-09-12T08:37:35.49776Z","iopub.status.idle":"2026-09-12T08:37:35.516449Z","shell.execute_reply.started":"2026-09-12T08:37:35.497714Z","shell.execute_reply":"2026-09-12T08:37:35.515378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports[\"Medial_Meniscus_extracted_v2\"] = (\n labeled_reports[\"Report\"].apply(\n extract_medial_meniscus_v2\n )\n)\n\ncomparison = labeled_reports[\n [\"Medial Meniscus\", \"Medial_Meniscus_extracted_v2\"]\n].copy()\n\nknown = comparison[\"Medial_Meniscus_extracted_v2\"].notna()\n\nofficial = comparison.loc[known, \"Medial Meniscus\"]\npred = comparison.loc[known, \"Medial_Meniscus_extracted_v2\"]\n\nprint(comparison.to_string(index=False))\n\nprint(\"\\nValue counts:\")\nprint(\n comparison[\"Medial_Meniscus_extracted_v2\"]\n .value_counts(dropna=False)\n)\n\nprint(\"\\nConfusion table:\")\nprint(pd.crosstab(official, pred))\n\nprint(\n f\"\\nCoverage: {known.sum()}/{len(comparison)} \"\n f\"({known.mean()*100:.1f}%)\"\n)\n\nif known.sum() > 0:\n agreement = (official == pred).mean() * 100\n print(\n f\"Agreement among extracted cases: \"\n f\"{agreement:.1f}%\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.517547Z","iopub.execute_input":"2026-09-12T08:37:35.517856Z","iopub.status.idle":"2026-09-12T08:37:35.569305Z","shell.execute_reply.started":"2026-09-12T08:37:35.51783Z","shell.execute_reply":"2026-09-12T08:37:35.568435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_v2 = comparison[\n comparison[\"Medial_Meniscus_extracted_v2\"].notna()\n & (\n comparison[\"Medial Meniscus\"]\n != comparison[\"Medial_Meniscus_extracted_v2\"]\n )\n].copy()\n\nmismatch_v2[\"Report\"] = labeled_reports.loc[\n mismatch_v2.index, \"Report\"\n]\n\nprint(\n mismatch_v2[\n [\n \"Medial Meniscus\",\n \"Medial_Meniscus_extracted_v2\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.57079Z","iopub.execute_input":"2026-09-12T08:37:35.571215Z","iopub.status.idle":"2026-09-12T08:37:35.599651Z","shell.execute_reply.started":"2026-09-12T08:37:35.571175Z","shell.execute_reply":"2026-09-12T08:37:35.598542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_lateral_meniscus_sentences(row):\n report = str(row[\"Report\"])\n \n sentences = re.split(r'(?<=[.!?])\\s+', report)\n \n keywords = [\n \"lateral meniscus\",\n \"meniscus lateralis\",\n \"lateralis meniscus\",\n \"lateral menisc\",\n \"menisco lateral\",\n \"menisco externo\",\n \"meniskus lateralis\",\n \"lateral menisküs\",\n \"lateralni menisk\",\n \"lateralni meniskus\",\n \"laterální menisk\",\n ]\n \n matched = []\n \n for sentence in sentences:\n sentence_lower = sentence.lower()\n \n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n \n return \" \".join(matched)\n\n\nlateral_meniscus_cases = labeled_reports[\n labeled_reports[\"Lateral Meniscus\"].notna()\n].copy()\n\nlateral_meniscus_cases[\"Lateral_Meniscus_sentences\"] = (\n lateral_meniscus_cases.apply(\n get_lateral_meniscus_sentences,\n axis=1\n )\n)\n\nprint(\n lateral_meniscus_cases[\n [\"Lateral Meniscus\", \"Lateral_Meniscus_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.601109Z","iopub.execute_input":"2026-09-12T08:37:35.601819Z","iopub.status.idle":"2026-09-12T08:37:35.64178Z","shell.execute_reply.started":"2026-09-12T08:37:35.601775Z","shell.execute_reply":"2026-09-12T08:37:35.640592Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_lateral_meniscus_v1(report):\n text = str(report).lower()\n\n if (\n \"lateral menisküs\" in text\n and \"kova sapı yırtığı\" in text\n ):\n return 1\n\n negative_patterns = [\n # English\n r\"\\blateral meniscus\\b.*\\bno tear\\b\",\n r\"\\bno lateral meniscal tear\\b\",\n r\"\\bno evidence of tear\\b.*\\b(?:lateral|meniscus)\\b\",\n r\"\\blateral meniscus\\b.*\\bnormal\\b\",\n r\"\\blateral meniscus\\b.*\\bintact\\b\",\n r\"\\bnormal medial and lateral menisci\\b\",\n\n # Spanish\n r\"menisco lateral.*sin signos de rotura\",\n r\"menisco lateral.*sin.*rotura\",\n r\"menisco externo.*sin.*rotura\",\n\n # Turkish\n r\"lateral menisküs.*normal\",\n r\"lateral menisküs.*yırtık.*yok\",\n\n # Croatian / Bosnian / Serbian\n r\"lateralni menisk.*bez znakova rupture\",\n r\"lateralni menisk.*bez.*rupture\",\n r\"lateralni menisk.*bez.*degeneracije ili rupture\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n sentences = re.split(\n r'(?<=[.!?])\\s+|\\n+',\n text\n )\n\n lateral_terms = [\n \"lateral meniscus\",\n \"lateral menisc\",\n \"meniscus lateralis\",\n \"lateralis meniscus\",\n \"menisco lateral\",\n \"menisco externo\",\n \"lateral menisküs\",\n \"lateralni menisk\",\n \"lateralni meniskus\",\n ]\n\n strong_tear_terms = [\n # English\n \"complete tear\",\n \"complete tearing\",\n \"radial tear\",\n \"root tear\",\n \"bucket-handle tear\",\n \"bucket handle tear\",\n \"buckethandle tear\",\n \"complex tear\",\n \"vertical tear\",\n \"horizontal tear\",\n \"oblique tear\",\n \"tear of anterior horn\",\n \"tear of the lateral meniscus\",\n \"lateral meniscus tear\",\n \"recurrent tear\",\n\n # Spanish\n \"rotura del menisco lateral\",\n \"rotura de menisco lateral\",\n \"rotura del menisco externo\",\n \"rotura de menisco externo\",\n\n # Turkish\n \"radial yırtık\",\n \"horizontal yırtık\",\n \"kova sapı yırtığı\",\n \"yırtık ile uyumlu\",\n\n # Croatian / Bosnian / Serbian\n \"ruptura lateralnog menisk\",\n \"ruptura lateralni menisk\",\n ]\n\n for sentence in sentences:\n\n has_lateral = any(\n term in sentence\n for term in lateral_terms\n )\n\n has_tear = any(\n term in sentence\n for term in strong_tear_terms\n )\n\n if has_lateral and has_tear:\n\n # Explicit uncertainty → unknown\n uncertain_terms = [\n \"suspected\",\n \"suspicious\",\n \"possible\",\n \"r/o\",\n \"could represent\",\n ]\n\n if any(\n term in sentence\n for term in uncertain_terms\n ):\n continue\n\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.643355Z","iopub.execute_input":"2026-09-12T08:37:35.643796Z","iopub.status.idle":"2026-09-12T08:37:35.667271Z","shell.execute_reply.started":"2026-09-12T08:37:35.643728Z","shell.execute_reply":"2026-09-12T08:37:35.665966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_reports[\"Lateral_Meniscus_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(\n extract_lateral_meniscus_v1\n )\n)\n\ncomparison = labeled_reports[\n [\"Lateral Meniscus\",\n \"Lateral_Meniscus_extracted_v1\"]\n].copy()\n\nknown = comparison[\n \"Lateral_Meniscus_extracted_v1\"\n].notna()\n\nofficial = comparison.loc[\n known, \"Lateral Meniscus\"\n]\n\npred = comparison.loc[\n known, \"Lateral_Meniscus_extracted_v1\"\n]\n\nprint(comparison.to_string(index=False))\n\nprint(\"\\nValue counts:\")\nprint(\n comparison[\n \"Lateral_Meniscus_extracted_v1\"\n ].value_counts(dropna=False)\n)\n\nprint(\"\\nConfusion table:\")\nprint(pd.crosstab(official, pred))\n\nprint(\n f\"\\nCoverage: {known.sum()}/{len(comparison)} \"\n f\"({known.mean()*100:.1f}%)\"\n)\n\nif known.sum() > 0:\n agreement = (official == pred).mean() * 100\n print(\n f\"Agreement among extracted cases: \"\n f\"{agreement:.1f}%\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.668269Z","iopub.execute_input":"2026-09-12T08:37:35.66854Z","iopub.status.idle":"2026-09-12T08:37:35.726799Z","shell.execute_reply.started":"2026-09-12T08:37:35.668515Z","shell.execute_reply":"2026-09-12T08:37:35.725707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_lat = comparison[\n comparison[\"Lateral_Meniscus_extracted_v1\"].notna()\n & (\n comparison[\"Lateral Meniscus\"]\n != comparison[\"Lateral_Meniscus_extracted_v1\"]\n )\n].copy()\n\nmismatch_lat[\"Report\"] = labeled_reports.loc[\n mismatch_lat.index, \"Report\"\n]\n\nprint(\n mismatch_lat[\n [\n \"Lateral Meniscus\",\n \"Lateral_Meniscus_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.728113Z","iopub.execute_input":"2026-09-12T08:37:35.729103Z","iopub.status.idle":"2026-09-12T08:37:35.740435Z","shell.execute_reply.started":"2026-09-12T08:37:35.728992Z","shell.execute_reply":"2026-09-12T08:37:35.739238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_medial_oa_sentences(row):\n report = str(row[\"Report\"]).replace(\"\\n\", \" \")\n \n sentences = re.split(r'(?<=[.!?])\\s+', report)\n \n keywords = [\n # English\n \"medial osteoarthritis\",\n \"medial oa\",\n \"medial compartment\",\n \"medial femorotibial\",\n \"medial femorotibial compartment\",\n \"medial femoral condyle\",\n \"medial tibial plateau\",\n \"medial joint space\",\n \"medial joint-space\",\n \"medial compartment cartilage\",\n \"medial cartilage\",\n \"medial chondrosis\",\n \"medial chondropathy\",\n \"medial degenerative\",\n \"osteoarthritis medial\",\n\n # Spanish\n \"compartimento medial\",\n \"femorotibial medial\",\n \"cóndilo femoral medial\",\n \"platillo tibial medial\",\n\n # Turkish\n \"medial kompartman\",\n \"medial femorotibial\",\n \"medial kondil\",\n \"medial tibial plato\",\n\n # Croatian / Bosnian / Serbian\n \"medijalni kompart\",\n \"medijalni kondil\",\n \"medijalni tibijalni\",\n ]\n \n matched = []\n \n for sentence in sentences:\n sentence_lower = sentence.lower()\n \n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n \n return \" \".join(matched)\n\n\nmedial_oa_cases = labeled_reports[\n labeled_reports[\"Medial OA\"].notna()\n].copy()\n\nmedial_oa_cases[\"Medial_OA_sentences\"] = (\n medial_oa_cases.apply(\n get_medial_oa_sentences,\n axis=1\n )\n)\n\nprint(\n medial_oa_cases[\n [\"Medial OA\", \"Medial_OA_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.742647Z","iopub.execute_input":"2026-09-12T08:37:35.743155Z","iopub.status.idle":"2026-09-12T08:37:35.786537Z","shell.execute_reply.started":"2026-09-12T08:37:35.743107Z","shell.execute_reply":"2026-09-12T08:37:35.785588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Recreate the clean labeled reports dataframe\nlabeled_reports = train[\n train[target_columns].notna().all(axis=1)\n].copy()\n\nprint(labeled_reports.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.787735Z","iopub.execute_input":"2026-09-12T08:37:35.788165Z","iopub.status.idle":"2026-09-12T08:37:35.807875Z","shell.execute_reply.started":"2026-09-12T08:37:35.788118Z","shell.execute_reply":"2026-09-12T08:37:35.806847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"[name for name in globals() if \"medial_oa\" in name.lower()]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.809476Z","iopub.execute_input":"2026-09-12T08:37:35.809758Z","iopub.status.idle":"2026-09-12T08:37:35.831237Z","shell.execute_reply.started":"2026-09-12T08:37:35.809731Z","shell.execute_reply":"2026-09-12T08:37:35.830245Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_medial_oa_v2(report):\n text = str(report).lower()\n negative_patterns = [\n # English\n r\"medial compartment cartilage\\s*:\\s*intact\",\n r\"medial compartment cartilage.*\\bnormal\\b\",\n r\"medial cartilage.*\\bintact\\b\",\n r\"medial cartilage.*\\bnormal\\b\",\n r\"medial compartment.*no.*chondrosis\",\n r\"medial compartment.*without.*chondral\",\n r\"medial compartment.*no.*cartilage loss\",\n r\"medial compartment.*preserved\",\n\n # Spanish\n r\"compartimento femorotibial medial.*sin alteraciones\",\n r\"compartimento medial.*sin.*condropat\",\n r\"compartimento medial.*cart[ií]lago.*normal\",\n r\"cart[ií]lago.*compartimento medial.*normal\",\n\n # Turkish\n r\"medial kompartman.*normal\",\n r\"medial kompartman.*intakt\",\n r\"medial kompartman.*kıkırdak.*normal\",\n r\"medial kompartman.*kıkırdak.*sağlam\",\n\n # Croatian / Bosnian / Serbian\n r\"medijalni kompart.*bez.*promjena\",\n r\"medijalni kompart.*uredan\",\n r\"medijalni kompartment.*bez.*promjena\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n positive_patterns = [\n # English\n r\"\\bmedial osteoarthritis\\b\",\n r\"\\bosteoarthritis\\b.*\\bmedial compartment\\b\",\n r\"\\bmedial compartment\\b.*\\bosteoarthritis\\b\",\n r\"\\boa of medial compartment\\b\",\n r\"\\bmedial oa\\b\",\n\n # Spanish\n r\"osteoartritis.*compartimento medial\",\n r\"artrosis.*compartimento medial\",\n r\"oa femorotibial medial\",\n\n # Turkish\n r\"medial kompartman.*osteoartrit\",\n r\"medial kompartman.*artroz\",\n\n # Croatian / Bosnian / Serbian\n r\"medijalni kompart.*artroz\",\n r\"medijalni kompart.*osteoartr\",\n\n # Strong cartilage loss / degeneration\n r\"full[- ]thickness cartilage loss.*medial\",\n r\"high[- ]grade cartilage.*medial\",\n r\"high[- ]grade.*chondrosis.*medial\",\n r\"diffuse high[- ]grade.*chondrosis.*medial\",\n\n # Spanish high-grade chondropathy\n r\"condropat[ií]a.*grado [3-4].*c[oó]ndilo femoral medial\",\n r\"condropat[ií]a.*grado [3-4].*compartimento medial\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.832604Z","iopub.execute_input":"2026-09-12T08:37:35.832968Z","iopub.status.idle":"2026-09-12T08:37:35.855674Z","shell.execute_reply.started":"2026-09-12T08:37:35.832932Z","shell.execute_reply":"2026-09-12T08:37:35.854393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison = labeled_reports.copy()\n\ncomparison[\"Medial_OA_extracted_v2\"] = (\n labeled_reports[\"Report\"].apply(extract_medial_oa_v2)\n)\n\nprint(comparison[\"Medial_OA_extracted_v2\"].value_counts(dropna=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.868644Z","iopub.execute_input":"2026-09-12T08:37:35.869064Z","iopub.status.idle":"2026-09-12T08:37:35.910541Z","shell.execute_reply.started":"2026-09-12T08:37:35.869018Z","shell.execute_reply":"2026-09-12T08:37:35.909283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mask = comparison[\"Medial_OA_extracted_v2\"].notna()\n\nprint(\"\\nConfusion:\")\nprint(\n pd.crosstab(\n labeled_reports.loc[mask, \"Medial OA\"],\n comparison.loc[mask, \"Medial_OA_extracted_v2\"],\n rownames=[\"Official\"],\n colnames=[\"Extracted\"]\n )\n)\n\nprint(\n \"\\nCoverage:\",\n mask.sum(),\n \"/\",\n len(comparison),\n \"=\",\n round(mask.mean() * 100, 1),\n \"%\"\n)\n\nprint(\n \"Agreement:\",\n round(\n (\n labeled_reports.loc[mask, \"Medial OA\"]\n ==\n comparison.loc[mask, \"Medial_OA_extracted_v2\"]\n ).mean() * 100,\n 1\n ),\n \"%\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.912006Z","iopub.execute_input":"2026-09-12T08:37:35.913023Z","iopub.status.idle":"2026-09-12T08:37:35.943958Z","shell.execute_reply.started":"2026-09-12T08:37:35.912974Z","shell.execute_reply":"2026-09-12T08:37:35.942629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_medoa_v2 = comparison[\n comparison[\"Medial_OA_extracted_v2\"].notna()\n & (\n labeled_reports[\"Medial OA\"]\n != comparison[\"Medial_OA_extracted_v2\"]\n )\n].copy()\n\nmismatch_medoa_v2[\"Medial OA\"] = labeled_reports.loc[\n mismatch_medoa_v2.index, \"Medial OA\"\n]\n\nmismatch_medoa_v2[\"Report\"] = labeled_reports.loc[\n mismatch_medoa_v2.index, \"Report\"\n]\n\nprint(\n mismatch_medoa_v2[\n [\n \"Medial OA\",\n \"Medial_OA_extracted_v2\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.945462Z","iopub.execute_input":"2026-09-12T08:37:35.946089Z","iopub.status.idle":"2026-09-12T08:37:35.971869Z","shell.execute_reply.started":"2026-09-12T08:37:35.946047Z","shell.execute_reply":"2026-09-12T08:37:35.970618Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lateral_oa_inspection = labeled_reports[\n [\"Lateral OA\", \"Report\"]\n].copy()\n\nprint(\n lateral_oa_inspection[\n lateral_oa_inspection[\"Lateral OA\"].notna()\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:35.973126Z","iopub.execute_input":"2026-09-12T08:37:35.973601Z","iopub.status.idle":"2026-09-12T08:37:36.004798Z","shell.execute_reply.started":"2026-09-12T08:37:35.973572Z","shell.execute_reply":"2026-09-12T08:37:36.003428Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_lateral_oa_v1(report):\n text = str(report).lower()\n\n negative_patterns = [\n # English\n r\"lateral compartment cartilage\\s*:\\s*intact\",\n r\"lateral compartment cartilage.*\\bnormal\\b\",\n r\"lateral cartilage.*\\bintact\\b\",\n r\"lateral cartilage.*\\bnormal\\b\",\n r\"lateral compartment.*no focal chondrosis\",\n r\"lateral compartment.*no focal chondral injury\",\n r\"preserved.*lateral femorotibial articular cartilage\",\n\n # Spanish\n r\"cart[ií]lagos.*compartimento.*sin alteraciones\",\n r\"compartimento femorotibial lateral.*sin alteraciones\",\n r\"compartimento lateral.*sin alteraciones\",\n\n # Turkish\n r\"lateral kompartman.*normal\",\n r\"lateral kompartman.*intakt\",\n r\"lateral kompartman.*k[kı]k[kı]rdak.*normal\",\n\n # Croatian / Bosnian / Serbian\n r\"lateralni kompartment.*bez.*promjena\",\n r\"lateralni kompartment.*uredan\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n positive_patterns = [\n # English\n r\"\\blateral osteoarthritis\\b\",\n r\"\\bosteoarthritis\\b.*\\blateral compartment\\b\",\n r\"\\blateral compartment\\b.*\\bosteoarthritis\\b\",\n r\"\\btricompartmental osteoarthritis\\b\",\n r\"\\btricompartmental .*oa\\b\",\n r\"\\blateral oa\\b\",\n\n # Spanish\n r\"oa femorotibial.*lateral\",\n r\"osteoartritis.*compartimento.*lateral\",\n r\"artrosis.*compartimento.*lateral\",\n\n # Croatian / Bosnian / Serbian\n r\"oa.*lateralno\",\n r\"oa.*lateral\",\n r\"osteoartr.*lateral\",\n r\"artroz.*lateral\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n high_grade_lateral_patterns = [\n # English\n r\"advanced.*lateral.*cartilage.*loss\",\n r\"full[- ]thickness cartilage loss.*lateral\",\n r\"high[- ]grade.*chondrosis.*lateral\",\n r\"high[- ]grade.*cartilage.*lateral\",\n r\"severe.*lateral.*chondrosis\",\n\n # Spanish\n r\"condropat[ií]a.*grado [3-4].*compartimento femorotibial lateral\",\n r\"condropat[ií]a.*lesiones grado 3 y 4.*lateral\",\n r\"condropat[ií]a.*grado 3.*lateral\",\n r\"condropat[ií]a.*grado 4.*lateral\",\n\n # Croatian / Bosnian / Serbian\n r\"hondromalacija.*[i1]{1,2}i[v4].*lateral\",\n r\"lateral.*oa.*osteofit\",\n ]\n\n for pattern in high_grade_lateral_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.006184Z","iopub.execute_input":"2026-09-12T08:37:36.00668Z","iopub.status.idle":"2026-09-12T08:37:36.033117Z","shell.execute_reply.started":"2026-09-12T08:37:36.006649Z","shell.execute_reply":"2026-09-12T08:37:36.03168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Lateral_OA_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(extract_lateral_oa_v1)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.034465Z","iopub.execute_input":"2026-09-12T08:37:36.034833Z","iopub.status.idle":"2026-09-12T08:37:36.082255Z","shell.execute_reply.started":"2026-09-12T08:37:36.034797Z","shell.execute_reply":"2026-09-12T08:37:36.080778Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Distribution:\")\nprint(\n comparison[\"Lateral_OA_extracted_v1\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"Lateral_OA_extracted_v1\"].notna()\n\nprint(\"\\nConfusion:\")\nprint(\n pd.crosstab(\n labeled_reports.loc[mask, \"Lateral OA\"],\n comparison.loc[mask, \"Lateral_OA_extracted_v1\"],\n rownames=[\"Official\"],\n colnames=[\"Extracted\"]\n )\n)\n\ncoverage = mask.sum()\n\nagreement = (\n labeled_reports.loc[mask, \"Lateral OA\"]\n == comparison.loc[mask, \"Lateral_OA_extracted_v1\"]\n).mean()\n\nprint(\n f\"\\nCoverage: {coverage} / {len(labeled_reports)} \"\n f\"= {coverage / len(labeled_reports):.1%}\"\n)\n\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.083536Z","iopub.execute_input":"2026-09-12T08:37:36.083932Z","iopub.status.idle":"2026-09-12T08:37:36.105289Z","shell.execute_reply.started":"2026-09-12T08:37:36.083903Z","shell.execute_reply":"2026-09-12T08:37:36.104008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_lat_oa_v1 = comparison[\n comparison[\"Lateral_OA_extracted_v1\"].notna()\n & (\n labeled_reports[\"Lateral OA\"]\n != comparison[\"Lateral_OA_extracted_v1\"]\n )\n].copy()\n\nmismatch_lat_oa_v1[\"Lateral OA\"] = labeled_reports.loc[\n mismatch_lat_oa_v1.index, \"Lateral OA\"\n]\n\nmismatch_lat_oa_v1[\"Report\"] = labeled_reports.loc[\n mismatch_lat_oa_v1.index, \"Report\"\n]\n\nprint(\n mismatch_lat_oa_v1[\n [\n \"Lateral OA\",\n \"Lateral_OA_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.107001Z","iopub.execute_input":"2026-09-12T08:37:36.107552Z","iopub.status.idle":"2026-09-12T08:37:36.135976Z","shell.execute_reply.started":"2026-09-12T08:37:36.107519Z","shell.execute_reply":"2026-09-12T08:37:36.134961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pf_oa_inspection = labeled_reports[\n [\"StudyInstanceUID\", \"PF OA\", \"Report\"]\n].copy()\n\nfor label in [0, 1]:\n print(\"\\n\" + \"=\" * 100)\n print(f\"PF OA = {label}\")\n print(\"=\" * 100)\n\n subset = pf_oa_inspection[\n pf_oa_inspection[\"PF OA\"] == label\n ]\n\n for i, (_, row) in enumerate(subset.iterrows(), 1):\n print(f\"\\n--- Example {i} ---\")\n print(row[\"Report\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.137531Z","iopub.execute_input":"2026-09-12T08:37:36.137889Z","iopub.status.idle":"2026-09-12T08:37:36.168784Z","shell.execute_reply.started":"2026-09-12T08:37:36.13785Z","shell.execute_reply":"2026-09-12T08:37:36.167891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_pf_oa_v1(report):\n text = str(report).lower()\n\n negative_patterns = [\n # English\n r\"patellofemoral.*cartilage.*\\bnormal\\b\",\n r\"patellofemoral.*cartilage.*\\bintact\\b\",\n r\"patellar.*cartilage.*\\bnormal\\b\",\n r\"patellar.*cartilage.*\\bintact\\b\",\n r\"trochlear cartilage.*\\bnormal\\b\",\n r\"trochlear cartilage.*\\bintact\\b\",\n r\"patellar and trochlear cartilage.*without chondral lesion\",\n r\"patellar and trochlear cartilage.*normal\",\n\n # Spanish\n r\"cart[ií]lago rotuliano.*sin alteraciones\",\n r\"cart[ií]lago.*patelofemoral.*sin alteraciones\",\n r\"cart[ií]lagos.*patelofemorales.*sin alteraciones\",\n r\"cart[ií]lago.*tr[oó]clea.*sin alteraciones\",\n\n # Turkish\n r\"patella.*normal\",\n r\"patellar.*kıkırdak.*normal\",\n\n # Croatian / related\n r\"hrskavica.*patele.*uredn\",\n r\"hrskavica.*patele.*bez.*promjena\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n positive_patterns = [\n # English\n r\"\\bpatellofemoral osteoarthritis\\b\",\n r\"\\bpatellofemoral oa\\b\",\n r\"\\bpatellofemoral.*oste[o]?arthritis\\b\",\n r\"\\bosteoarthritis.*patellofemoral\\b\",\n r\"\\bpatellofemoral joint.*osteoarthritis\\b\",\n\n # Spanish\n r\"oa patelofemoral\",\n r\"osteoartritis.*patelofemoral\",\n r\"artrosis.*patelofemoral\",\n\n # Croatian / related\n r\"pf artrotske promjene\",\n r\"patelofemoral.*artroz\",\n r\"patelofemoral.*osteoartr\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n strong_patterns = [\n # English: high-grade / full thickness\n r\"full[- ]thickness.*cartilage.*patell\",\n r\"full[- ]thickness.*cartilage.*trochlea\",\n r\"high[- ]grade cartilage loss.*patell\",\n r\"high[- ]grade cartilage loss.*trochlea\",\n r\"high[- ]grade.*chondrosis.*patell\",\n r\"high[- ]grade.*chondrosis.*trochlea\",\n r\"grade 3[- ]?4.*chondromalacia.*trochlea\",\n r\"grade 3[- ]?4.*chondromalacia.*patell\",\n r\"grade 3[- ]?4.*chondrosis.*patell\",\n r\"grade 3[- ]?4.*chondrosis.*trochlea\",\n\n # Spanish\n r\"condropat[ií]a.*grado [3-4].*r[oó]tula\",\n r\"condropat[ií]a.*grado 3 y 4.*r[oó]tula\",\n r\"condropat[ií]a.*grado [3-4].*tr[oó]clea\",\n r\"condropat[ií]a.*espesor total.*r[oó]tula\",\n r\"condropat[ií]a.*espesor total.*tr[oó]clea\",\n\n # Croatian / related\n r\"hondromalacija.*iii.*patele\",\n r\"hondromalacija.*iv.*patele\",\n r\"hondromalaci.*iii.*patele\",\n r\"hondromalaci.*iv.*patele\",\n r\"uznapredoval.*oa.*fp\",\n r\"oa.*fp zgloba\",\n\n # Greek\n r\"οστεοαρθρίτιδ.*επιγονα\",\n r\"χονδρομαλακία.*iii\",\n r\"χονδρομαλακία.*iv\",\n ]\n\n for pattern in strong_patterns:\n if re.search(pattern, text):\n return 1\n\n advanced_patterns = [\n r\"patellofemoral.*osteophytes\",\n r\"patellofemoral.*subchondral edema.*osteophytes\",\n r\"patellofemoral.*joint space narrowing\",\n r\"patellofemoral.*cartilage loss.*subchondral\",\n r\"patellar.*cartilage loss.*subchondral\",\n r\"trochlear.*cartilage loss.*subchondral\",\n ]\n\n for pattern in advanced_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.170116Z","iopub.execute_input":"2026-09-12T08:37:36.170535Z","iopub.status.idle":"2026-09-12T08:37:36.200662Z","shell.execute_reply.started":"2026-09-12T08:37:36.170495Z","shell.execute_reply":"2026-09-12T08:37:36.199286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"PF_OA_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(extract_pf_oa_v1)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.202135Z","iopub.execute_input":"2026-09-12T08:37:36.20253Z","iopub.status.idle":"2026-09-12T08:37:36.243706Z","shell.execute_reply.started":"2026-09-12T08:37:36.202488Z","shell.execute_reply":"2026-09-12T08:37:36.242452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Distribution:\")\nprint(\n comparison[\"PF_OA_extracted_v1\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"PF_OA_extracted_v1\"].notna()\n\nprint(\"\\nConfusion:\")\nprint(\n pd.crosstab(\n labeled_reports.loc[mask, \"PF OA\"],\n comparison.loc[mask, \"PF_OA_extracted_v1\"],\n rownames=[\"Official\"],\n colnames=[\"Extracted\"]\n )\n)\n\ncoverage = mask.sum()\n\nagreement = (\n labeled_reports.loc[mask, \"PF OA\"]\n == comparison.loc[mask, \"PF_OA_extracted_v1\"]\n).mean()\n\nprint(\n f\"\\nCoverage: {coverage} / {len(labeled_reports)} \"\n f\"= {coverage / len(labeled_reports):.1%}\"\n)\n\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.244938Z","iopub.execute_input":"2026-09-12T08:37:36.245289Z","iopub.status.idle":"2026-09-12T08:37:36.266996Z","shell.execute_reply.started":"2026-09-12T08:37:36.245259Z","shell.execute_reply":"2026-09-12T08:37:36.265796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_pf_oa_v1 = comparison[\n comparison[\"PF_OA_extracted_v1\"].notna()\n & (\n labeled_reports[\"PF OA\"]\n != comparison[\"PF_OA_extracted_v1\"]\n )\n].copy()\n\nmismatch_pf_oa_v1[\"PF OA\"] = labeled_reports.loc[\n mismatch_pf_oa_v1.index, \"PF OA\"\n]\n\nmismatch_pf_oa_v1[\"Report\"] = labeled_reports.loc[\n mismatch_pf_oa_v1.index, \"Report\"\n]\n\nprint(\n mismatch_pf_oa_v1[\n [\n \"PF OA\",\n \"PF_OA_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.268287Z","iopub.execute_input":"2026-09-12T08:37:36.268719Z","iopub.status.idle":"2026-09-12T08:37:36.294563Z","shell.execute_reply.started":"2026-09-12T08:37:36.268678Z","shell.execute_reply":"2026-09-12T08:37:36.2934Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_effusion_sentences(row):\n report = str(row[\"Report\"]).replace(\"\\n\", \" \")\n\n sentences = re.split(r'(?<=[.!?])\\s+', report)\n\n keywords = [\n # English\n \"effusion\",\n \"joint effusion\",\n \"knee effusion\",\n \"joint fluid\",\n \"intra-articular fluid\",\n \"fluid in the joint\",\n \"fluid within the joint\",\n\n # Spanish\n \"derrame\",\n \"derrame articular\",\n \"derrame de la articulación\",\n \"líquido articular\",\n\n # Turkish\n \"efüzyon\",\n \"eklem efüzyonu\",\n \"eklem sıvısı\",\n \"sıvı artışı\",\n\n # Croatian / Bosnian / Serbian\n \"izljev\",\n \"izliv\",\n \"zglobni izljev\",\n \"zglobni izliv\",\n \"tekućina\",\n \"tečnost\",\n\n # Greek\n \"υγρό\",\n \"αρθρική συλλογή\",\n \"συλλογή\",\n ]\n\n matched = []\n\n for sentence in sentences:\n sentence_lower = sentence.lower()\n\n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n\n return \" \".join(matched)\n\n\neffusion_cases = labeled_reports[\n labeled_reports[\"Effusion\"].notna()\n].copy()\n\neffusion_cases[\"Effusion_sentences\"] = (\n effusion_cases.apply(\n get_effusion_sentences,\n axis=1\n )\n)\n\nprint(\n effusion_cases[\n [\"Effusion\", \"Effusion_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.295636Z","iopub.execute_input":"2026-09-12T08:37:36.296209Z","iopub.status.idle":"2026-09-12T08:37:36.333066Z","shell.execute_reply.started":"2026-09-12T08:37:36.296158Z","shell.execute_reply":"2026-09-12T08:37:36.331827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_effusion_v1(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n\n negative_patterns = [\n # English\n r\"\\bno knee effusion\\b\",\n r\"\\bno joint effusion\\b\",\n r\"\\bno evidence of knee effusion\\b\",\n r\"\\bno evidence of joint effusion\\b\",\n r\"\\bno significant knee effusion\\b\",\n r\"\\bno significant joint effusion\\b\",\n r\"\\bwithout knee effusion\\b\",\n r\"\\bwithout joint effusion\\b\",\n\n # Spanish\n r\"\\bsin derrame articular\\b\",\n r\"\\bsin derrame de la articulación\\b\",\n r\"\\bno hay derrame articular\\b\",\n r\"\\bno se observa derrame articular\\b\",\n\n # Turkish\n r\"\\befüzyon yok\\b\",\n r\"\\beklem efüzyonu yok\\b\",\n r\"\\bdiz ekleminde efüzyon yok\\b\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bbez.*zglobnog izljeva\\b\",\n r\"\\bbez.*zglobnog izliva\\b\",\n r\"\\bnema.*izljeva\\b\",\n r\"\\bnema.*izliva\\b\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n positive_patterns = [\n # English\n r\"\\bjoint effusion\\b\",\n r\"\\bknee effusion\\b\",\n r\"\\bjoint fluid\\b\",\n r\"\\bfluid within the joint\\b\",\n r\"\\bintra[- ]articular fluid\\b\",\n r\"\\bintraarticular fluid\\b\",\n r\"\\beffusion with synovitis\\b\",\n r\"\\beffusion and synovitis\\b\",\n r\"\\blarge effusion\\b\",\n r\"\\bmild effusion\\b\",\n r\"\\bsmall effusion\\b\",\n\n # Spanish\n r\"\\bderrame articular\\b\",\n r\"\\bderrame con sinovitis\\b\",\n r\"\\bleve derrame articular\\b\",\n r\"\\bleve a moderado derrame articular\\b\",\n r\"\\bmoderado derrame articular\\b\",\n\n # Turkish — require knee/joint context\n r\"diz eklem(?:i|inde|mesafesinde).*sıvı artışı\",\n r\"diz eklem kapsülünde.*efüzyon\",\n r\"eklem.*efüzyon\",\n r\"eklem.*sıvı artışı\",\n\n # Greek\n r\"ενδαρθρικ.*συλλογή υγρού\",\n r\"αρθρική συλλογή\",\n r\"συλλογή υγρού.*ενδαρθρ\",\n r\"ικανή ποσότητα ενθαρθρικής συλλογής\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bzglobni izljev\\b\",\n r\"\\bzglobni izliv\\b\",\n r\"\\bizljev u zglob\\b\",\n r\"\\bizliv u zglob\\b\",\n r\"\\bizljeva u zglob\\b\",\n r\"\\bizljeva suprapatelarno\\b\",\n r\"\\bmanja količina izljeva\\b\",\n r\"\\bopsežan zglobni izljev\\b\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.334497Z","iopub.execute_input":"2026-09-12T08:37:36.334963Z","iopub.status.idle":"2026-09-12T08:37:36.359495Z","shell.execute_reply.started":"2026-09-12T08:37:36.334934Z","shell.execute_reply":"2026-09-12T08:37:36.358183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Effusion_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(extract_effusion_v1)\n)\n\nprint(\"Distribution:\")\nprint(\n comparison[\"Effusion_extracted_v1\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"Effusion_extracted_v1\"].notna()\n\nprint(\"\\nConfusion:\")\nprint(\n pd.crosstab(\n labeled_reports.loc[mask, \"Effusion\"],\n comparison.loc[mask, \"Effusion_extracted_v1\"],\n rownames=[\"Official\"],\n colnames=[\"Extracted\"]\n )\n)\n\ncoverage = mask.sum()\n\nagreement = (\n labeled_reports.loc[mask, \"Effusion\"]\n == comparison.loc[mask, \"Effusion_extracted_v1\"]\n).mean()\n\nprint(\n f\"\\nCoverage: {coverage}/{len(labeled_reports)} \"\n f\"= {coverage/len(labeled_reports):.1%}\"\n)\n\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.360851Z","iopub.execute_input":"2026-09-12T08:37:36.361236Z","iopub.status.idle":"2026-09-12T08:37:36.446228Z","shell.execute_reply.started":"2026-09-12T08:37:36.361194Z","shell.execute_reply":"2026-09-12T08:37:36.444681Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_effusion_v1 = comparison[\n comparison[\"Effusion_extracted_v1\"].notna()\n & (\n labeled_reports[\"Effusion\"]\n != comparison[\"Effusion_extracted_v1\"]\n )\n].copy()\n\nmismatch_effusion_v1[\"Effusion\"] = labeled_reports.loc[\n mismatch_effusion_v1.index, \"Effusion\"\n]\n\nmismatch_effusion_v1[\"Report\"] = labeled_reports.loc[\n mismatch_effusion_v1.index, \"Report\"\n]\n\nprint(\n mismatch_effusion_v1[\n [\n \"Effusion\",\n \"Effusion_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.447793Z","iopub.execute_input":"2026-09-12T08:37:36.448279Z","iopub.status.idle":"2026-09-12T08:37:36.463609Z","shell.execute_reply.started":"2026-09-12T08:37:36.448247Z","shell.execute_reply":"2026-09-12T08:37:36.46253Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_synovitis_sentences(row):\n report = str(row[\"Report\"]).replace(\"\\n\", \" \")\n\n sentences = re.split(r'(?<=[.!?])\\s+', report)\n\n keywords = [\n # English\n \"synovitis\",\n \"synovial hypertrophy\",\n \"hypertrophy of the synovium\",\n \"synovial thickening\",\n \"thickening of the synovium\",\n \"synovial proliferation\",\n \"synovial tissue\",\n\n # Spanish\n \"sinovitis\",\n \"hipertrofia sinovial\",\n \"hipertrofia de la sinovial\",\n \"engrosamiento sinovial\",\n \"engrosamiento de la sinovial\",\n \"proliferación sinovial\",\n\n # Turkish\n \"sinovit\",\n \"sinovyal hipertrofi\",\n \"sinovyal kalınlaşma\",\n \"sinoviyal hipertrofi\",\n \"sinoviyal kalınlaşma\",\n\n # Croatian / Bosnian / Serbian\n \"sinovitis\",\n \"sinovijalna hipertrofija\",\n \"hipertrofija sinovije\",\n \"zadebljanje sinovije\",\n \"zadebljanje sinovijal\",\n \"proliferacija sinovije\",\n\n # Greek\n \"υμενίτιδα\",\n \"υμενοϋπερτροφία\",\n \"πάχυνση του υμένα\",\n \"πάχυνση υμένα\",\n \"υπερτροφία του υμένα\",\n ]\n\n matched = []\n\n for sentence in sentences:\n sentence_lower = sentence.lower()\n\n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n\n return \" \".join(matched)\n\n\nsynovitis_cases = labeled_reports[\n labeled_reports[\"Synovitis\"].notna()\n].copy()\n\nsynovitis_cases[\"Synovitis_sentences\"] = (\n synovitis_cases.apply(\n get_synovitis_sentences,\n axis=1\n )\n)\n\nprint(\n synovitis_cases[\n [\"Synovitis\", \"Synovitis_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.465153Z","iopub.execute_input":"2026-09-12T08:37:36.465629Z","iopub.status.idle":"2026-09-12T08:37:36.494409Z","shell.execute_reply.started":"2026-09-12T08:37:36.465594Z","shell.execute_reply":"2026-09-12T08:37:36.493294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_synovitis_v1(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n\n negative_patterns = [\n # English\n r\"\\bno synovitis\\b\",\n r\"\\bno evidence of synovitis\\b\",\n r\"\\bwithout synovitis\\b\",\n r\"\\bno synovial hypertrophy\\b\",\n r\"\\bno synovial thickening\\b\",\n\n # Spanish\n r\"\\bsin sinovitis\\b\",\n r\"\\bno hay sinovitis\\b\",\n r\"\\bsin hipertrofia sinovial\\b\",\n r\"\\bsin engrosamiento sinovial\\b\",\n\n # Turkish\n r\"\\bsinovit yok\\b\",\n r\"\\bsinovyal hipertrofi yok\\b\",\n r\"\\bsinovyal kalınlaşma yok\\b\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bbez sinovitisa\\b\",\n r\"\\bnema sinovitisa\\b\",\n r\"\\bbez hipertrofije sinovije\\b\",\n r\"\\bbez zadebljanja sinovije\\b\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n positive_patterns = [\n # English\n r\"\\bsynovitis\\b\",\n r\"\\bsynovial hypertrophy\\b\",\n r\"\\bhypertrophy of the synovium\\b\",\n r\"\\bsynovial thickening\\b\",\n r\"\\bthickening of the synovium\\b\",\n r\"\\bsynovial proliferation\\b\",\n r\"\\bproliferation of the synovium\\b\",\n r\"\\bthickened synovial tissue\\b\",\n r\"\\bhypertrophy of synovial tissue\\b\",\n\n # Spanish\n r\"\\bsinovitis\\b\",\n r\"\\bhipertrofia sinovial\\b\",\n r\"\\bhipertrofia de la sinovial\\b\",\n r\"\\bengrosamiento sinovial\\b\",\n r\"\\bengrosamiento de la sinovial\\b\",\n r\"\\bproliferación sinovial\\b\",\n r\"\\bproliferacion sinovial\\b\",\n\n # Turkish\n r\"\\bsinovit\\b\",\n r\"\\bsinovyal hipertrofi\\b\",\n r\"\\bsinovyal kalınlaşma\\b\",\n r\"\\bsinoviyal hipertrofi\\b\",\n r\"\\bsinoviyal kalınlaşma\\b\",\n\n # Dutch\n r\"\\bchronische synovitis\\b\",\n r\"\\bsynovitis\\b\",\n r\"\\bsynoviale hypertrofie\\b\",\n r\"\\bsynoviale verdikking\\b\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bsinovitis\\b\",\n r\"\\bsinovijalna hipertrofija\\b\",\n r\"\\bhipertrofija sinovije\\b\",\n r\"\\bzadebljanje sinovije\\b\",\n r\"\\bzadebljanje sinovijal\",\n r\"\\bproliferacija sinovije\\b\",\n\n # Greek\n r\"υμενίτιδα\",\n r\"υμενοϋπερτροφία\",\n r\"πάχυνση του υμένα\",\n r\"πάχυνση υμένα\",\n r\"υπερτροφία του υμένα\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.495469Z","iopub.execute_input":"2026-09-12T08:37:36.495878Z","iopub.status.idle":"2026-09-12T08:37:36.522107Z","shell.execute_reply.started":"2026-09-12T08:37:36.495832Z","shell.execute_reply":"2026-09-12T08:37:36.520986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Synovitis_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(extract_synovitis_v1)\n)\n\nprint(\"Distribution:\")\nprint(\n comparison[\"Synovitis_extracted_v1\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"Synovitis_extracted_v1\"].notna()\n\nprint(\"\\nConfusion:\")\nprint(\n pd.crosstab(\n labeled_reports.loc[mask, \"Synovitis\"],\n comparison.loc[mask, \"Synovitis_extracted_v1\"],\n rownames=[\"Official\"],\n colnames=[\"Extracted\"]\n )\n)\n\ncoverage = mask.sum()\n\nagreement = (\n labeled_reports.loc[mask, \"Synovitis\"]\n == comparison.loc[mask, \"Synovitis_extracted_v1\"]\n).mean()\n\nprint(\n f\"\\nCoverage: {coverage}/{len(labeled_reports)} \"\n f\"= {coverage/len(labeled_reports):.1%}\"\n)\n\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.523529Z","iopub.execute_input":"2026-09-12T08:37:36.523878Z","iopub.status.idle":"2026-09-12T08:37:36.617577Z","shell.execute_reply.started":"2026-09-12T08:37:36.523847Z","shell.execute_reply":"2026-09-12T08:37:36.616516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_synovitis_v1 = comparison[\n comparison[\"Synovitis_extracted_v1\"].notna()\n & (\n labeled_reports[\"Synovitis\"]\n != comparison[\"Synovitis_extracted_v1\"]\n )\n].copy()\n\nmismatch_synovitis_v1[\"Synovitis\"] = labeled_reports.loc[\n mismatch_synovitis_v1.index, \"Synovitis\"\n]\n\nmismatch_synovitis_v1[\"Report\"] = labeled_reports.loc[\n mismatch_synovitis_v1.index, \"Report\"\n]\n\nprint(\n mismatch_synovitis_v1[\n [\n \"Synovitis\",\n \"Synovitis_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.618724Z","iopub.execute_input":"2026-09-12T08:37:36.619108Z","iopub.status.idle":"2026-09-12T08:37:36.632528Z","shell.execute_reply.started":"2026-09-12T08:37:36.619068Z","shell.execute_reply":"2026-09-12T08:37:36.631161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_bakers_sentences(row):\n report = str(row[\"Report\"]).replace(\"\\n\", \" \")\n\n sentences = re.split(r'(?<=[.!?])\\s+', report)\n\n keywords = [\n # English\n \"baker's cyst\",\n \"baker cyst\",\n \"popliteal cyst\",\n \"popliteal cystic\",\n \"cyst in the popliteal fossa\",\n\n # Spanish\n \"quiste poplíteo\",\n \"quiste de baker\",\n \"quiste de bakers\",\n\n # Turkish\n \"baker kisti\",\n \"baker kisti\",\n \"popliteal kist\",\n\n # Croatian / Bosnian / Serbian\n \"baker cista\",\n \"bakerova cista\",\n \"poplitealna cista\",\n \"poplitealna ciste\",\n\n # Greek\n \"κύστη baker\",\n \"κύστη baker\",\n \"ιγνυακή κύστη\",\n ]\n\n matched = []\n\n for sentence in sentences:\n sentence_lower = sentence.lower()\n\n if any(keyword in sentence_lower for keyword in keywords):\n matched.append(sentence.strip())\n\n return \" \".join(matched)\n\n\nbakers_cases = labeled_reports[\n labeled_reports[\"Baker's\"].notna()\n].copy()\n\nbakers_cases[\"Bakers_sentences\"] = (\n bakers_cases.apply(\n get_bakers_sentences,\n axis=1\n )\n)\n\nprint(\n bakers_cases[\n [\"Baker's\", \"Bakers_sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.63392Z","iopub.execute_input":"2026-09-12T08:37:36.634652Z","iopub.status.idle":"2026-09-12T08:37:36.669251Z","shell.execute_reply.started":"2026-09-12T08:37:36.634592Z","shell.execute_reply":"2026-09-12T08:37:36.667814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_bakers_v2(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n negative_patterns = [\n r\"\\bno baker['’]?s cyst\\b\",\n r\"\\bbaker['’]?s cyst\\s*:\\s*none\\b\",\n r\"\\bno baker cyst\\b\",\n r\"\\bno popliteal cyst\\b\",\n r\"\\bwithout baker['’]?s cyst\\b\",\n r\"\\bwithout popliteal cyst\\b\",\n r\"\\bsin quiste popl[ií]teo\\b\",\n r\"\\bno hay quiste popl[ií]teo\\b\",\n r\"\\bsin quiste de baker\\b\",\n r\"\\bbaker kisti yok\\b\",\n r\"\\bpopliteal kist yok\\b\",\n r\"\\bbez baker.*cist\",\n r\"\\bbez popliteal.*cist\",\n r\"\\bnema popliteal.*cist\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n uncertain_patterns = [\n r\"\\btrace baker['’]?s cyst\\b\",\n r\"\\btrace baker cyst\\b\",\n r\"\\btrace popliteal cyst\\b\",\n r\"\\bsmall baker['’]?s cyst\\b\",\n r\"\\bsmall baker cyst\\b\",\n r\"\\bsmall popliteal cyst\\b\",\n r\"\\bpequeño quiste popl[ií]teo\\b\",\n r\"\\bpequeña quiste popl[ií]teo\\b\",\n r\"\\bpetit kyste poplit\",\n r\"\\bsmall baker kisti\\b\",\n r\"\\bsmall popliteal kist\\b\",\n r\"\\bmala baker.*cist\",\n r\"\\bmanja baker.*cist\",\n ]\n\n for pattern in uncertain_patterns:\n if re.search(pattern, text):\n return np.nan\n positive_patterns = [\n r\"\\bbaker['’]?s cyst\\b\",\n r\"\\bbaker cyst\\b\",\n r\"\\bpopliteal cyst\\b\",\n r\"\\bcyst in the popliteal fossa\\b\",\n r\"\\bpopliteal cystic\\b\",\n r\"\\bquiste popl[ií]teo\\b\",\n r\"\\bquiste de baker\\b\",\n r\"\\bquiste de bakers\\b\",\n r\"\\bbaker kisti\\b\",\n r\"\\bpopliteal kist\\b\",\n r\"\\bbaker cista\\b\",\n r\"\\bbakerova cista\\b\",\n r\"\\bpoplitealna cista\\b\",\n r\"\\bpoplitealna ciste\\b\",\n r\"κύστη baker\",\n r\"ιγνυακή κύστη\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.670568Z","iopub.execute_input":"2026-09-12T08:37:36.67092Z","iopub.status.idle":"2026-09-12T08:37:36.716322Z","shell.execute_reply.started":"2026-09-12T08:37:36.670889Z","shell.execute_reply":"2026-09-12T08:37:36.715005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Bakers_extracted_v2\"] = (\n labeled_reports[\"Report\"].apply(extract_bakers_v2)\n)\n\nprint(\"Distribution:\")\nprint(comparison[\"Bakers_extracted_v2\"].value_counts(dropna=False))\n\nmask = comparison[\"Bakers_extracted_v2\"].notna()\n\nconfusion = pd.crosstab(\n labeled_reports.loc[mask, \"Baker's\"],\n comparison.loc[mask, \"Bakers_extracted_v2\"]\n)\n\nprint(\"\\nConfusion:\")\nprint(confusion)\n\ncoverage = mask.sum()\ntotal = len(comparison)\n\nagreement = (\n labeled_reports.loc[mask, \"Baker's\"].values\n == comparison.loc[mask, \"Bakers_extracted_v2\"].values\n).mean()\n\nprint(f\"\\nCoverage: {coverage}/{total} = {coverage/total:.1%}\")\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.717544Z","iopub.execute_input":"2026-09-12T08:37:36.717877Z","iopub.status.idle":"2026-09-12T08:37:36.806539Z","shell.execute_reply.started":"2026-09-12T08:37:36.717849Z","shell.execute_reply":"2026-09-12T08:37:36.805509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_bakers_v2 = comparison[\n comparison[\"Bakers_extracted_v2\"].notna()\n & (\n labeled_reports[\"Baker's\"]\n != comparison[\"Bakers_extracted_v2\"]\n )\n].copy()\n\nmismatch_bakers_v2[\"Baker's\"] = labeled_reports.loc[\n mismatch_bakers_v2.index, \"Baker's\"\n]\n\nmismatch_bakers_v2[\"Report\"] = labeled_reports.loc[\n mismatch_bakers_v2.index, \"Report\"\n]\n\nprint(\n mismatch_bakers_v2[\n [\n \"Baker's\",\n \"Bakers_extracted_v2\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.807828Z","iopub.execute_input":"2026-09-12T08:37:36.80825Z","iopub.status.idle":"2026-09-12T08:37:36.822656Z","shell.execute_reply.started":"2026-09-12T08:37:36.808222Z","shell.execute_reply":"2026-09-12T08:37:36.82141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_contusion_sentences(row):\n report = str(row[\"Report\"])\n \n sentences = re.split(r'(?<=[.!?])\\s+|\\n+', report)\n \n keywords = [\n \"contusion\",\n \"bone bruise\",\n \"bone marrow edema\",\n \"marrow edema\",\n \"osseous edema\",\n \"bone injury\",\n \"bone impact\",\n \"impaction\",\n \"oedema\",\n \"edema\",\n \"contusión\",\n \"contusion\",\n \"ödem\",\n \"kemik ödemi\",\n \"kemik iliği\",\n \"koštanog edema\",\n \"koštani edem\",\n \"οστικό οίδημα\",\n \"μυελ\",\n ]\n \n matched = [\n s.strip()\n for s in sentences\n if any(k in s.lower() for k in keywords)\n ]\n \n return matched\n\n\ncontusion_inspection = []\n\nfor idx, row in labeled_reports.iterrows():\n sentences = get_contusion_sentences(row)\n \n if sentences:\n contusion_inspection.append({\n \"index\": idx,\n \"Contusion\": row[\"Contusion\"],\n \"sentences\": \" | \".join(sentences)\n })\n\ncontusion_inspection_df = pd.DataFrame(contusion_inspection)\n\nprint(\n contusion_inspection_df[\n [\"Contusion\", \"sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.823974Z","iopub.execute_input":"2026-09-12T08:37:36.824537Z","iopub.status.idle":"2026-09-12T08:37:36.871429Z","shell.execute_reply.started":"2026-09-12T08:37:36.824505Z","shell.execute_reply":"2026-09-12T08:37:36.870347Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_contusion_v2(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n\n negative_patterns = [\n r\"\\bno fracture or bone bruise\\b\",\n r\"\\bno fracture or bone contusion\\b\",\n r\"\\bwithout bone contusion\\b\",\n r\"\\bno bone contusion\\b\",\n r\"\\bno evidence of bone contusion\\b\",\n r\"\\bwithout evidence of bone contusion\\b\",\n r\"\\bno acute fracture or bone bruise\\b\",\n r\"\\bno evidence of bone bruise\\b\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n fracture_context = [\n r\"fracture.*bone contusion\",\n r\"bone contusion.*fracture\",\n r\"fracture.*bone marrow edema\",\n r\"bone marrow edema.*fracture\",\n r\"fracture line.*bone contusion\",\n r\"fracture.*contusion\",\n r\"contusion.*fracture\",\n r\"impaction fracture.*contusion\",\n r\"contusion.*impaction fracture\",\n ]\n\n has_fracture_context = any(\n re.search(pattern, text)\n for pattern in fracture_context\n )\n\n positive_patterns = [\n\n # English — specifically bone/osseous\n r\"\\bbone contusion\\b\",\n r\"\\bbone bruise\\b\",\n r\"\\bbone bruising\\b\",\n r\"\\bbone contusion(?:al)?\\b\",\n r\"\\bosseous contusion\\b\",\n r\"\\bsuspect bone contusion\\b\",\n r\"\\bsuspected bone contusion\\b\",\n\n # Explicit bone marrow + contusion\n r\"\\bbone marrow edema.*suspect bone contusion\\b\",\n r\"\\bbone marrow edema.*bone contusion\\b\",\n r\"\\bbone marrow edema.*in favor of bone contusion\\b\",\n r\"\\bbone marrow edema.*likely a bone contusion\\b\",\n\n # Spanish\n r\"\\bcontusiones óseas\\b\",\n r\"\\bcontusiones oseas\\b\",\n r\"\\bcontusión ósea\\b\",\n r\"\\bcontusion osea\\b\",\n r\"\\bcontusiones.*femorotibial\",\n r\"\\bcontusión.*ósea\\b\",\n r\"\\bcontusion.*osea\\b\",\n\n # Turkish — explicitly bone contusion\n r\"\\bkontüzyonel kemik iliği ödem\",\n r\"\\bkontuzyonel kemik iligi odem\",\n r\"\\bkemik iliği ödem.*kontüzyon\\b\",\n r\"\\bkemik iligi odem.*kontuzyon\\b\",\n r\"\\bkissing kontüzyon\\b\",\n r\"\\bkissing kontuzyon\\b\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bkoštana kontuz\",\n r\"\\bkostana kontuz\",\n r\"\\bkontuzija kosti\\b\",\n ]\n\n matched_positive = any(\n re.search(pattern, text)\n for pattern in positive_patterns\n )\n\n if matched_positive:\n\n if has_fracture_context:\n return np.nan\n\n return 1\n\n standalone_contusion_patterns = [\n r\"bone marrow edema.*likely a contusion\",\n r\"bone marrow edema.*likely.*bone contusion\",\n r\"bone marrow edema.*suspect.*bone contusion\",\n r\"marrow edema.*compatible with.*bone contusion\",\n r\"marrow edema.*consistent with.*bone contusion\",\n r\"bone marrow edema.*favor.*bone contusion\",\n ]\n\n for pattern in standalone_contusion_patterns:\n if re.search(pattern, text):\n if not has_fracture_context:\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.872744Z","iopub.execute_input":"2026-09-12T08:37:36.873142Z","iopub.status.idle":"2026-09-12T08:37:36.894779Z","shell.execute_reply.started":"2026-09-12T08:37:36.873095Z","shell.execute_reply":"2026-09-12T08:37:36.893643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Contusion_extracted_v2\"] = (\n labeled_reports[\"Report\"].apply(extract_contusion_v2)\n)\n\nprint(\"Distribution:\")\nprint(\n comparison[\"Contusion_extracted_v2\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"Contusion_extracted_v2\"].notna()\n\nconfusion = pd.crosstab(\n labeled_reports.loc[mask, \"Contusion\"],\n comparison.loc[mask, \"Contusion_extracted_v2\"]\n)\n\nprint(\"\\nConfusion:\")\nprint(confusion)\n\ncoverage = mask.sum()\ntotal = len(comparison)\n\nagreement = (\n labeled_reports.loc[mask, \"Contusion\"].values\n ==\n comparison.loc[mask, \"Contusion_extracted_v2\"].values\n).mean()\n\nprint(f\"\\nCoverage: {coverage}/{total} = {coverage/total:.1%}\")\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.896219Z","iopub.execute_input":"2026-09-12T08:37:36.896542Z","iopub.status.idle":"2026-09-12T08:37:36.978542Z","shell.execute_reply.started":"2026-09-12T08:37:36.896513Z","shell.execute_reply":"2026-09-12T08:37:36.977308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_contusion_v2 = comparison[\n comparison[\"Contusion_extracted_v2\"].notna()\n &\n (\n labeled_reports[\"Contusion\"]\n != comparison[\"Contusion_extracted_v2\"]\n )\n].copy()\n\nmismatch_contusion_v2[\"Contusion\"] = labeled_reports.loc[\n mismatch_contusion_v2.index, \"Contusion\"\n]\n\nmismatch_contusion_v2[\"Report\"] = labeled_reports.loc[\n mismatch_contusion_v2.index, \"Report\"\n]\n\nprint(\n mismatch_contusion_v2[\n [\n \"Contusion\",\n \"Contusion_extracted_v2\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.979951Z","iopub.execute_input":"2026-09-12T08:37:36.980376Z","iopub.status.idle":"2026-09-12T08:37:36.994279Z","shell.execute_reply.started":"2026-09-12T08:37:36.980341Z","shell.execute_reply":"2026-09-12T08:37:36.993342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_fracture_sentences(row):\n report = str(row[\"Report\"])\n\n sentences = re.split(r'(?<=[.!?])\\s+|\\n+', report)\n\n keywords = [\n \"fracture\",\n \"fractured\",\n \"fracture line\",\n \"subchondral fracture\",\n \"insufficiency fracture\",\n \"impaction fracture\",\n \"avulsion fracture\",\n \"osteochondral fracture\",\n \"stress fracture\",\n \"microfracture\",\n \"microtrabecular fracture\",\n \"fractura\",\n \"fracture\",\n \"kırık\",\n \"kırığı\",\n \"kırık hattı\",\n \"mikrofraktür\",\n \"fraktura\",\n \"frakture\",\n \"prijelom\",\n \"prelom\",\n \"κάταγμα\",\n \"κατάγματος\",\n \"κατάγματα\",\n ]\n\n matched = [\n s.strip()\n for s in sentences\n if any(k in s.lower() for k in keywords)\n ]\n\n return matched\n\n\nfracture_inspection = []\n\nfor idx, row in labeled_reports.iterrows():\n sentences = get_fracture_sentences(row)\n\n if sentences:\n fracture_inspection.append({\n \"index\": idx,\n \"Fracture\": row[\"Fracture\"],\n \"sentences\": \" | \".join(sentences)\n })\n\nfracture_inspection_df = pd.DataFrame(fracture_inspection)\n\nprint(\n fracture_inspection_df[\n [\"Fracture\", \"sentences\"]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:36.995552Z","iopub.execute_input":"2026-09-12T08:37:36.995917Z","iopub.status.idle":"2026-09-12T08:37:37.054562Z","shell.execute_reply.started":"2026-09-12T08:37:36.995886Z","shell.execute_reply":"2026-09-12T08:37:37.053471Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_fracture_v1(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n\n negative_patterns = [\n r\"\\bno fracture\\b\",\n r\"\\bno acute fracture\\b\",\n r\"\\bno evidence of fracture\\b\",\n r\"\\bwithout fracture\\b\",\n r\"\\bwithout evidence of fracture\\b\",\n r\"\\bno fractures\\b\",\n r\"\\bno acute fractures\\b\",\n r\"\\bno fracture or bone bruise\\b\",\n r\"\\bno fracture or bone contusion\\b\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n uncertain_patterns = [\n r\"\\bsuspect(?:ed)? fracture\\b\",\n r\"\\bsuspicious for fracture\\b\",\n r\"\\bpossible fracture\\b\",\n r\"\\bpossible.*fracture\\b\",\n r\"\\bfracture.*cannot be excluded\\b\",\n r\"\\bcannot exclude.*fracture\\b\",\n r\"\\br/o.*fracture\\b\",\n r\"\\brule out.*fracture\\b\",\n r\"\\bin favor of.*fracture\\b\",\n r\"\\bcompatible with.*fracture\\b\",\n r\"\\bconsistent with.*fracture\\b\",\n r\"\\bcould represent.*fracture\\b\",\n r\"\\bmay represent.*fracture\\b\",\n r\"\\bpossibly.*fracture\\b\",\n r\"\\bprobable.*fracture\\b\",\n ]\n\n for pattern in uncertain_patterns:\n if re.search(pattern, text):\n return np.nan\n\n positive_patterns = [\n\n # English\n r\"\\bfracture\\b\",\n r\"\\bfractures\\b\",\n r\"\\bfracture line\\b\",\n r\"\\bsubchondral fracture\\b\",\n r\"\\binsufficiency fracture\\b\",\n r\"\\bavulsion fracture\\b\",\n r\"\\bosteochondral fracture\\b\",\n r\"\\bimpaction fracture\\b\",\n r\"\\bosteochondral impaction fracture\\b\",\n r\"\\bhairline fracture\\b\",\n r\"\\bcompression fracture\\b\",\n r\"\\bstress fracture\\b\",\n r\"\\bmicrofracture\\b\",\n r\"\\bmicrotrabecular fracture\\b\",\n\n # Spanish\n r\"\\bfractura\\b\",\n r\"\\bfracturas\\b\",\n r\"\\blínea de fractura\\b\",\n r\"\\bfractura subcondral\\b\",\n r\"\\bfractura osteocondral\\b\",\n r\"\\bfractura por insuficiencia\\b\",\n r\"\\bfractura por impactación\\b\",\n\n # Turkish\n r\"\\bkırık\\b\",\n r\"\\bkırığı\\b\",\n r\"\\bkırık hattı\\b\",\n r\"\\bsubkondral kırık\\b\",\n r\"\\bosteokondral kırık\\b\",\n r\"\\bimpaksiyon kırığı\\b\",\n r\"\\byetersizlik kırığı\\b\",\n r\"\\bmikrofraktür\\b\",\n\n # Croatian / Bosnian / Serbian\n r\"\\bfraktura\\b\",\n r\"\\bfrakture\\b\",\n r\"\\bprijelom\\b\",\n r\"\\bprijeloma\\b\",\n r\"\\bprelom\\b\",\n r\"\\bpreloma\\b\",\n r\"\\bimpresijska fraktura\\b\",\n\n # Greek\n r\"\\bκάταγμα\\b\",\n r\"\\bκατάγματος\\b\",\n r\"\\bκατάγματα\\b\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:37.055735Z","iopub.execute_input":"2026-09-12T08:37:37.056139Z","iopub.status.idle":"2026-09-12T08:37:37.067991Z","shell.execute_reply.started":"2026-09-12T08:37:37.056093Z","shell.execute_reply":"2026-09-12T08:37:37.066917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comparison[\"Fracture_extracted_v1\"] = (\n labeled_reports[\"Report\"].apply(extract_fracture_v1)\n)\n\nprint(\"Distribution:\")\nprint(\n comparison[\"Fracture_extracted_v1\"]\n .value_counts(dropna=False)\n)\n\nmask = comparison[\"Fracture_extracted_v1\"].notna()\n\nconfusion = pd.crosstab(\n labeled_reports.loc[mask, \"Fracture\"],\n comparison.loc[mask, \"Fracture_extracted_v1\"]\n)\n\nprint(\"\\nConfusion:\")\nprint(confusion)\n\ncoverage = mask.sum()\ntotal = len(comparison)\n\nagreement = (\n labeled_reports.loc[mask, \"Fracture\"].values\n ==\n comparison.loc[mask, \"Fracture_extracted_v1\"].values\n).mean()\n\nprint(f\"\\nCoverage: {coverage}/{total} = {coverage/total:.1%}\")\nprint(f\"Agreement: {agreement:.1%}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:37.069725Z","iopub.execute_input":"2026-09-12T08:37:37.070132Z","iopub.status.idle":"2026-09-12T08:37:37.171791Z","shell.execute_reply.started":"2026-09-12T08:37:37.070093Z","shell.execute_reply":"2026-09-12T08:37:37.170588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mismatch_fracture_v1 = comparison[\n comparison[\"Fracture_extracted_v1\"].notna()\n &\n (\n labeled_reports[\"Fracture\"]\n != comparison[\"Fracture_extracted_v1\"]\n )\n].copy()\n\nmismatch_fracture_v1[\"Fracture\"] = labeled_reports.loc[\n mismatch_fracture_v1.index, \"Fracture\"\n]\n\nmismatch_fracture_v1[\"Report\"] = labeled_reports.loc[\n mismatch_fracture_v1.index, \"Report\"\n]\n\nprint(\n mismatch_fracture_v1[\n [\n \"Fracture\",\n \"Fracture_extracted_v1\",\n \"Report\"\n ]\n ].to_string(index=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:37:37.173222Z","iopub.execute_input":"2026-09-12T08:37:37.173643Z","iopub.status.idle":"2026-09-12T08:37:37.188501Z","shell.execute_reply.started":"2026-09-12T08:37:37.173549Z","shell.execute_reply":"2026-09-12T08:37:37.18743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef extract_medial_meniscus_v1(report):\n text = str(report).lower().replace(\"\\n\", \" \")\n\n # Explicit negatives\n negative_patterns = [\n r\"medial meniscus.*\\bnormal\\b\",\n r\"medial meniscus.*\\bintact\\b\",\n r\"medial meniscus.*no tear\",\n r\"medial meniscus.*without.*tear\",\n r\"no.*medial meniscal tear\",\n r\"no.*tear.*medial meniscus\",\n r\"medial meniscus.*no evidence.*tear\",\n r\"menisco medial.*sin.*rotura\",\n r\"menisco interno.*sin.*rotura\",\n r\"menisco medial.*normal\",\n r\"menisco interno.*normal\",\n r\"medial meniscus.*keine.*ruptur\",\n r\"medial meniscus.*keine.*riss\",\n r\"medial meniscus.*normaldir\",\n r\"medial menisk.*normal\",\n r\"medial menisk.*intakt\",\n r\"medijalni menisk.*bez.*rupture\",\n r\"medijalni menisk.*uredan\",\n r\"medijalni menisk.*bez.*znakova rupture\",\n ]\n\n for pattern in negative_patterns:\n if re.search(pattern, text):\n return 0\n\n # Explicit strong tear terminology\n positive_patterns = [\n # English\n r\"medial meniscus.*\\btear\\b\",\n r\"medial meniscus.*\\bradial tear\\b\",\n r\"medial meniscus.*\\bhorizontal tear\\b\",\n r\"medial meniscus.*\\bvertical tear\\b\",\n r\"medial meniscus.*\\bcomplex tear\\b\",\n r\"medial meniscus.*\\broot tear\\b\",\n r\"tear.*medial meniscus\",\n r\"medial meniscal tear\",\n r\"tear of.*medial meniscus\",\n\n # Spanish\n r\"menisco medial.*rotura\",\n r\"menisco interno.*rotura\",\n r\"rotura.*menisco medial\",\n r\"rotura.*menisco interno\",\n\n # Turkish\n r\"medial menisküs.*yırtık\",\n r\"medial menisküs.*yırtığı\",\n r\"medial menisk.*yirtik\",\n r\"medial menisk.*yirtigi\",\n r\"iç menisküs.*yırtık\",\n r\"iç menisküs.*yırtığı\",\n\n # Croatian / Bosnian / Serbian\n r\"medijalni menisk.*ruptur\",\n r\"medijalni menisk.*rascjep\",\n r\"medijalni menisk.*oštećen\",\n r\"medijalni menisk.*lezij\",\n\n # Greek\n r\"έσω μηνίσκ.*ρήξ\",\n r\"έσω μηνίσκ.*ρήξη\",\n ]\n\n for pattern in positive_patterns:\n if re.search(pattern, text):\n return 1\n\n return np.nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:15.889105Z","iopub.execute_input":"2026-09-12T08:38:15.889546Z","iopub.status.idle":"2026-09-12T08:38:15.900321Z","shell.execute_reply.started":"2026-09-12T08:38:15.889512Z","shell.execute_reply":"2026-09-12T08:38:15.899304Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(extract_medial_meniscus_v1(\"Medial meniscus: complete tear.\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:20.626963Z","iopub.execute_input":"2026-09-12T08:38:20.628075Z","iopub.status.idle":"2026-09-12T08:38:20.634525Z","shell.execute_reply.started":"2026-09-12T08:38:20.62802Z","shell.execute_reply":"2026-09-12T08:38:20.63369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_labels = train[\n train[target_columns].isna().all(axis=1)\n].copy()\n\nprint(\"Unlabeled studies:\", len(weak_labels))\n\n# Best-performing extractor versions\nweak_labels[\"ACL_weak\"] = (\n weak_labels[\"Report\"].apply(extract_acl_v2)\n)\n\nweak_labels[\"MCL_weak\"] = (\n weak_labels[\"Report\"].apply(extract_mcl_v2)\n)\n\nweak_labels[\"Medial_Meniscus_weak\"] = (\n weak_labels[\"Report\"].apply(extract_medial_meniscus_v1)\n)\n\nweak_labels[\"Lateral_Meniscus_weak\"] = (\n weak_labels[\"Report\"].apply(extract_lateral_meniscus_v1)\n)\n\nweak_labels[\"Medial_OA_weak\"] = (\n weak_labels[\"Report\"].apply(extract_medial_oa_v2)\n)\n\nweak_labels[\"Lateral_OA_weak\"] = (\n weak_labels[\"Report\"].apply(extract_lateral_oa_v1)\n)\n\nweak_labels[\"PF_OA_weak\"] = (\n weak_labels[\"Report\"].apply(extract_pf_oa_v1)\n)\n\nweak_labels[\"Effusion_weak\"] = (\n weak_labels[\"Report\"].apply(extract_effusion_v1)\n)\n\nweak_labels[\"Synovitis_weak\"] = (\n weak_labels[\"Report\"].apply(extract_synovitis_v1)\n)\n\nweak_labels[\"Bakers_weak\"] = (\n weak_labels[\"Report\"].apply(extract_bakers_v2)\n)\n\nweak_labels[\"Contusion_weak\"] = (\n weak_labels[\"Report\"].apply(extract_contusion_v2)\n)\n\nweak_labels[\"Fracture_weak\"] = (\n weak_labels[\"Report\"].apply(extract_fracture_v1)\n)\n\nprint(\"\\nWeak-label table shape:\")\nprint(weak_labels.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:24.798633Z","iopub.execute_input":"2026-09-12T08:38:24.799612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_columns = [\n \"ACL_weak\",\n \"MCL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"Medial_OA_weak\",\n \"Lateral_OA_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Synovitis_weak\",\n \"Bakers_weak\",\n \"Contusion_weak\",\n \"Fracture_weak\",\n]\n\ncoverage_summary = pd.DataFrame({\n \"Abnormality\": weak_columns,\n \"Positive\": [\n (weak_labels[c] == 1).sum()\n for c in weak_columns\n ],\n \"Negative\": [\n (weak_labels[c] == 0).sum()\n for c in weak_columns\n ],\n \"Uncertain\": [\n weak_labels[c].isna().sum()\n for c in weak_columns\n ],\n})\n\ncoverage_summary[\"Labeled\"] = (\n coverage_summary[\"Positive\"]\n + coverage_summary[\"Negative\"]\n)\n\ncoverage_summary[\"Coverage_%\"] = (\n coverage_summary[\"Labeled\"]\n / len(weak_labels)\n * 100\n)\n\ncoverage_summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.348833Z","iopub.execute_input":"2026-09-12T08:38:48.349751Z","iopub.status.idle":"2026-09-12T08:38:48.378595Z","shell.execute_reply.started":"2026-09-12T08:38:48.349719Z","shell.execute_reply":"2026-09-12T08:38:48.377411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Weak-label quality summary\nquality_summary = coverage_summary.copy()\n\nquality_summary[\"Positive_%_of_labeled\"] = (\n quality_summary[\"Positive\"]\n / quality_summary[\"Labeled\"].replace(0, np.nan)\n * 100\n)\n\nquality_summary = quality_summary.sort_values(\n \"Coverage_%\",\n ascending=False\n).reset_index(drop=True)\n\nquality_summary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.380316Z","iopub.execute_input":"2026-09-12T08:38:48.381206Z","iopub.status.idle":"2026-09-12T08:38:48.4161Z","shell.execute_reply.started":"2026-09-12T08:38:48.381174Z","shell.execute_reply":"2026-09-12T08:38:48.414565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"core_weak = [\n \"ACL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Bakers_weak\",\n]\n\nsecondary_weak = [\n \"MCL_weak\",\n \"Fracture_weak\",\n \"Contusion_weak\",\n]\n\nauxiliary_weak = [\n \"Lateral_OA_weak\",\n \"Synovitis_weak\",\n \"Medial_OA_weak\",\n]\n\nprint(\"Core weak labels:\")\nprint(core_weak)\n\nprint(\"\\nSecondary weak labels:\")\nprint(secondary_weak)\n\nprint(\"\\nAuxiliary report labels:\")\nprint(auxiliary_weak)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.418675Z","iopub.execute_input":"2026-09-12T08:38:48.419607Z","iopub.status.idle":"2026-09-12T08:38:48.441749Z","shell.execute_reply.started":"2026-09-12T08:38:48.419524Z","shell.execute_reply":"2026-09-12T08:38:48.44076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_train = weak_labels.copy()\n# Keep only the columns we need\nweak_train = weak_train[\n [\"StudyInstanceUID\", \"Report\"] +\n weak_columns\n].copy()\n\nprint(\"Weak training pool:\")\nprint(\"Studies:\", len(weak_train))\nprint(\"Columns:\", weak_train.shape[1])\n\n# Show how many studies have at least one usable weak label\nhas_any_weak_label = weak_train[weak_columns].notna().any(axis=1)\n\nprint(\n \"Studies with at least one weak label:\",\n has_any_weak_label.sum()\n)\n\nprint(\n \"Studies with no weak labels:\",\n (~has_any_weak_label).sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.443151Z","iopub.execute_input":"2026-09-12T08:38:48.443592Z","iopub.status.idle":"2026-09-12T08:38:48.475886Z","shell.execute_reply.started":"2026-09-12T08:38:48.443509Z","shell.execute_reply":"2026-09-12T08:38:48.474283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"core_has_label = weak_train[core_weak].notna().any(axis=1)\n\ncore_weak_train = weak_train.loc[core_has_label].copy()\n\nprint(\"Core weak-supervision studies:\", len(core_weak_train))\n\nprint(\"\\nLabels available per study:\")\nprint(\n core_weak_train[core_weak]\n .notna()\n .sum(axis=1)\n .value_counts()\n .sort_index()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.478085Z","iopub.execute_input":"2026-09-12T08:38:48.478365Z","iopub.status.idle":"2026-09-12T08:38:48.494306Z","shell.execute_reply.started":"2026-09-12T08:38:48.47834Z","shell.execute_reply":"2026-09-12T08:38:48.493077Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"core_distribution = pd.DataFrame({\n \"Abnormality\": core_weak,\n \"Positive\": [\n (core_weak_train[c] == 1).sum()\n for c in core_weak\n ],\n \"Negative\": [\n (core_weak_train[c] == 0).sum()\n for c in core_weak\n ],\n \"Uncertain\": [\n core_weak_train[c].isna().sum()\n for c in core_weak\n ],\n})\n\ncore_distribution[\"Labeled\"] = (\n core_distribution[\"Positive\"]\n + core_distribution[\"Negative\"]\n)\n\ncore_distribution[\"Positive_%\"] = (\n core_distribution[\"Positive\"]\n / core_distribution[\"Labeled\"]\n * 100\n)\n\ncore_distribution","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:48.728814Z","iopub.execute_input":"2026-09-12T08:38:48.729277Z","iopub.status.idle":"2026-09-12T08:38:48.751062Z","shell.execute_reply.started":"2026-09-12T08:38:48.729137Z","shell.execute_reply":"2026-09-12T08:38:48.749584Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prepare weak-supervision dataframe\nweak_label_map = {\n \"ACL_weak\": \"ACL\",\n \"Medial_Meniscus_weak\": \"Medial Meniscus\",\n \"Lateral_Meniscus_weak\": \"Lateral Meniscus\",\n \"PF_OA_weak\": \"PF OA\",\n \"Effusion_weak\": \"Effusion\",\n \"Bakers_weak\": \"Baker's\",\n}\n\nweak_core_df = core_weak_train[\n [\"StudyInstanceUID\", \"Report\"] + core_weak\n].copy()\n\nweak_core_df = weak_core_df.rename(columns=weak_label_map)\n\nseries_info = study_index[\n [\n \"StudyInstanceUID\",\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n].copy()\n\nweak_core_df = weak_core_df.merge(\n series_info,\n on=\"StudyInstanceUID\",\n how=\"left\"\n)\n\nprint(\"Weak core dataframe shape:\", weak_core_df.shape)\nprint(\"Studies:\", len(weak_core_df))\n\nprint(\"\\nMissing MRI series:\")\nprint(\n weak_core_df[\n [\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n ].isna().sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:50.883624Z","iopub.execute_input":"2026-09-12T08:38:50.885214Z","iopub.status.idle":"2026-09-12T08:38:50.905265Z","shell.execute_reply.started":"2026-09-12T08:38:50.885008Z","shell.execute_reply":"2026-09-12T08:38:50.903926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"series_columns = [\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n]\n\nhas_all_planes = weak_core_df[series_columns].notna().all(axis=1)\n\nprint(\"Studies with all 3 MRI planes:\", has_all_planes.sum())\nprint(\"Studies missing at least one plane:\", (~has_all_planes).sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:51.223872Z","iopub.execute_input":"2026-09-12T08:38:51.224306Z","iopub.status.idle":"2026-09-12T08:38:51.235289Z","shell.execute_reply.started":"2026-09-12T08:38:51.22427Z","shell.execute_reply":"2026-09-12T08:38:51.233821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"weak_core_df StudyInstanceUID examples:\")\nprint(weak_core_df[\"StudyInstanceUID\"].head().tolist())\n\nprint(\"\\nstudy_index StudyInstanceUID examples:\")\nprint(study_index[\"StudyInstanceUID\"].head().tolist())\n\nprint(\"\\nData types:\")\nprint(\"weak_core_df:\",\n weak_core_df[\"StudyInstanceUID\"].dtype)\nprint(\"study_index:\",\n study_index[\"StudyInstanceUID\"].dtype)\n\n# Check overlap\nweak_ids = set(weak_core_df[\"StudyInstanceUID\"].astype(str))\nindex_ids = set(study_index[\"StudyInstanceUID\"].astype(str))\n\nprint(\"\\nWeak studies:\", len(weak_ids))\nprint(\"Study index studies:\", len(index_ids))\nprint(\"Matching StudyInstanceUIDs:\", len(weak_ids & index_ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:55.269459Z","iopub.execute_input":"2026-09-12T08:38:55.269772Z","iopub.status.idle":"2026-09-12T08:38:55.27984Z","shell.execute_reply.started":"2026-09-12T08:38:55.269746Z","shell.execute_reply":"2026-09-12T08:38:55.278634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_series shape:\", train_series.shape)\nprint(\"Columns:\")\nprint(train_series.columns.tolist())\n\ndisplay(train_series.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:55.607743Z","iopub.execute_input":"2026-09-12T08:38:55.608164Z","iopub.status.idle":"2026-09-12T08:38:55.621063Z","shell.execute_reply.started":"2026-09-12T08:38:55.608131Z","shell.execute_reply":"2026-09-12T08:38:55.619742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_ids = set(weak_core_df[\"StudyInstanceUID\"].astype(str))\nseries_ids = set(train_series[\"StudyInstanceUID\"].astype(str))\n\nprint(\"Weak studies:\", len(weak_ids))\nprint(\"Weak studies found in train_series:\", len(weak_ids & series_ids))\nprint(\"Weak studies missing from train_series:\", len(weak_ids - series_ids))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:56.011591Z","iopub.execute_input":"2026-09-12T08:38:56.011943Z","iopub.status.idle":"2026-09-12T08:38:56.026883Z","shell.execute_reply.started":"2026-09-12T08:38:56.011915Z","shell.execute_reply":"2026-09-12T08:38:56.025531Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select best MRI series for the weakly labeled studies\nweak_study_ids = set(\n weak_core_df[\"StudyInstanceUID\"].astype(str)\n)\n\nweak_series_info = train_series[\n train_series[\"StudyInstanceUID\"].astype(str).isin(weak_study_ids)\n].copy()\n\nprint(\"Weak-study series rows:\", len(weak_series_info))\nprint(\"Weak studies represented:\",\n weak_series_info[\"StudyInstanceUID\"].nunique())\n\nprint(\"\\nPlane distribution:\")\nprint(\n weak_series_info[\"Anatomical_Plane\"]\n .value_counts(dropna=False)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:59.140842Z","iopub.execute_input":"2026-09-12T08:38:59.141256Z","iopub.status.idle":"2026-09-12T08:38:59.160873Z","shell.execute_reply.started":"2026-09-12T08:38:59.141223Z","shell.execute_reply":"2026-09-12T08:38:59.159852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Apply the existing best-series selector\nweak_selected_rows = []\n\nfor study_uid in weak_study_ids:\n for plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n series_uid = select_best_series(study_uid, plane)\n\n weak_selected_rows.append({\n \"StudyInstanceUID\": study_uid,\n f\"{plane}_SeriesUID\": series_uid\n })\n\nweak_selected_series = pd.DataFrame(weak_selected_rows)\n\n# Convert the three plane rows into one row per study\nweak_selected_series = (\n weak_selected_series\n .groupby(\"StudyInstanceUID\", as_index=False)\n .first()\n)\n\nprint(\"Selected weak-study series shape:\",\n weak_selected_series.shape)\n\nprint(\"\\nMissing selected series:\")\nprint(\n weak_selected_series[\n [\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n ].isna().sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:38:59.459791Z","iopub.execute_input":"2026-09-12T08:38:59.460614Z","iopub.status.idle":"2026-09-12T08:39:05.282645Z","shell.execute_reply.started":"2026-09-12T08:38:59.460576Z","shell.execute_reply":"2026-09-12T08:39:05.281118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_study = weak_core_df[\"StudyInstanceUID\"].iloc[0]\n\nprint(\"Test study:\")\nprint(test_study)\n\nprint(\"\\nAll series for this study:\")\ndisplay(\n train_series[\n train_series[\"StudyInstanceUID\"].astype(str) == str(test_study)\n ]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:39:05.284642Z","iopub.execute_input":"2026-09-12T08:39:05.285147Z","iopub.status.idle":"2026-09-12T08:39:05.301467Z","shell.execute_reply.started":"2026-09-12T08:39:05.285103Z","shell.execute_reply":"2026-09-12T08:39:05.300537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n result = select_best_series(test_study, plane)\n print(f\"{plane}: {result}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:39:05.302938Z","iopub.execute_input":"2026-09-12T08:39:05.303467Z","iopub.status.idle":"2026-09-12T08:39:05.327631Z","shell.execute_reply.started":"2026-09-12T08:39:05.303416Z","shell.execute_reply":"2026-09-12T08:39:05.326524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def select_best_weak_series(study_uid, plane):\n candidates = train_series[\n (train_series[\"StudyInstanceUID\"].astype(str) == str(study_uid)) &\n (train_series[\"Anatomical_Plane\"] == plane)\n ].copy()\n\n if candidates.empty:\n return np.nan\n\n candidates[\"priority\"] = (\n candidates[\"Fluid_Sensitive\"].astype(int) * 2\n + candidates[\"Fat_Suppression\"].astype(int)\n )\n\n candidates = candidates.sort_values(\n \"priority\",\n ascending=False\n )\n\n return candidates.iloc[0][\"SeriesInstanceUID\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:39:05.329709Z","iopub.execute_input":"2026-09-12T08:39:05.330559Z","iopub.status.idle":"2026-09-12T08:39:05.350334Z","shell.execute_reply.started":"2026-09-12T08:39:05.330526Z","shell.execute_reply":"2026-09-12T08:39:05.349148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for plane in [\"Axial\", \"Coronal\", \"Sagittal\"]:\n result = select_best_weak_series(test_study, plane)\n print(f\"{plane}: {result}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:39:05.574429Z","iopub.execute_input":"2026-09-12T08:39:05.575273Z","iopub.status.idle":"2026-09-12T08:39:05.604093Z","shell.execute_reply.started":"2026-09-12T08:39:05.575199Z","shell.execute_reply":"2026-09-12T08:39:05.602936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_selected_rows = []\n\nfor study_uid in weak_study_ids:\n weak_selected_rows.append({\n \"StudyInstanceUID\": study_uid,\n \"Axial_SeriesUID\":\n select_best_weak_series(study_uid, \"Axial\"),\n \"Coronal_SeriesUID\":\n select_best_weak_series(study_uid, \"Coronal\"),\n \"Sagittal_SeriesUID\":\n select_best_weak_series(study_uid, \"Sagittal\"),\n })\n\nweak_selected_series = pd.DataFrame(weak_selected_rows)\n\nprint(\"Selected weak-study series shape:\",\n weak_selected_series.shape)\n\nprint(\"\\nMissing selected series:\")\nprint(\n weak_selected_series[\n [\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n ].isna().sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:39:05.867429Z","iopub.execute_input":"2026-09-12T08:39:05.867906Z","iopub.status.idle":"2026-09-12T08:40:10.416685Z","shell.execute_reply.started":"2026-09-12T08:39:05.867872Z","shell.execute_reply":"2026-09-12T08:40:10.415589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\nStudies with all 3 planes:\")\nprint(\n weak_selected_series[\n [\"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"]\n ].notna().all(axis=1).sum()\n)\n\nprint(\"\\nStudies missing at least one plane:\")\nprint(\n weak_selected_series[\n [\"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"]\n ].isna().any(axis=1).sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:10.418761Z","iopub.execute_input":"2026-09-12T08:40:10.419201Z","iopub.status.idle":"2026-09-12T08:40:10.43334Z","shell.execute_reply.started":"2026-09-12T08:40:10.419169Z","shell.execute_reply":"2026-09-12T08:40:10.432128Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_core_labels = [\n \"ACL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"PF OA\",\n \"Effusion\",\n \"Baker's\"\n]\nprint(\"Weak dataframe columns:\")\nprint(weak_core_df.columns.tolist())\n\nweak_dataset_labels = {\n \"ACL\": \"ACL_weak\",\n \"Medial Meniscus\": \"Medial_Meniscus_weak\",\n \"Lateral Meniscus\": \"Lateral_Meniscus_weak\",\n \"PF OA\": \"PF_OA_weak\",\n \"Effusion\": \"Effusion_weak\",\n \"Baker's\": \"Bakers_weak\"\n}\n\nweak_dataset = (\n weak_core_df[\n [\"StudyInstanceUID\"] + weak_core_labels\n ]\n .rename(columns=weak_dataset_labels)\n .merge(\n weak_selected_series,\n on=\"StudyInstanceUID\",\n how=\"inner\"\n )\n)\n\nprint(\"Final weak dataset shape:\", weak_dataset.shape)\n\ndisplay(weak_dataset.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:10.434797Z","iopub.execute_input":"2026-09-12T08:40:10.435425Z","iopub.status.idle":"2026-09-12T08:40:10.482474Z","shell.execute_reply.started":"2026-09-12T08:40:10.43539Z","shell.execute_reply":"2026-09-12T08:40:10.481396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Number of studies:\", len(weak_dataset))\n\nprint(\"\\nMissing MRI series:\")\nprint(\n weak_dataset[\n [\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n ].isna().sum()\n)\n\nprint(\"\\nWeak-label availability:\")\nprint(\n weak_dataset[\n list(weak_dataset_labels.values())\n ].notna().sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:10.485159Z","iopub.execute_input":"2026-09-12T08:40:10.485454Z","iopub.status.idle":"2026-09-12T08:40:10.498465Z","shell.execute_reply.started":"2026-09-12T08:40:10.485429Z","shell.execute_reply":"2026-09-12T08:40:10.497407Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class RSNAWeakKneeDataset(Dataset):\n\n def __init__(self, dataframe, target_shape=(32, 128, 128)):\n self.dataframe = dataframe.reset_index(drop=True)\n self.target_shape = target_shape\n\n def __len__(self):\n return len(self.dataframe)\n\n def _load_plane(self, study_uid, series_uid):\n volume = load_series_volume(study_uid, series_uid)\n\n if volume is None:\n return torch.zeros(\n (1, *self.target_shape),\n dtype=torch.float32\n )\n\n return prepare_volume(\n volume,\n target_shape=self.target_shape\n )\n\n def __getitem__(self, idx):\n row = self.dataframe.iloc[idx]\n study_uid = row[\"StudyInstanceUID\"]\n\n axial = self._load_plane(\n study_uid,\n row[\"Axial_SeriesUID\"]\n )\n\n coronal = self._load_plane(\n study_uid,\n row[\"Coronal_SeriesUID\"]\n )\n\n sagittal = self._load_plane(\n study_uid,\n row[\"Sagittal_SeriesUID\"]\n )\n\n # 3 channels = Axial, Coronal, Sagittal\n image = torch.cat(\n [axial, coronal, sagittal],\n dim=0\n )\n\n # Weak labels\n label_columns = [\n \"ACL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Bakers_weak\"\n ]\n\n labels = torch.tensor(\n row[label_columns].values.astype(np.float32),\n dtype=torch.float32\n )\n\n mask = ~torch.isnan(labels)\n labels = torch.nan_to_num(labels, nan=0.0)\n\n return image, labels, mask\n\n\nweak_torch_dataset = RSNAWeakKneeDataset(\n weak_dataset,\n target_shape=(32, 128, 128)\n)\n\nimage, labels, mask = weak_torch_dataset[0]\n\nprint(\"Single study\")\nprint(\"Image shape :\", image.shape)\nprint(\"Image dtype :\", image.dtype)\nprint(\"Image range :\", image.min().item(), \"to\", image.max().item())\nprint(\"Labels :\", labels)\nprint(\"Mask :\", mask)\nprint(\"Known labels:\", mask.sum().item())\n\nweak_loader = DataLoader(\n weak_torch_dataset,\n batch_size=2,\n shuffle=True,\n num_workers=0\n)\n\nimages, labels, masks = next(iter(weak_loader))\n\nprint(\"\\nBatch test\")\nprint(\"Images :\", images.shape)\nprint(\"Labels :\", labels.shape)\nprint(\"Masks :\", masks.shape)\nprint(\"NaNs in labels:\", torch.isnan(labels).sum().item())\nprint(\"Known labels in batch:\", masks.sum().item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:10.499537Z","iopub.execute_input":"2026-09-12T08:40:10.499958Z","iopub.status.idle":"2026-09-12T08:40:14.223671Z","shell.execute_reply.started":"2026-09-12T08:40:10.499924Z","shell.execute_reply":"2026-09-12T08:40:14.222822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn.functional as F","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:14.233385Z","iopub.execute_input":"2026-09-12T08:40:14.233927Z","iopub.status.idle":"2026-09-12T08:40:14.261479Z","shell.execute_reply.started":"2026-09-12T08:40:14.233881Z","shell.execute_reply":"2026-09-12T08:40:14.260147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def masked_bce_loss(logits, labels, mask, pos_weight=None):\n\n if pos_weight is not None:\n loss = F.binary_cross_entropy_with_logits(\n logits,\n labels,\n pos_weight=pos_weight,\n reduction=\"none\"\n )\n else:\n loss = F.binary_cross_entropy_with_logits(\n logits,\n labels,\n reduction=\"none\"\n )\n\n # Ignore uncertain / NaN report labels\n loss = loss * mask.float()\n\n # Average only over known labels\n denominator = mask.float().sum().clamp(min=1.0)\n\n return loss.sum() / denominator","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:40:36.001156Z","iopub.execute_input":"2026-09-12T08:40:36.001509Z","iopub.status.idle":"2026-09-12T08:40:36.009278Z","shell.execute_reply.started":"2026-09-12T08:40:36.001481Z","shell.execute_reply":"2026-09-12T08:40:36.008076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = RSNAKnee3DCNN(num_classes=6)\n\nimages, labels, masks = next(iter(weak_loader))\n\nlogits = model(images)\n\nloss = masked_bce_loss(\n logits,\n labels,\n masks\n)\n\nprint(\"Logits shape:\", logits.shape)\nprint(\"Labels shape:\", labels.shape)\nprint(\"Loss:\", loss.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:51:17.237431Z","iopub.execute_input":"2026-09-12T08:51:17.239501Z","iopub.status.idle":"2026-09-12T08:51:20.230946Z","shell.execute_reply.started":"2026-09-12T08:51:17.239402Z","shell.execute_reply":"2026-09-12T08:51:20.229599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = RSNAKnee3DCNN(num_classes=6)\n\noptimizer = torch.optim.Adam(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\n# One small training epoch to verify the complete pipeline\nmodel.train()\n\nrunning_loss = 0.0\n\nfor batch_idx, (images, labels, masks) in enumerate(weak_loader):\n\n optimizer.zero_grad()\n\n logits = model(images)\n\n loss = masked_bce_loss(\n logits,\n labels,\n masks\n )\n\n loss.backward()\n optimizer.step()\n\n running_loss += loss.item()\n\n print(\n f\"Batch {batch_idx + 1}/{len(weak_loader)} \"\n f\"- Loss: {loss.item():.4f}\"\n )\n\n if batch_idx == 4:\n break\n\nprint(\"\\n5-batch training test complete.\")\nprint(\"Average loss:\",\n running_loss / min(5, len(weak_loader)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:51:25.360236Z","iopub.execute_input":"2026-09-12T08:51:25.360928Z","iopub.status.idle":"2026-09-12T08:51:42.598552Z","shell.execute_reply.started":"2026-09-12T08:51:25.360894Z","shell.execute_reply":"2026-09-12T08:51:42.597453Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport torch\nfrom tqdm.auto import tqdm\n\nCACHE_DIR = \"/kaggle/working/weak_mri_cache\"\nos.makedirs(CACHE_DIR, exist_ok=True)\n\ntarget_shape = (32, 128, 128)\n\ndef cache_weak_volume(row):\n study_uid = str(row[\"StudyInstanceUID\"])\n cache_path = os.path.join(CACHE_DIR, f\"{study_uid}.pt\")\n\n # Skip if already cached\n if os.path.exists(cache_path):\n return True\n\n try:\n axial = prepare_volume(\n load_series_volume(study_uid, row[\"Axial_SeriesUID\"]),\n target_shape=target_shape\n )\n\n coronal = prepare_volume(\n load_series_volume(study_uid, row[\"Coronal_SeriesUID\"]),\n target_shape=target_shape\n )\n\n sagittal = prepare_volume(\n load_series_volume(study_uid, row[\"Sagittal_SeriesUID\"]),\n target_shape=target_shape\n )\n\n image = torch.cat([axial, coronal, sagittal], dim=0)\n\n torch.save(image, cache_path)\n\n return True\n\n except Exception as e:\n print(f\"Failed: {study_uid} -> {e}\")\n return False\n\n\nsuccessful = 0\n\nfor _, row in tqdm(\n weak_dataset.iterrows(),\n total=len(weak_dataset),\n desc=\"Caching weak MRI volumes\"\n):\n if cache_weak_volume(row):\n successful += 1\n\nprint(f\"\\nCaching complete.\")\nprint(f\"Successfully cached: {successful}/{len(weak_dataset)}\")\nprint(f\"Cache directory: {CACHE_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T08:54:55.544715Z","iopub.execute_input":"2026-09-12T08:54:55.546325Z","iopub.status.idle":"2026-09-12T10:02:36.984009Z","shell.execute_reply.started":"2026-09-12T08:54:55.546251Z","shell.execute_reply":"2026-09-12T10:02:36.98257Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CachedWeakKneeDataset(Dataset):\n\n def __init__(self, dataframe, cache_dir):\n self.dataframe = dataframe.reset_index(drop=True)\n self.cache_dir = cache_dir\n\n self.label_columns = [\n \"ACL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Bakers_weak\"\n ]\n\n def __len__(self):\n return len(self.dataframe)\n\n def __getitem__(self, idx):\n\n row = self.dataframe.iloc[idx]\n\n study_uid = str(row[\"StudyInstanceUID\"])\n cache_path = os.path.join(\n self.cache_dir,\n f\"{study_uid}.pt\"\n )\n\n image = torch.load(\n cache_path,\n map_location=\"cpu\",\n weights_only=True\n )\n\n labels = torch.tensor(\n row[self.label_columns].values.astype(np.float32),\n dtype=torch.float32\n )\n\n # True = known weak label\n # False = unknown / NaN\n mask = ~torch.isnan(labels)\n\n # Replace NaN with 0 because the mask controls whether\n # that position contributes to the loss.\n labels = torch.nan_to_num(labels, nan=0.0)\n\n return image, labels, mask","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T10:08:40.632362Z","iopub.execute_input":"2026-09-12T10:08:40.63375Z","iopub.status.idle":"2026-09-12T10:08:40.643265Z","shell.execute_reply.started":"2026-09-12T10:08:40.63369Z","shell.execute_reply":"2026-09-12T10:08:40.642214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cached_weak_dataset_df = weak_dataset[\n weak_dataset[\"StudyInstanceUID\"].apply(\n lambda x: os.path.exists(\n os.path.join(\n CACHE_DIR,\n f\"{x}.pt\"\n )\n )\n )\n].reset_index(drop=True)\n\nprint(\"Original weak studies:\", len(weak_dataset))\nprint(\"Cached weak studies:\", len(cached_weak_dataset_df))\nprint(\"Excluded studies:\", len(weak_dataset) - len(cached_weak_dataset_df))\n\ncached_weak_dataset = CachedWeakKneeDataset(\n cached_weak_dataset_df,\n CACHE_DIR\n)\n\ncached_weak_loader = DataLoader(\n cached_weak_dataset,\n batch_size=2,\n shuffle=True,\n num_workers=0\n)\n\nprint(\"Batches:\", len(cached_weak_loader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T10:08:53.122205Z","iopub.execute_input":"2026-09-12T10:08:53.122834Z","iopub.status.idle":"2026-09-12T10:08:53.183567Z","shell.execute_reply.started":"2026-09-12T10:08:53.122801Z","shell.execute_reply":"2026-09-12T10:08:53.182145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images, labels, masks = next(iter(cached_weak_loader))\n\nprint(\"Images:\", images.shape)\nprint(\"Labels:\", labels.shape)\nprint(\"Masks:\", masks.shape)\nprint(\"Known labels:\", masks.sum().item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T10:09:07.972701Z","iopub.execute_input":"2026-09-12T10:09:07.973588Z","iopub.status.idle":"2026-09-12T10:09:08.165714Z","shell.execute_reply.started":"2026-09-12T10:09:07.973549Z","shell.execute_reply":"2026-09-12T10:09:08.164579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nprint(\"Device:\", device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T10:54:32.011535Z","iopub.execute_input":"2026-09-12T10:54:32.01231Z","iopub.status.idle":"2026-09-12T10:54:32.028876Z","shell.execute_reply.started":"2026-09-12T10:54:32.012266Z","shell.execute_reply":"2026-09-12T10:54:32.027406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fresh 6-class model for weak-supervised training\n\nweak_model = RSNAKnee3DCNN(num_classes=6).to(device)\n\nweak_optimizer = torch.optim.Adam(\n weak_model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nweak_model.train()\n\nrunning_loss = 0.0\n\nfor batch_idx, (images, labels, masks) in enumerate(cached_weak_loader):\n\n images = images.to(device)\n labels = labels.to(device)\n masks = masks.to(device)\n\n weak_optimizer.zero_grad()\n\n logits = weak_model(images)\n\n loss = masked_bce_loss(\n logits,\n labels,\n masks\n )\n\n loss.backward()\n weak_optimizer.step()\n\n running_loss += loss.item()\n\n if (batch_idx + 1) % 100 == 0:\n print(\n f\"Batch {batch_idx + 1}/{len(cached_weak_loader)} \"\n f\"- Loss: {loss.item():.4f}\"\n )\n\nprint(\"\\nWeak-supervised training epoch complete.\")\nprint(\"Average loss:\", running_loss / len(cached_weak_loader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T10:54:35.696228Z","iopub.execute_input":"2026-09-12T10:54:35.696695Z","iopub.status.idle":"2026-09-12T11:19:53.482738Z","shell.execute_reply.started":"2026-09-12T10:54:35.696653Z","shell.execute_reply":"2026-09-12T11:19:53.479446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MODEL_PATH = \"/kaggle/working/weak_supervised_model_epoch1.pth\"\n\ntorch.save({\n \"model_state_dict\": weak_model.state_dict(),\n \"optimizer_state_dict\": weak_optimizer.state_dict(),\n \"epoch\": 1,\n \"average_loss\": running_loss / len(cached_weak_loader)\n}, MODEL_PATH)\n\nprint(\"Model saved successfully.\")\nprint(\"Path:\", MODEL_PATH)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:28:20.076654Z","iopub.execute_input":"2026-09-12T11:28:20.078132Z","iopub.status.idle":"2026-09-12T11:28:20.105226Z","shell.execute_reply.started":"2026-09-12T11:28:20.078087Z","shell.execute_reply":"2026-09-12T11:28:20.103447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nimport os\n\nCLEAN_CACHE_DIR = \"/kaggle/working/clean_eval_cache\"\n\nif os.path.exists(CLEAN_CACHE_DIR):\n shutil.rmtree(CLEAN_CACHE_DIR)\n\nos.makedirs(CLEAN_CACHE_DIR, exist_ok=True)\n\nprint(\"Clean evaluation cache reset.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:38:11.912144Z","iopub.execute_input":"2026-09-12T11:38:11.915831Z","iopub.status.idle":"2026-09-12T11:38:12.033767Z","shell.execute_reply.started":"2026-09-12T11:38:11.915712Z","shell.execute_reply":"2026-09-12T11:38:12.032365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_eval_labels = [\n \"ACL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"PF OA\",\n \"Effusion\",\n \"Baker's\"\n]\n\nweak_eval_df = (\n val_df[[\"StudyInstanceUID\"] + weak_eval_labels]\n .merge(\n study_index[\n [\n \"StudyInstanceUID\",\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n ]\n ],\n on=\"StudyInstanceUID\",\n how=\"left\"\n )\n)\n\nprint(\"Clean validation studies:\", len(weak_eval_df))\nprint(weak_eval_df[[\n \"StudyInstanceUID\",\n \"Axial_SeriesUID\",\n \"Coronal_SeriesUID\",\n \"Sagittal_SeriesUID\"\n]])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:38:34.217211Z","iopub.execute_input":"2026-09-12T11:38:34.217857Z","iopub.status.idle":"2026-09-12T11:38:34.254231Z","shell.execute_reply.started":"2026-09-12T11:38:34.217785Z","shell.execute_reply":"2026-09-12T11:38:34.252532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_prepared_or_zero(study_uid, series_uid, target_shape=(32,128,128)):\n if pd.isna(series_uid):\n return torch.zeros((1, *target_shape), dtype=torch.float32)\n\n volume = load_series_volume(study_uid, series_uid)\n\n if volume is None:\n return torch.zeros((1, *target_shape), dtype=torch.float32)\n\n return prepare_volume(volume, target_shape=target_shape)\n\n\ndef cache_clean_volume(row):\n study_uid = str(row[\"StudyInstanceUID\"])\n cache_path = os.path.join(\n CLEAN_CACHE_DIR,\n f\"{study_uid}.pt\"\n )\n\n if os.path.exists(cache_path):\n return True\n\n try:\n axial = load_prepared_or_zero(\n study_uid,\n row[\"Axial_SeriesUID\"]\n )\n\n coronal = load_prepared_or_zero(\n study_uid,\n row[\"Coronal_SeriesUID\"]\n )\n\n sagittal = load_prepared_or_zero(\n study_uid,\n row[\"Sagittal_SeriesUID\"]\n )\n\n image = torch.cat(\n [axial, coronal, sagittal],\n dim=0\n )\n\n torch.save(image, cache_path)\n\n return True\n\n except Exception as e:\n print(f\"Failed: {study_uid} -> {e}\")\n return False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:38:54.987798Z","iopub.execute_input":"2026-09-12T11:38:54.98853Z","iopub.status.idle":"2026-09-12T11:38:54.999807Z","shell.execute_reply.started":"2026-09-12T11:38:54.98849Z","shell.execute_reply":"2026-09-12T11:38:54.998477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\nsuccess = 0\n\nfor _, row in tqdm(\n weak_eval_df.iterrows(),\n total=len(weak_eval_df),\n desc=\"Caching clean validation MRI\"\n):\n if cache_clean_volume(row):\n success += 1\n\nprint(f\"Successfully cached: {success}/{len(weak_eval_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:39:08.130323Z","iopub.execute_input":"2026-09-12T11:39:08.130776Z","iopub.status.idle":"2026-09-12T11:39:14.337457Z","shell.execute_reply.started":"2026-09-12T11:39:08.130743Z","shell.execute_reply":"2026-09-12T11:39:14.335928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the weakly supervised model checkpoint\n\ncheckpoint_path = \"/kaggle/working/weak_supervised_model_epoch1.pth\"\n\ncheckpoint = torch.load(\n checkpoint_path,\n map_location=device,\n weights_only=True\n)\n\nweak_model = RSNAKnee3DCNN(num_classes=6).to(device)\n\nweak_model.load_state_dict(\n checkpoint[\"model_state_dict\"]\n)\n\nprint(\"Checkpoint loaded successfully.\")\nprint(\"Epoch:\", checkpoint[\"epoch\"])\nprint(\"Average training loss:\", checkpoint[\"average_loss\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:42:37.104213Z","iopub.execute_input":"2026-09-12T11:42:37.104709Z","iopub.status.idle":"2026-09-12T11:42:37.135216Z","shell.execute_reply.started":"2026-09-12T11:42:37.104673Z","shell.execute_reply":"2026-09-12T11:42:37.133894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nclass CachedCleanEvalDataset(Dataset):\n def __init__(self, dataframe, cache_dir):\n self.dataframe = dataframe.reset_index(drop=True)\n self.cache_dir = cache_dir\n self.label_columns = [\n \"ACL\",\n \"Medial Meniscus\",\n \"Lateral Meniscus\",\n \"PF OA\",\n \"Effusion\",\n \"Baker's\"\n ]\n\n def __len__(self):\n return len(self.dataframe)\n\n def __getitem__(self, idx):\n row = self.dataframe.iloc[idx]\n\n study_uid = str(row[\"StudyInstanceUID\"])\n cache_path = os.path.join(\n self.cache_dir,\n f\"{study_uid}.pt\"\n )\n\n image = torch.load(\n cache_path,\n map_location=\"cpu\",\n weights_only=True\n )\n\n labels = torch.tensor(\n row[self.label_columns].values.astype(np.float32),\n dtype=torch.float32\n )\n\n return image, labels, study_uid\n\n\nclean_eval_dataset = CachedCleanEvalDataset(\n weak_eval_df,\n CLEAN_CACHE_DIR\n)\n\nclean_eval_loader = DataLoader(\n clean_eval_dataset,\n batch_size=2,\n shuffle=False,\n num_workers=0\n)\n\nprint(\"Validation studies:\", len(clean_eval_dataset))\nprint(\"Validation batches:\", len(clean_eval_loader))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:39:48.455863Z","iopub.execute_input":"2026-09-12T11:39:48.457108Z","iopub.status.idle":"2026-09-12T11:39:48.472858Z","shell.execute_reply.started":"2026-09-12T11:39:48.457014Z","shell.execute_reply":"2026-09-12T11:39:48.471585Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_model.eval()\n\nall_predictions = []\nall_labels = []\nall_study_ids = []\n\nwith torch.no_grad():\n\n for images, labels, study_ids in clean_eval_loader:\n\n images = images.to(device)\n\n logits = weak_model(images)\n probabilities = torch.sigmoid(logits)\n\n all_predictions.append(\n probabilities.cpu().numpy()\n )\n\n all_labels.append(\n labels.numpy()\n )\n\n all_study_ids.extend(study_ids)\n\n\nall_predictions = np.concatenate(all_predictions, axis=0)\nall_labels = np.concatenate(all_labels, axis=0)\n\nprint(\"Predictions shape:\", all_predictions.shape)\nprint(\"Labels shape:\", all_labels.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:43:00.931348Z","iopub.execute_input":"2026-09-12T11:43:00.932393Z","iopub.status.idle":"2026-09-12T11:43:02.293311Z","shell.execute_reply.started":"2026-09-12T11:43:00.932354Z","shell.execute_reply":"2026-09-12T11:43:02.291784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nweak_auc_results = {}\n\nfor i, label in enumerate(weak_eval_labels):\n\n y_true = all_labels[:, i]\n y_prob = all_predictions[:, i]\n\n if len(np.unique(y_true)) < 2:\n weak_auc_results[label] = np.nan\n print(f\"{label}: N/A\")\n else:\n auc = roc_auc_score(y_true, y_prob)\n weak_auc_results[label] = auc\n print(f\"{label}: {auc:.4f}\")\n\n\nvalid_aucs = [\n value\n for value in weak_auc_results.values()\n if not np.isnan(value)\n]\n\nprint(\"\\nMean ROC-AUC:\", np.mean(valid_aucs))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:43:21.10898Z","iopub.execute_input":"2026-09-12T11:43:21.109451Z","iopub.status.idle":"2026-09-12T11:43:21.14766Z","shell.execute_reply.started":"2026-09-12T11:43:21.109412Z","shell.execute_reply":"2026-09-12T11:43:21.146312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Weak model prediction statistics:\\n\")\n\nfor i, label in enumerate(weak_eval_labels):\n probs = all_predictions[:, i]\n true = all_labels[:, i]\n\n print(\n f\"{label:20s} | \"\n f\"True positive rate: {true.mean():.2f} | \"\n f\"Mean prediction: {probs.mean():.3f} | \"\n f\"Min: {probs.min():.3f} | \"\n f\"Max: {probs.max():.3f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:44:26.410076Z","iopub.execute_input":"2026-09-12T11:44:26.411416Z","iopub.status.idle":"2026-09-12T11:44:26.419437Z","shell.execute_reply.started":"2026-09-12T11:44:26.411361Z","shell.execute_reply":"2026-09-12T11:44:26.418187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Continue weak-supervised training from epoch 1\n\ncheckpoint_path = \"/kaggle/working/weak_supervised_model_epoch1.pth\"\n\ncheckpoint = torch.load(\n checkpoint_path,\n map_location=device,\n weights_only=True\n)\n\nweak_model = RSNAKnee3DCNN(num_classes=6).to(device)\n\nweak_model.load_state_dict(\n checkpoint[\"model_state_dict\"]\n)\n\nweak_optimizer = torch.optim.Adam(\n weak_model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nweak_optimizer.load_state_dict(\n checkpoint[\"optimizer_state_dict\"]\n)\n\nweak_model.train()\n\nrunning_loss = 0.0\n\nfor batch_idx, (images, labels, masks) in enumerate(cached_weak_loader):\n\n images = images.to(device)\n labels = labels.to(device)\n masks = masks.to(device)\n\n weak_optimizer.zero_grad()\n\n logits = weak_model(images)\n\n loss = masked_bce_loss(\n logits,\n labels,\n masks\n )\n\n loss.backward()\n weak_optimizer.step()\n\n running_loss += loss.item()\n\n if (batch_idx + 1) % 200 == 0:\n print(\n f\"Batch {batch_idx + 1}/{len(cached_weak_loader)} \"\n f\"- Loss: {loss.item():.4f}\"\n )\n\nepoch2_loss = running_loss / len(cached_weak_loader)\n\nprint(\"\\nEpoch 2 average loss:\", epoch2_loss)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T11:44:45.313773Z","iopub.execute_input":"2026-09-12T11:44:45.314877Z","iopub.status.idle":"2026-09-12T12:06:50.890829Z","shell.execute_reply.started":"2026-09-12T11:44:45.314837Z","shell.execute_reply":"2026-09-12T12:06:50.889333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epoch2_checkpoint = \"/kaggle/working/weak_supervised_model_epoch2.pth\"\n\ntorch.save(\n {\n \"epoch\": 2,\n \"model_state_dict\": weak_model.state_dict(),\n \"optimizer_state_dict\": weak_optimizer.state_dict(),\n \"average_loss\": epoch2_loss\n },\n epoch2_checkpoint\n)\n\nprint(\"Saved:\", epoch2_checkpoint)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:08:21.673128Z","iopub.execute_input":"2026-09-12T12:08:21.6738Z","iopub.status.idle":"2026-09-12T12:08:21.69301Z","shell.execute_reply.started":"2026-09-12T12:08:21.673761Z","shell.execute_reply":"2026-09-12T12:08:21.691488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate Epoch 2 on the same clean 12-study validation set\n\nweak_model.eval()\n\nall_predictions_epoch2 = []\nall_labels_epoch2 = []\n\nwith torch.no_grad():\n\n for images, labels, study_ids in clean_eval_loader:\n\n images = images.to(device)\n\n logits = weak_model(images)\n probabilities = torch.sigmoid(logits)\n\n all_predictions_epoch2.append(\n probabilities.cpu().numpy()\n )\n\n all_labels_epoch2.append(\n labels.numpy()\n )\n\n\nall_predictions_epoch2 = np.concatenate(\n all_predictions_epoch2,\n axis=0\n)\n\nall_labels_epoch2 = np.concatenate(\n all_labels_epoch2,\n axis=0\n)\n\nprint(\"Predictions:\", all_predictions_epoch2.shape)\nprint(\"Labels:\", all_labels_epoch2.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:09:30.763199Z","iopub.execute_input":"2026-09-12T12:09:30.764173Z","iopub.status.idle":"2026-09-12T12:09:32.210467Z","shell.execute_reply.started":"2026-09-12T12:09:30.763999Z","shell.execute_reply":"2026-09-12T12:09:32.20927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"weak_auc_epoch2 = {}\n\nfor i, label in enumerate(weak_eval_labels):\n\n y_true = all_labels_epoch2[:, i]\n y_prob = all_predictions_epoch2[:, i]\n\n if len(np.unique(y_true)) < 2:\n weak_auc_epoch2[label] = np.nan\n print(f\"{label}: N/A\")\n else:\n auc = roc_auc_score(y_true, y_prob)\n weak_auc_epoch2[label] = auc\n print(f\"{label}: {auc:.4f}\")\n\n\nvalid_aucs = [\n x for x in weak_auc_epoch2.values()\n if not np.isnan(x)\n]\n\nprint(\"\\nEpoch 2 Mean ROC-AUC:\", np.mean(valid_aucs))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:09:43.744699Z","iopub.execute_input":"2026-09-12T12:09:43.745116Z","iopub.status.idle":"2026-09-12T12:09:43.777985Z","shell.execute_reply.started":"2026-09-12T12:09:43.745074Z","shell.execute_reply":"2026-09-12T12:09:43.776742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndef get_size_gb(path):\n total = 0\n\n if os.path.isfile(path):\n total = os.path.getsize(path)\n\n elif os.path.isdir(path):\n for root, dirs, files in os.walk(path):\n for file in files:\n try:\n total += os.path.getsize(os.path.join(root, file))\n except OSError:\n pass\n\n return total / (1024 ** 3)\n\n\nprint(\"Weak MRI cache:\",\n round(get_size_gb(\"/kaggle/working/weak_mri_cache\"), 2),\n \"GB\")\n\nprint(\"Clean validation cache:\",\n round(get_size_gb(\"/kaggle/working/clean_eval_cache\"), 2),\n \"GB\")\n\nprint(\"Epoch 1 model:\",\n round(get_size_gb(\"/kaggle/working/weak_supervised_model_epoch1.pth\"), 3),\n \"GB\")\n\nprint(\"Epoch 2 model:\",\n round(get_size_gb(\"/kaggle/working/weak_supervised_model_epoch2.pth\"), 3),\n \"GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:19:42.96418Z","iopub.execute_input":"2026-09-12T12:19:42.966948Z","iopub.status.idle":"2026-09-12T12:19:43.193453Z","shell.execute_reply.started":"2026-09-12T12:19:42.966889Z","shell.execute_reply":"2026-09-12T12:19:43.192325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\nimport os\n\nWEAK_CACHE_DIR = \"/kaggle/working/weak_mri_cache\"\n\nif os.path.exists(WEAK_CACHE_DIR):\n shutil.rmtree(WEAK_CACHE_DIR)\n print(\"Weak MRI cache deleted.\")\nelse:\n print(\"Weak MRI cache not found.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:20:29.270863Z","iopub.execute_input":"2026-09-12T12:20:29.27144Z","iopub.status.idle":"2026-09-12T12:20:31.61691Z","shell.execute_reply.started":"2026-09-12T12:20:29.271403Z","shell.execute_reply":"2026-09-12T12:20:31.615439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Weak cache exists:\", os.path.exists(WEAK_CACHE_DIR))\n\nprint(\"Epoch 1 exists:\",\n os.path.exists(\"/kaggle/working/weak_supervised_model_epoch1.pth\"))\n\nprint(\"Epoch 2 exists:\",\n os.path.exists(\"/kaggle/working/weak_supervised_model_epoch2.pth\"))\n\nprint(\"Clean validation cache exists:\",\n os.path.exists(\"/kaggle/working/clean_eval_cache\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:47:26.131398Z","iopub.execute_input":"2026-09-12T12:47:26.132607Z","iopub.status.idle":"2026-09-12T12:47:26.141635Z","shell.execute_reply.started":"2026-09-12T12:47:26.132554Z","shell.execute_reply":"2026-09-12T12:47:26.140367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"baseline_model = RSNAKnee3DCNN(num_classes=12).to(device)\n\ncriterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight)\n\noptimizer = torch.optim.Adam(\n baseline_model.parameters(),\n lr=1e-4,\n weight_decay=1e-5\n)\n\nnum_epochs = 10\n\nfor epoch in range(num_epochs):\n baseline_model.train()\n running_loss = 0.0\n\n for batch_idx, (images, labels) in enumerate(train_loader):\n images = images.to(device)\n labels = labels.float().to(device)\n\n optimizer.zero_grad()\n\n logits = baseline_model(images)\n loss = criterion(logits, labels)\n\n loss.backward()\n optimizer.step()\n\n running_loss += loss.item()\n\n train_loss = running_loss / len(train_loader)\n\n # Validation\n baseline_model.eval()\n val_loss = 0.0\n\n with torch.no_grad():\n for images, labels in val_loader:\n images = images.to(device)\n labels = labels.float().to(device)\n\n logits = baseline_model(images)\n loss = criterion(logits, labels)\n\n val_loss += loss.item()\n\n val_loss /= len(val_loader)\n\n print(\n f\"Epoch {epoch+1}/{num_epochs} \"\n f\"- Train Loss: {train_loss:.4f} \"\n f\"- Val Loss: {val_loss:.4f}\"\n )\n\n# Save the baseline permanently\nbaseline_checkpoint = \"/kaggle/working/baseline_model.pth\"\n\ntorch.save({\n \"model_state_dict\": baseline_model.state_dict(),\n \"optimizer_state_dict\": optimizer.state_dict(),\n \"epoch\": num_epochs,\n \"train_loss\": train_loss,\n \"val_loss\": val_loss,\n}, baseline_checkpoint)\n\nbaseline_model.eval()\n\nprint(\"\\nBaseline model saved successfully.\")\nprint(\"Checkpoint:\", baseline_checkpoint)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:47:31.628166Z","iopub.execute_input":"2026-09-12T12:47:31.628953Z","iopub.status.idle":"2026-09-12T12:54:46.108217Z","shell.execute_reply.started":"2026-09-12T12:47:31.628878Z","shell.execute_reply":"2026-09-12T12:54:46.107275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load the saved baseline model\n\nbaseline_checkpoint = \"/kaggle/working/baseline_model.pth\"\n\ncheckpoint = torch.load(\n baseline_checkpoint,\n map_location=device,\n weights_only=True\n)\n\nbaseline_model = RSNAKnee3DCNN(num_classes=12).to(device)\nbaseline_model.load_state_dict(checkpoint[\"model_state_dict\"])\nbaseline_model.eval()\n\nprint(\"Baseline checkpoint loaded successfully.\")\nprint(\"Output classes:\", checkpoint[\"model_state_dict\"][\"classifier.4.bias\"].shape[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:55:37.777451Z","iopub.execute_input":"2026-09-12T12:55:37.778785Z","iopub.status.idle":"2026-09-12T12:55:37.803408Z","shell.execute_reply.started":"2026-09-12T12:55:37.77874Z","shell.execute_reply":"2026-09-12T12:55:37.802427Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 3D GRAD-CAM FOR THE BASELINE MODEL\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n# Target layer: final Conv3D layer before global average pooling\ntarget_layer = baseline_model.features[8]\n\nactivations = None\ngradients = None\n\n\ndef forward_hook(module, input, output):\n global activations\n activations = output\n\n\ndef backward_hook(module, grad_input, grad_output):\n global gradients\n gradients = grad_output[0]\n\n\n# Register hooks\nforward_handle = target_layer.register_forward_hook(forward_hook)\nbackward_handle = target_layer.register_full_backward_hook(backward_hook)\n\n\ndef generate_gradcam(model, image, class_idx):\n \"\"\"\n Generate a 3D Grad-CAM heatmap for one abnormality.\n \n image shape: [1, 3, D, H, W]\n \"\"\"\n\n model.eval()\n model.zero_grad()\n\n output = model(image)\n\n # Target the requested abnormality\n target = output[0, class_idx]\n\n # Backpropagate\n target.backward()\n\n # Activations: [1, C, D, H, W]\n # Gradients: [1, C, D, H, W]\n weights = gradients.mean(dim=(2, 3, 4), keepdim=True)\n\n # Weighted combination of feature maps\n cam = (weights * activations).sum(dim=1, keepdim=True)\n\n # Keep positive influence\n cam = F.relu(cam)\n\n # Normalize\n cam = cam - cam.min()\n cam = cam / (cam.max() + 1e-8)\n\n return cam.detach().cpu().numpy()[0, 0], output.detach().cpu()\n\n\nprint(\"Grad-CAM hooks registered successfully.\")\nprint(\"Target layer:\", target_layer)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:56:33.201886Z","iopub.execute_input":"2026-09-12T12:56:33.203222Z","iopub.status.idle":"2026-09-12T12:56:33.215739Z","shell.execute_reply.started":"2026-09-12T12:56:33.203177Z","shell.execute_reply":"2026-09-12T12:56:33.213899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# GRAD-CAM — LOAD ONE STUDY FROM CLEAN VALIDATION CACHE\nimport os\nimport torch\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n# Inspect the clean validation cache\nprint(\"Clean validation cache exists:\", os.path.exists(\"/kaggle/working/clean_eval_cache\"))\n\ncache_files = sorted([\n f for f in os.listdir(\"/kaggle/working/clean_eval_cache\")\n if f.endswith(\".pt\")\n])\n\nprint(\"Cached validation studies:\", len(cache_files))\n\n# Pick the first cached study\ncache_file = cache_files[0]\ncache_path = os.path.join(\"/kaggle/working/clean_eval_cache\", cache_file)\n\ndata = torch.load(cache_path, map_location=\"cpu\", weights_only=True)\n\nprint(\"\\nCache file:\", cache_file)\nprint(\"Stored object type:\", type(data))\n\nif isinstance(data, dict):\n print(\"Keys:\", data.keys())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:58:20.751679Z","iopub.execute_input":"2026-09-12T12:58:20.752182Z","iopub.status.idle":"2026-09-12T12:58:20.804247Z","shell.execute_reply.started":"2026-09-12T12:58:20.752144Z","shell.execute_reply":"2026-09-12T12:58:20.803057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_cols = [\n 'ACL',\n 'MCL',\n 'Medial Meniscus',\n 'Lateral Meniscus',\n 'Medial OA',\n 'Lateral OA',\n 'PF OA',\n 'Effusion',\n 'Synovitis',\n \"Baker's\",\n 'Contusion',\n 'Fracture'\n]\n\n# Find official labels\nlabel_row = train[\n train[\"StudyInstanceUID\"].astype(str) == str(sample_study)\n].iloc[0]\n\nprint(\"Official labels:\")\nfor col in target_cols:\n print(f\"{col:20s}: {int(label_row[col])}\")\n\n# Baseline predictions\nbaseline_model.eval()\n\nwith torch.no_grad():\n logits = baseline_model(image)\n probabilities = torch.sigmoid(logits)[0].cpu().numpy()\n\nprint(\"\\nModel predictions:\")\nfor name, prob in zip(target_cols, probabilities):\n print(f\"{name:20s}: {prob:.3f}\")\n\n# Select strongest prediction\ntarget_class = int(np.argmax(probabilities))\n\nprint(\"\\nSelected Grad-CAM target:\")\nprint(\"Abnormality:\", target_cols[target_class])\nprint(\"Probability:\", round(probabilities[target_class], 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T12:59:57.873251Z","iopub.execute_input":"2026-09-12T12:59:57.87424Z","iopub.status.idle":"2026-09-12T12:59:58.53746Z","shell.execute_reply.started":"2026-09-12T12:59:57.874196Z","shell.execute_reply":"2026-09-12T12:59:58.535882Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"image.requires_grad_(True)\n\n# Generate Grad-CAM for the selected class\ncam, output = generate_gradcam(\n baseline_model,\n image,\n target_class\n)\n\nprint(\"Grad-CAM generated successfully.\")\nprint(\"CAM shape:\", cam.shape)\nprint(\"Target:\", target_cols[target_class])\nprint(\"Prediction:\", round(probabilities[target_class], 3))\nprint(\"Official label:\", int(label_row[target_cols[target_class]]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:00:56.948516Z","iopub.execute_input":"2026-09-12T13:00:56.948884Z","iopub.status.idle":"2026-09-12T13:00:58.202088Z","shell.execute_reply.started":"2026-09-12T13:00:56.948856Z","shell.execute_reply":"2026-09-12T13:00:58.201154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# VISUALIZE 3D GRAD-CAM\n# Move image back to CPU\nimage_np = image.detach().cpu().numpy()[0]\n\n# image_np shape = [3, 32, 128, 128]\n# cam shape = [8, 32, 32]\n\n# Resize CAM back to the input volume size\ncam_tensor = torch.tensor(cam).unsqueeze(0).unsqueeze(0)\n\ncam_resized = F.interpolate(\n cam_tensor,\n size=(32, 128, 128),\n mode=\"trilinear\",\n align_corners=False\n)\n\ncam_resized = cam_resized[0, 0].numpy()\n\n# Central depth slice\ndepth_idx = image_np.shape[1] // 2\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\nplane_names = [\"Axial\", \"Coronal\", \"Sagittal\"]\n\nfor i, (ax, plane) in enumerate(zip(axes, plane_names)):\n\n mri_slice = image_np[i, depth_idx]\n cam_slice = cam_resized[depth_idx]\n\n ax.imshow(mri_slice, cmap=\"gray\")\n ax.imshow(\n cam_slice,\n cmap=\"jet\",\n alpha=0.45,\n vmin=0,\n vmax=1\n )\n\n ax.set_title(f\"{plane}\\nGrad-CAM: {target_cols[target_class]}\")\n ax.axis(\"off\")\n\nplt.suptitle(\n f\"3D Grad-CAM — {target_cols[target_class]} \"\n f\"(Prediction: {probabilities[target_class]:.3f}, \"\n f\"Actual: {int(label_row[target_cols[target_class]])})\",\n fontsize=14\n)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:01:47.930354Z","iopub.execute_input":"2026-09-12T13:01:47.931655Z","iopub.status.idle":"2026-09-12T13:01:48.606276Z","shell.execute_reply.started":"2026-09-12T13:01:47.931594Z","shell.execute_reply":"2026-09-12T13:01:48.605109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(\"Weak cache exists:\", os.path.exists(\"/kaggle/working/weak_mri_cache\"))\nprint(\"Weak dataset exists:\", \"weak_dataset\" in globals())\n\nif \"weak_dataset\" in globals():\n print(\"Weak dataset shape:\", weak_dataset.shape)\n print(\"Columns:\", weak_dataset.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:03:41.554823Z","iopub.execute_input":"2026-09-12T13:03:41.556103Z","iopub.status.idle":"2026-09-12T13:03:41.565387Z","shell.execute_reply.started":"2026-09-12T13:03:41.556043Z","shell.execute_reply":"2026-09-12T13:03:41.563997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check overlap between official 58 labeled studies and weak-label studies\nofficial_uids = set(train[\"StudyInstanceUID\"].astype(str))\nweak_uids = set(weak_dataset[\"StudyInstanceUID\"].astype(str))\n\noverlap = official_uids & weak_uids\n\nprint(\"Official studies:\", len(official_uids))\nprint(\"Weak studies:\", len(weak_uids))\nprint(\"Overlap:\", len(overlap))\n\n# Check weak-label availability\nweak_label_cols = [\n \"ACL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Bakers_weak\"\n]\n\nprint(\"\\nWeak-label coverage:\")\nfor col in weak_label_cols:\n print(f\"{col:25s}: {weak_dataset[col].notna().sum()}\")\n\nprint(\"\\nAll 3 planes available:\")\nprint(\n weak_dataset[\n [\"Axial_SeriesUID\", \"Coronal_SeriesUID\", \"Sagittal_SeriesUID\"]\n ].notna().all(axis=1).sum(),\n \"/\",\n len(weak_dataset)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:04:23.635208Z","iopub.execute_input":"2026-09-12T13:04:23.636417Z","iopub.status.idle":"2026-09-12T13:04:23.669657Z","shell.execute_reply.started":"2026-09-12T13:04:23.63637Z","shell.execute_reply":"2026-09-12T13:04:23.668041Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Separate the 58 officially labeled studies from weak supervision\n\nofficial_labeled_uids = set(\n train.loc[train[target_cols].notna().all(axis=1), \"StudyInstanceUID\"]\n .astype(str)\n)\n\nweak_train_df = weak_dataset[\n ~weak_dataset[\"StudyInstanceUID\"].astype(str).isin(official_labeled_uids)\n].copy()\n\nprint(\"Officially labeled studies:\", len(official_labeled_uids))\nprint(\"Weak studies before removing official:\", len(weak_dataset))\nprint(\"Weak studies after removing official:\", len(weak_train_df))\n\nprint(\"\\nWeak label availability:\")\nfor col in weak_label_cols:\n print(f\"{col:25s}: {weak_train_df[col].notna().sum()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:05:30.715131Z","iopub.execute_input":"2026-09-12T13:05:30.715536Z","iopub.status.idle":"2026-09-12T13:05:30.739267Z","shell.execute_reply.started":"2026-09-12T13:05:30.715502Z","shell.execute_reply":"2026-09-12T13:05:30.737823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\ntotal, used, free = shutil.disk_usage(\"/kaggle/working\")\n\nprint(f\"Total storage : {total / (1024**3):.2f} GB\")\nprint(f\"Used storage : {used / (1024**3):.2f} GB\")\nprint(f\"Free storage : {free / (1024**3):.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:06:07.543727Z","iopub.execute_input":"2026-09-12T13:06:07.544161Z","iopub.status.idle":"2026-09-12T13:06:07.556978Z","shell.execute_reply.started":"2026-09-12T13:06:07.544126Z","shell.execute_reply":"2026-09-12T13:06:07.553971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport time\nimport torch\n\nweak_cache_dir = \"/kaggle/working/weak_mri_cache\"\nos.makedirs(weak_cache_dir, exist_ok=True)\n\n# Only process studies that don't already have a cached file\ncached = {\n os.path.splitext(f)[0]\n for f in os.listdir(weak_cache_dir)\n if f.endswith(\".pt\")\n}\n\nremaining = weak_train_df[\n ~weak_train_df[\"StudyInstanceUID\"].astype(str).isin(cached)\n].copy()\n\nprint(\"Weak studies:\", len(weak_train_df))\nprint(\"Already cached:\", len(cached))\nprint(\"Remaining:\", len(remaining))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:06:34.822427Z","iopub.execute_input":"2026-09-12T13:06:34.822765Z","iopub.status.idle":"2026-09-12T13:06:34.835871Z","shell.execute_reply.started":"2026-09-12T13:06:34.822739Z","shell.execute_reply":"2026-09-12T13:06:34.834481Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport time\nimport torch\n\nweak_cache_dir = \"/kaggle/working/weak_mri_cache\"\nos.makedirs(weak_cache_dir, exist_ok=True)\n\nstart_time = time.time()\nsuccess = 0\nfailed = 0\n\nfor i, (_, row) in enumerate(weak_train_df.iterrows(), 1):\n\n study_uid = str(row[\"StudyInstanceUID\"])\n cache_path = os.path.join(weak_cache_dir, f\"{study_uid}.pt\")\n\n # Skip if already cached\n if os.path.exists(cache_path):\n success += 1\n continue\n\n try:\n # Load the three selected MRI series\n axial = load_series_volume(\n study_uid,\n str(row[\"Axial_SeriesUID\"])\n )\n\n coronal = load_series_volume(\n study_uid,\n str(row[\"Coronal_SeriesUID\"])\n )\n\n sagittal = load_series_volume(\n study_uid,\n str(row[\"Sagittal_SeriesUID\"])\n )\n\n # Convert each volume to [1, 32, 128, 128]\n axial = prepare_volume(axial)\n coronal = prepare_volume(coronal)\n sagittal = prepare_volume(sagittal)\n\n # Combine planes → [3, 32, 128, 128]\n image = torch.cat([axial, coronal, sagittal], dim=0)\n\n torch.save(image, cache_path)\n success += 1\n\n except Exception as e:\n failed += 1\n print(f\"\\nFAILED {study_uid}\")\n print(\"Error:\", str(e))\n\n if i % 100 == 0:\n elapsed = time.time() - start_time\n print(\n f\"{i}/{len(weak_train_df)} | \"\n f\"success={success} | failed={failed} | \"\n f\"elapsed={elapsed/60:.1f} min\"\n )\n\nprint(\"\\nCACHE COMPLETE\")\nprint(\"Successful:\", success)\nprint(\"Failed:\", failed)\nprint(\n \"Time:\",\n round((time.time() - start_time) / 60, 2),\n \"minutes\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T13:07:09.006314Z","iopub.execute_input":"2026-09-12T13:07:09.006643Z","iopub.status.idle":"2026-09-12T14:19:00.21265Z","shell.execute_reply.started":"2026-09-12T13:07:09.006613Z","shell.execute_reply":"2026-09-12T14:19:00.211413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport shutil\n\ncache_files = [\n f for f in os.listdir(\"/kaggle/working/weak_mri_cache\")\n if f.endswith(\".pt\")\n]\n\ntotal, used, free = shutil.disk_usage(\"/kaggle/working\")\n\nprint(\"Weak MRI cache files:\", len(cache_files))\nprint(f\"Cache size: {sum(os.path.getsize(os.path.join('/kaggle/working/weak_mri_cache', f)) for f in cache_files) / (1024**3):.2f} GB\")\nprint(f\"Free storage: {free / (1024**3):.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:19:43.465731Z","iopub.execute_input":"2026-09-12T14:19:43.466788Z","iopub.status.idle":"2026-09-12T14:19:43.516993Z","shell.execute_reply.started":"2026-09-12T14:19:43.466742Z","shell.execute_reply":"2026-09-12T14:19:43.515749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfor path in [\n \"/kaggle/working/weak_mri_cache\",\n \"/kaggle/working/clean_eval_cache\",\n \"/kaggle/working/baseline_model.pth\",\n \"/kaggle/working/weak_supervised_model_epoch1.pth\",\n \"/kaggle/working/weak_supervised_model_epoch2.pth\"\n]:\n if os.path.exists(path):\n if os.path.isfile(path):\n size = os.path.getsize(path) / (1024**2)\n print(f\"{path}: {size:.2f} MB\")\n else:\n size = sum(\n os.path.getsize(os.path.join(root, f))\n for root, _, files in os.walk(path)\n for f in files\n ) / (1024**3)\n print(f\"{path}: {size:.2f} GB\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:20:32.315813Z","iopub.execute_input":"2026-09-12T14:20:32.316762Z","iopub.status.idle":"2026-09-12T14:20:32.353727Z","shell.execute_reply.started":"2026-09-12T14:20:32.316723Z","shell.execute_reply":"2026-09-12T14:20:32.352564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"train_df exists:\", \"train_df\" in globals())\nprint(\"train_dataset exists:\", \"train_dataset\" in globals())\nprint(\"val_df exists:\", \"val_df\" in globals())\n\nif \"train_df\" in globals():\n print(\"Clean training studies:\", len(train_df))\n\nif \"val_df\" in globals():\n print(\"Clean validation studies:\", len(val_df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:21:15.51319Z","iopub.execute_input":"2026-09-12T14:21:15.514012Z","iopub.status.idle":"2026-09-12T14:21:15.521594Z","shell.execute_reply.started":"2026-09-12T14:21:15.51396Z","shell.execute_reply":"2026-09-12T14:21:15.520483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import Dataset\nimport torch\nimport os\nimport numpy as np\n\nweak_label_cols = [\n \"ACL_weak\",\n \"Medial_Meniscus_weak\",\n \"Lateral_Meniscus_weak\",\n \"PF_OA_weak\",\n \"Effusion_weak\",\n \"Bakers_weak\"\n]\n\n# Mapping weak labels → positions in the 12-class output\nweak_to_target = {\n \"ACL_weak\": 0,\n \"Medial_Meniscus_weak\": 2,\n \"Lateral_Meniscus_weak\": 3,\n \"PF_OA_weak\": 6,\n \"Effusion_weak\": 7,\n \"Bakers_weak\": 9\n}\n\nclass CombinedKneeDataset(Dataset):\n\n def __init__(self, clean_dataset, weak_df, weak_cache_dir):\n self.clean_dataset = clean_dataset\n self.weak_df = weak_df.reset_index(drop=True)\n self.weak_cache_dir = weak_cache_dir\n\n self.n_clean = len(clean_dataset)\n self.n_weak = len(self.weak_df)\n\n def __len__(self):\n return self.n_clean + self.n_weak\n\n def __getitem__(self, idx):\n\n # Clean official sample\n if idx < self.n_clean:\n\n image, labels = self.clean_dataset[idx]\n\n labels = torch.as_tensor(\n labels,\n dtype=torch.float32\n )\n\n # All 12 labels are valid\n mask = torch.ones(12, dtype=torch.bool)\n\n # Strong supervision\n weight = torch.tensor(1.0)\n\n return image, labels, mask, weight\n\n # Weakly supervised sample\n widx = idx - self.n_clean\n row = self.weak_df.iloc[widx]\n\n study_uid = str(row[\"StudyInstanceUID\"])\n\n cache_path = os.path.join(\n self.weak_cache_dir,\n f\"{study_uid}.pt\"\n )\n\n image = torch.load(\n cache_path,\n map_location=\"cpu\",\n weights_only=True\n )\n\n # Start with unknown labels\n labels = torch.zeros(12, dtype=torch.float32)\n mask = torch.zeros(12, dtype=torch.bool)\n\n # Insert only available weak labels\n for weak_col, target_idx in weak_to_target.items():\n\n value = row[weak_col]\n\n if not pd.isna(value):\n labels[target_idx] = float(value)\n mask[target_idx] = True\n\n # Weak supervision gets lower loss weight\n weight = torch.tensor(0.35)\n\n return image, labels, mask, weight","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:22:21.857325Z","iopub.execute_input":"2026-09-12T14:22:21.857852Z","iopub.status.idle":"2026-09-12T14:22:21.880442Z","shell.execute_reply.started":"2026-09-12T14:22:21.857816Z","shell.execute_reply":"2026-09-12T14:22:21.87886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:22:40.536572Z","iopub.execute_input":"2026-09-12T14:22:40.537675Z","iopub.status.idle":"2026-09-12T14:22:40.543507Z","shell.execute_reply.started":"2026-09-12T14:22:40.537636Z","shell.execute_reply":"2026-09-12T14:22:40.542142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"combined_dataset = CombinedKneeDataset(\n clean_dataset=train_dataset,\n weak_df=weak_train_df,\n weak_cache_dir=\"/kaggle/working/weak_mri_cache\"\n)\n\nprint(\"Combined dataset size:\", len(combined_dataset))\nprint(\"Clean samples:\", len(train_dataset))\nprint(\"Weak samples:\", len(weak_train_df))\n\n# Test one clean sample\nx, y, mask, weight = combined_dataset[0]\n\nprint(\"\\nClean sample:\")\nprint(\"Image:\", x.shape)\nprint(\"Labels:\", y.shape)\nprint(\"Valid labels:\", mask.sum().item())\nprint(\"Weight:\", weight.item())\n\n# Test one weak sample\nx, y, mask, weight = combined_dataset[len(train_dataset)]\n\nprint(\"\\nWeak sample:\")\nprint(\"Image:\", x.shape)\nprint(\"Labels:\", y.shape)\nprint(\"Valid labels:\", mask.sum().item())\nprint(\"Weight:\", weight.item())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-12T14:22:50.45335Z","iopub.execute_input":"2026-09-12T14:22:50.45434Z","iopub.status.idle":"2026-09-12T14:22:51.461491Z","shell.execute_reply.started":"2026-09-12T14:22:50.454297Z","shell.execute_reply":"2026-09-12T14:22:51.459984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}