{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 本方案采用EffNetV2五折交叉验证模型融合得到的结果\nOriginal Notebooks : \n\n- [[infer] PyTorch-EffNetV2 single-model LB:0.49](https://www.kaggle.com/code/vslaykovsky/infer-pytorch-effnetv2-single-model-lb-0-49/data?scriptVersionId=104434908)","metadata":{}},{"cell_type":"markdown","source":"<div class=\"alert alert-block alert-info\" style=\"text-align:center; font-size:28px;\">\n     PyTorch-EffNetV2 融合模型 LB:0.4761 | PLB:0.5215\n</div>","metadata":{}},{"cell_type":"markdown","source":"### 1. 导入依赖项，设置常量","metadata":{}},{"cell_type":"code","source":"try:\n    import pylibjpeg\nexcept:\n    # Offline dependencies:\n    !mkdir -p /root/.cache/torch/hub/checkpoints/\n    !cp ../input/rsna-2022-whl/efficientnet_v2_s-dd5fe13b.pth  /root/.cache/torch/hub/checkpoints/\n\n    !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n    !pip install /kaggle/input/rsna-2022-whl/{torch-1.12.1-cp37-cp37m-manylinux1_x86_64.whl,torchvision-0.13.1-cp37-cp37m-manylinux1_x86_64.whl}","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:11:52.92593Z","iopub.execute_input":"2023-05-24T08:11:52.92641Z","iopub.status.idle":"2023-05-24T08:13:50.088712Z","shell.execute_reply.started":"2023-05-24T08:11:52.926314Z","shell.execute_reply":"2023-05-24T08:13:50.087394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import GroupKFold\nimport numpy as np\nimport pydicom as dicom\nimport matplotlib.pyplot as plt\nimport cv2\nimport glob\nimport gc\nimport re\nimport os\nimport torch\nimport torchvision as tv\nimport gdcm\nimport torch.nn as nn\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:13:50.090872Z","iopub.execute_input":"2023-05-24T08:13:50.092335Z","iopub.status.idle":"2023-05-24T08:13:51.864769Z","shell.execute_reply.started":"2023-05-24T08:13:50.09229Z","shell.execute_reply":"2023-05-24T08:13:51.863768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport glob\nimport os\nimport re\n\nimport cv2\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport pydicom as dicom\nimport torch\nimport torchvision as tv\nfrom sklearn.model_selection import GroupKFold\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torchvision.models.feature_extraction import create_feature_extractor\nfrom tqdm.notebook import tqdm\n\nimport wandb\n\npd.set_option('display.max_rows', 1000)\npd.set_option('display.max_columns', 1000)\nplt.rcParams['figure.figsize'] = (20, 5)\n\n\n# Effnet\nWEIGHTS = tv.models.efficientnet.EfficientNet_V2_S_Weights.DEFAULT\nRSNA_2022_PATH = '../input/rsna-2022-cervical-spine-fracture-detection'\nTRAIN_IMAGES_PATH = f'{RSNA_2022_PATH}/train_images'\nTEST_IMAGES_PATH = f'{RSNA_2022_PATH}/test_images'\nEFFNET_CHECKPOINTS_PATH = '../input/rsnaefv2ep2'\n\n# MODEL_NAMES = [f'effnetv2']\n\n# This notebook supports ensembles and single model predictions. Uncomment to switch to ensemble prediction:\nMODEL_NAMES = [f'yolov5_train_{i}ep2' for i in range(5)]\n\n# Common\nFRAC_COLS = [f'C{i}_effnet_frac' for i in range(1, 8)]\nVERT_COLS = [f'C{i}_effnet_vert' for i in range(1, 8)]\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    IS_KAGGLE = True\nexcept:\n    IS_KAGGLE = False\n\n\n# Switch to offline for submission\nos.environ[\"WANDB_MODE\"] = \"offline\"\n\nif os.environ[\"WANDB_MODE\"] == \"online\":\n    if IS_KAGGLE:\n        os.environ['WANDB_API_KEY'] = UserSecretsClient().get_secret(\"WANDB_API_KEY\")\n\nif not IS_KAGGLE:\n    print('Running locally')\n    RSNA_2022_PATH = '/mnt/rsna2022'\n    TRAIN_IMAGES_PATH = '/mnt/rsna2022/train_images'\n    TEST_IMAGES_PATH = '/mnt/rsna2022/test_images'\n    METADATA_PATH = '/home/vslaykovsky/Downloads/'\n    EFFNET_CHECKPOINTS_PATH = 'frac_checkpoints'\n    os.environ['WANDB_API_KEY'] = 'yourkeyhere'\n\n\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nif DEVICE == 'cuda':\n    BATCH_SIZE = 32\nelse:\n    BATCH_SIZE = 2","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-05-24T08:13:51.866586Z","iopub.execute_input":"2023-05-24T08:13:51.8673Z","iopub.status.idle":"2023-05-24T08:13:52.692922Z","shell.execute_reply.started":"2023-05-24T08:13:51.867259Z","shell.execute_reply":"2023-05-24T08:13:52.691888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2. 导入 train/eval/test dataframes","metadata":{}},{"cell_type":"code","source":"def load_df_test():\n    df_test = pd.read_csv(f'{RSNA_2022_PATH}/test.csv')\n\n    if df_test.iloc[0].row_id == '1.2.826.0.1.3680043.10197_C1':\n        # test_images and test.csv are inconsistent in the dev dataset, fixing labels for the dev run.\n        df_test = pd.DataFrame({\n            \"row_id\": ['1.2.826.0.1.3680043.22327_C1', '1.2.826.0.1.3680043.25399_C1', '1.2.826.0.1.3680043.5876_C1'],\n            \"StudyInstanceUID\": ['1.2.826.0.1.3680043.22327', '1.2.826.0.1.3680043.25399', '1.2.826.0.1.3680043.5876'],\n            \"prediction_type\": [\"C1\", \"C1\", \"patient_overall\"]}\n        )\n    return df_test\n\ndf_test = load_df_test()\ndf_test","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:13:52.696381Z","iopub.execute_input":"2023-05-24T08:13:52.69748Z","iopub.status.idle":"2023-05-24T08:13:52.759071Z","shell.execute_reply.started":"2023-05-24T08:13:52.697434Z","shell.execute_reply":"2023-05-24T08:13:52.757999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_slices = glob.glob(f'{TEST_IMAGES_PATH}/*/*')\ntest_slices = [re.findall(f'{TEST_IMAGES_PATH}/(.*)/(.*).dcm', s)[0] for s in test_slices]\ndf_test_slices = pd.DataFrame(data=test_slices, columns=['StudyInstanceUID', 'Slice']).astype({'Slice': int}).sort_values(['StudyInstanceUID', 'Slice']).reset_index(drop=True)\ndf_test_slices","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:13:52.763694Z","iopub.execute_input":"2023-05-24T08:13:52.765464Z","iopub.status.idle":"2023-05-24T08:13:53.585348Z","shell.execute_reply.started":"2023-05-24T08:13:52.765419Z","shell.execute_reply":"2023-05-24T08:13:53.584275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    \"\"\"\n    This supports loading both regular and compressed JPEG images.\n    See the first sell with `pip install` commands for the necessary dependencies\n    \"\"\"\n    img=dicom.dcmread(path)\n    img.PhotometricInterpretation = 'YBR_FULL'\n    data = img.pixel_array\n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data=(data * 255).astype(np.uint8)\n    return cv2.cvtColor(data, cv2.COLOR_GRAY2RGB), img\n\nclass EffnetDataSet(torch.utils.data.Dataset):\n    def __init__(self, df, path, transforms=None):\n        super().__init__()\n        self.df = df\n        self.path = path\n        self.transforms = transforms\n\n    def __getitem__(self, i):\n        path = os.path.join(self.path, self.df.iloc[i].StudyInstanceUID, f'{self.df.iloc[i].Slice}.dcm')\n\n        try:\n            img = load_dicom(path)[0]\n            # Pytorch uses (batch, channel, height, width) order. Converting (height, width, channel) -> (channel, height, width)\n            # img = np.transpose(img, (2, 0, 1))\n            img = cv2.resize(img, (512, 512)).transpose(2, 0, 1)\n            if self.transforms is not None:\n                img = self.transforms(torch.as_tensor(img))\n        except Exception as ex:\n            print(ex)\n            return None\n\n        if 'C1_fracture' in self.df:\n            # frac_targets = torch.as_tensor(self.df.iloc[i][['C1_fracture', 'C2_fracture', 'C3_fracture', 'C4_fracture',\n            #                                                 'C5_fracture', 'C6_fracture', 'C7_fracture']].astype(\n            #     'float32').values)\n            vert_targets = torch.as_tensor(\n                self.df.iloc[i][['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']].astype('float32').values)\n            if vert_targets[0]==vert_targets[1]==vert_targets[2]==vert_targets[3]==vert_targets[4]==vert_targets[5]==vert_targets[6]==0:\n                vert_targets=torch.as_tensor([0, 1])\n            else:\n                vert_targets=torch.as_tensor([1, 0])\n#             frac_targets = frac_targets * vert_targets  # we only enable targets that are visible on the current slice\n            return img, vert_targets\n        return img\n\n    def __len__(self):\n        return len(self.df)\n\ndef save_model(name, model):\n    torch.save(model.state_dict(), f'{name}.tph')\ndef load_model(model, path='.'):\n    data = torch.load(path, map_location='cuda')\n    model.load_state_dict(data)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:13:53.587074Z","iopub.execute_input":"2023-05-24T08:13:53.587464Z","iopub.status.idle":"2023-05-24T08:13:53.611407Z","shell.execute_reply.started":"2023-05-24T08:13:53.587425Z","shell.execute_reply":"2023-05-24T08:13:53.610453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = nn.Softmax(dim=1)\nmodel_name = 'efficientnet_v2_s'\nmodel = torch.hub.load(r'../input/ef-v2-modle/EfficientNetV2-pytorch-main', model_name, nclass=10, skip_validation=True,source='local')\ndef evaluate_segeffnet(model, ds):\n    torch.manual_seed(42)\n    model = model.to('cuda')\n    with torch.no_grad():\n        model.eval()\n        pred = []\n        for X in tqdm(ds, leave=False):\n            with autocast():\n                y_vert_pred = model(X.cuda().float())\n                y_vert_pred = s(y_vert_pred[:,:2])\n            pred.append(y_vert_pred.cpu().numpy())\n            # acc = np.mean(np.mean((pred[-1] > 0.5) == y[-1], axis=0))\n        pred = np.concatenate(pred)\n        return pred","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:13:53.613161Z","iopub.execute_input":"2023-05-24T08:13:53.61353Z","iopub.status.idle":"2023-05-24T08:13:54.033527Z","shell.execute_reply.started":"2023-05-24T08:13:53.613494Z","shell.execute_reply":"2023-05-24T08:13:54.032501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_IMAGES_PATH = r'../input/rsna-2022-cervical-spine-fracture-detection/test_images'\nbatch_size = 128\ncriterion = nn.CrossEntropyLoss()\ntrain_loader = df_test_slices\nmodel_num = load_model(model, f'../input/ef-v2-modle/ef_v2_s.tph')\ntrain_loader = DataLoader(EffnetDataSet(train_loader, TEST_IMAGES_PATH), batch_size=batch_size, shuffle=False, num_workers=0, drop_last=False)\npred = evaluate_segeffnet(model_num, train_loader)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:13:54.035145Z","iopub.execute_input":"2023-05-24T08:13:54.035519Z","iopub.status.idle":"2023-05-24T08:14:41.396282Z","shell.execute_reply.started":"2023-05-24T08:13:54.035478Z","shell.execute_reply":"2023-05-24T08:14:41.395097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_slices['vert_targets'] = pred[:,:1]\ndf_test_slices['vert_targets'] = (df_test_slices['vert_targets'] > 0.5).astype(int)\ndf_test_slices = df_test_slices[df_test_slices['vert_targets']==1]\n# df_test_slices.heade()","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:14:41.398086Z","iopub.execute_input":"2023-05-24T08:14:41.398482Z","iopub.status.idle":"2023-05-24T08:14:41.409513Z","shell.execute_reply.started":"2023-05-24T08:14:41.398443Z","shell.execute_reply":"2023-05-24T08:14:41.408579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3. Dataset class","metadata":{}},{"cell_type":"code","source":"def load_dicom(path):\n    \"\"\"\n    This supports loading both regular and compressed JPEG images. \n    See the first sell with `pip install` commands for the necessary dependencies\n    \"\"\"\n    img=dicom.dcmread(path)\n    img.PhotometricInterpretation = 'YBR_FULL'\n    data = img.pixel_array    \n    data = data - np.min(data)\n    if np.max(data) != 0:\n        data = data / np.max(data)\n    data=(data * 255).astype(np.uint8)\n    return cv2.cvtColor(data, cv2.COLOR_GRAY2RGB), img\n\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10001/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('regular image')\n\nim, meta = load_dicom(f'{TRAIN_IMAGES_PATH}/1.2.826.0.1.3680043.10014/1.dcm')\nplt.figure()\nplt.imshow(im)\nplt.title('jpeg')","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:14:41.414746Z","iopub.execute_input":"2023-05-24T08:14:41.415478Z","iopub.status.idle":"2023-05-24T08:14:42.049785Z","shell.execute_reply.started":"2023-05-24T08:14:41.415403Z","shell.execute_reply":"2023-05-24T08:14:42.048835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EffnetDataSet(torch.utils.data.Dataset):    \n    def __init__(self, df, path, transforms=None):\n        super().__init__()\n        self.df = df\n        self.path = path\n        self.transforms = transforms\n        \n    def __getitem__(self, i):\n        path = os.path.join(self.path, self.df.iloc[i].StudyInstanceUID, f'{self.df.iloc[i].Slice}.dcm')        \n        \n        try:\n            img = load_dicom(path)[0]         \n            img = np.transpose(img, (2, 0, 1))  # Pytorch uses (batch, channel, height, width) order. Converting (height, width, channel) -> (channel, height, width)\n            if self.transforms is not None:\n                img = self.transforms(torch.as_tensor(img))\n        except Exception as ex:\n            print(ex)\n            return None\n        \n        if 'C1_fracture' in self.df:\n            frac_targets = torch.as_tensor(self.df.iloc[i][['C1_fracture', 'C2_fracture', 'C3_fracture', 'C4_fracture', 'C5_fracture', 'C6_fracture', 'C7_fracture']].astype('float32').values)\n            vert_targets = torch.as_tensor(self.df.iloc[i][['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']].astype('float32').values)\n            frac_targets = frac_targets * vert_targets   # we only enable targets that are visible on the current slice\n            return img, frac_targets, vert_targets\n        return img        \n    \n    def __len__(self):\n        return len(self.df)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:14:42.051424Z","iopub.execute_input":"2023-05-24T08:14:42.052078Z","iopub.status.idle":"2023-05-24T08:14:42.064899Z","shell.execute_reply.started":"2023-05-24T08:14:42.052036Z","shell.execute_reply":"2023-05-24T08:14:42.06399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only X values returned by the test dataset\nds_test = EffnetDataSet(df_test_slices, TEST_IMAGES_PATH, WEIGHTS.transforms())\nX = ds_test[42]\nX.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:14:42.066731Z","iopub.execute_input":"2023-05-24T08:14:42.067151Z","iopub.status.idle":"2023-05-24T08:14:42.100535Z","shell.execute_reply.started":"2023-05-24T08:14:42.067077Z","shell.execute_reply":"2023-05-24T08:14:42.099333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.导入model","metadata":{}},{"cell_type":"code","source":"class EffnetModel(torch.nn.Module):\n    def __init__(self):\n        super().__init__()\n        effnet = tv.models.efficientnet_v2_s()\n        self.model = create_feature_extractor(effnet, ['flatten'])\n        self.nn_fracture = torch.nn.Sequential(\n            torch.nn.Linear(1280, 7),\n        )\n        self.nn_vertebrae = torch.nn.Sequential(\n            torch.nn.Linear(1280, 7),\n        )\n\n    def forward(self, x):\n        # returns logits\n        x = self.model(x)['flatten']\n        return self.nn_fracture(x), self.nn_vertebrae(x)\n\n    def predict(self, x):\n        frac, vert = self.forward(x)\n        return torch.sigmoid(frac), torch.sigmoid(vert)\n\nmodel = EffnetModel()\nmodel.predict(torch.randn(1, 3, 512, 512))\ndel model","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:14:42.102311Z","iopub.execute_input":"2023-05-24T08:14:42.102907Z","iopub.status.idle":"2023-05-24T08:14:44.291257Z","shell.execute_reply.started":"2023-05-24T08:14:42.102864Z","shell.execute_reply":"2023-05-24T08:14:44.290215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_model(model, name, path='.'):\n    data = torch.load(os.path.join(path, f'{name}.tph'), map_location=DEVICE)\n    model.load_state_dict(data)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:14:44.292754Z","iopub.execute_input":"2023-05-24T08:14:44.293166Z","iopub.status.idle":"2023-05-24T08:14:44.299443Z","shell.execute_reply.started":"2023-05-24T08:14:44.293127Z","shell.execute_reply":"2023-05-24T08:14:44.298214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_models = [load_model(EffnetModel(), name, EFFNET_CHECKPOINTS_PATH).to(DEVICE) for name in MODEL_NAMES]\n","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:14:44.301176Z","iopub.execute_input":"2023-05-24T08:14:44.301899Z","iopub.status.idle":"2023-05-24T08:14:53.301816Z","shell.execute_reply.started":"2023-05-24T08:14:44.301861Z","shell.execute_reply":"2023-05-24T08:14:53.300771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 5. Submission","metadata":{}},{"cell_type":"code","source":"from typing import List\n\n\ndef predict_effnet(models: List[EffnetModel], ds, max_batches=1e9):\n    dl_test = torch.utils.data.DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=os.cpu_count())\n    for m in models:\n        m.eval()\n\n    with torch.no_grad():\n        predictions = []\n        for idx, X in enumerate(tqdm(dl_test, miniters=10)):\n            pred = torch.zeros(len(X), 14).to(DEVICE)\n            for m in models:\n                y1, y2 = m.predict(X.to(DEVICE))\n                pred += torch.concat([y1, y2], dim=1) / len(models)\n            predictions.append(pred)\n            if idx >= max_batches:\n                break\n        return torch.concat(predictions).cpu().numpy()\n\n# Quick test\npredict_effnet([EffnetModel().to(DEVICE)], ds_test, max_batches=2).shape","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:17.142212Z","iopub.execute_input":"2023-05-24T08:27:17.142738Z","iopub.status.idle":"2023-05-24T08:27:20.745133Z","shell.execute_reply.started":"2023-05-24T08:27:17.142697Z","shell.execute_reply":"2023-05-24T08:27:20.743851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"effnet_pred = predict_effnet(effnet_models, ds_test)\n\ndf_effnet_pred = pd.DataFrame(\n    data=effnet_pred, columns=[f'C{i}_effnet_frac' for i in range(1, 8)] + [f'C{i}_effnet_vert' for i in range(1, 8)]\n)\nprint(df_test_slices.shape)\ndf_effnet_pred.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:20.747794Z","iopub.execute_input":"2023-05-24T08:27:20.748502Z","iopub.status.idle":"2023-05-24T08:27:45.512564Z","shell.execute_reply.started":"2023-05-24T08:27:20.748458Z","shell.execute_reply":"2023-05-24T08:27:45.511404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_pred = df_effnet_pred\ndf_test_pred['StudyInstanceUID'] = list(df_test_slices['StudyInstanceUID'])\ndf_test_pred['Slice'] = list(df_test_slices['Slice'])\ndf_test_pred.head()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:27:45.514288Z","iopub.execute_input":"2023-05-24T08:27:45.515018Z","iopub.status.idle":"2023-05-24T08:27:45.545173Z","shell.execute_reply.started":"2023-05-24T08:27:45.514971Z","shell.execute_reply":"2023-05-24T08:27:45.544278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_sample_patient(df_pred):\n    patient = np.random.choice(df_pred.StudyInstanceUID)\n    df = df_pred.query('StudyInstanceUID == @patient').reset_index()\n\n    df[[f'C{i}_effnet_frac' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, fracture prediction',\n        ax=(plt.subplot(1, 2, 1)))\n\n    df[[f'C{i}_effnet_vert' for i in range(1, 8)]].plot(\n        title=f'Patient {patient}, vertebrae prediction',\n        ax=plt.subplot(1, 2, 2)\n    )\n\nplot_sample_patient(df_test_pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:45.548092Z","iopub.execute_input":"2023-05-24T08:27:45.548839Z","iopub.status.idle":"2023-05-24T08:27:46.158631Z","shell.execute_reply.started":"2023-05-24T08:27:45.548798Z","shell.execute_reply":"2023-05-24T08:27:46.157635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_c1c7(FRAC_COLS, VERT_COLS):\n\n    frac_c1c7 = [FRAC_COLS[:, i].tolist() for i in range(7)]\n    vert_c1c7 = [VERT_COLS[:, i].tolist() for i in range(7)]\n    c1c7 = []\n    for i in range(7):\n        frac = frac_c1c7[i]\n        vert = vert_c1c7[i]\n        frac_mean = np.mean(frac)\n\n        vert = [i if i > 0.5 else 0 for i in vert]\n        frac = [i if i > frac_mean else 0 for i in frac]\n        # print('vert', vert, 'frac', frac)\n        tmp = [frac[i] * vert[i] for i in range(len(frac))]\n        # print('tmp', tmp)\n        try:\n            mean = sum(tmp)\n            mean = mean / (len(tmp) - tmp.count(0))\n        except:\n            mean = 0\n        c1c7.append(mean)\n    return np.array(c1c7)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:46.160271Z","iopub.execute_input":"2023-05-24T08:27:46.160875Z","iopub.status.idle":"2023-05-24T08:27:46.17169Z","shell.execute_reply.started":"2023-05-24T08:27:46.160836Z","shell.execute_reply":"2023-05-24T08:27:46.170683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def patient_prediction(df):\n    c1c7 = to_c1c7(df[FRAC_COLS].values, df[VERT_COLS].values)\n    pred_patient_overall = 1 - np.prod(1 - c1c7)\n    return pd.Series(data=np.concatenate([[pred_patient_overall], c1c7]), index=['patient_overall'] + [f'C{i}' for i in range(1, 8)])\n\ndf_patient_pred = df_test_pred.groupby('StudyInstanceUID').apply(lambda df: patient_prediction(df))\ndf_patient_pred","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-24T08:27:46.173326Z","iopub.execute_input":"2023-05-24T08:27:46.173693Z","iopub.status.idle":"2023-05-24T08:27:46.210932Z","shell.execute_reply.started":"2023-05-24T08:27:46.173656Z","shell.execute_reply":"2023-05-24T08:27:46.209894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = df_test.copy()\ndf_sub = df_sub.set_index('StudyInstanceUID').join(df_patient_pred)\ndf_sub['fractured'] = df_sub.apply(lambda r: r[r.prediction_type], axis=1)\n#df_sub[['row_id', 'fractured']].to_csv('submission1.csv', index=False)\ndf_sub","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:46.21239Z","iopub.execute_input":"2023-05-24T08:27:46.212733Z","iopub.status.idle":"2023-05-24T08:27:46.236062Z","shell.execute_reply.started":"2023-05-24T08:27:46.212704Z","shell.execute_reply":"2023-05-24T08:27:46.235082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub[['row_id', 'fractured']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-24T08:27:46.237719Z","iopub.execute_input":"2023-05-24T08:27:46.238365Z","iopub.status.idle":"2023-05-24T08:27:46.250503Z","shell.execute_reply.started":"2023-05-24T08:27:46.238319Z","shell.execute_reply":"2023-05-24T08:27:46.249443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}