{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport sys\nsys.path = [\n    '../input/efficientnet-pytorch/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n] + sys.path","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport torch\nimport random\nimport pydicom\nimport datetime\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset\nfrom scipy.ndimage.interpolation import zoom\nfrom sklearn.model_selection import *","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Init"},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    \"\"\"\n    Seeds basic parameters for reproductibility of results\n\n    Arguments:\n        seed {int} -- Number of the seed\n    \"\"\"\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True  # False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 2020\nseed_everything(SEED)\n\nMEAN = np.array([0.485, 0.456, 0.406])\nSTD = np.array([0.229, 0.224, 0.225])\n\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n\nSIZE = 256\n\nIMG_TARGET = \"pe_present_on_image\"\n\nEXAM_TARGETS = [\n    \"negative_exam_for_pe\",\n    \"rv_lv_ratio_gte_1\",\n    \"rv_lv_ratio_lt_1\",\n    \"leftsided_pe\",\n    \"chronic_pe\",\n    \"rightsided_pe\",\n    \"acute_and_chronic_pe\",\n    \"central_pe\",\n    \"indeterminate\",\n]\n\nNUM_EXAM_TARGETS = len(EXAM_TARGETS)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATA_PATH = \"../input/rsna-str-pulmonary-embolism-detection/\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dicom processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_dicom(f):\n    dicom = pydicom.dcmread(f)\n    \n    M = float(dicom.RescaleSlope)\n    B = float(dicom.RescaleIntercept)\n    \n    z_pos = float(dicom.ImagePositionPatient[-1])\n    img = dicom.pixel_array * M + B\n    \n    return img, z_pos\n\n\ndef window(img, WL=50, WW=350):\n    upper, lower = WL + WW // 2, WL - WW // 2\n    X = np.clip(img, lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    \n    X = (X * 255.0).astype('uint8')\n    \n    return X\n\ndef dicom_to_img(path):\n    image, z_pos = load_dicom(path)\n\n    image_lung = window(image, WL=-600, WW=1500)[:, :, np.newaxis]\n    image_mediastinal = window(image, WL=40, WW=400)[:, :, np.newaxis]\n    image_pe_specific = window(image, WL=100, WW=700)[:, :, np.newaxis]\n    \n    image = np.concatenate([image_mediastinal, image_pe_specific, image_lung], axis=-1)\n    \n    rat = SIZE / np.max(image.shape[1:])\n    image = zoom(image, [rat, rat,1.], prefilter=False, order=1)\n    \n    return image, z_pos","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import albumentations as albu \nfrom albumentations import pytorch as AT\n\n\ndef normalizer(mean=MEAN, std=STD):\n    return albu.Compose([\n                albu.Normalize(mean=mean, std=std),\n                AT.transforms.ToTensor(),\n            ],\n            p=1,\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_jpg(img, name='img'):\n    cv2.imwrite(name + \".jpg\", img)\n    return cv2.imread(name + \".jpg\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PatientDataset(Dataset):\n    \"\"\"\n    Dataset for feature extraction\n    \"\"\"\n    def __init__(self, df, path, transforms=None): \n        self.df = df\n        self.path = path\n        self.img_paths = [sop + '.dcm' for sop in df['SOPInstanceUID'].values]\n        \n        assert len(self.img_paths) == len(self.df)\n        \n        self.transforms = transforms\n\n    def __len__(self):\n        return len(self.img_paths)\n\n    def __getitem__(self, idx):\n        image, z = dicom_to_img(self.path + self.img_paths[idx])\n        image = to_jpg(image, name='tmp')\n        \n        if self.transforms:\n            image = self.transforms(image=image)[\"image\"]\n\n        return image, z\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_model_weights(model, filename, verbose=1, cp_folder=\"\"):\n    \"\"\"\n    Loads the weights of a PyTorch model. The exception handles cpu/gpu incompatibilities\n\n    Arguments:\n        model {torch module} -- Model to load the weights to\n        filename {str} -- Name of the checkpoint\n\n    Keyword Arguments:\n        verbose {int} -- Whether to display infos (default: {1})\n        cp_folder {str} -- Folder to load from (default: {''})\n\n    Returns:\n        torch module -- Model with loaded weights\n    \"\"\"\n    if verbose:\n        print(f\"\\n -> Loading weights from {os.path.join(cp_folder,filename)}\\n\")\n    try:\n        model.load_state_dict(os.path.join(cp_folder, filename), strict=False)\n    except BaseException:\n        model.load_state_dict(\n            torch.load(os.path.join(cp_folder, filename), map_location=\"cpu\"),\n            strict=True,\n        )\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## First level"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn.functional as F\n\n\ndef add_ft_extractor(resnet):\n    resnet.extract_ft = lambda x: forward_extract_ft(resnet, x)\n    \n\ndef forward_extract_ft(self, x):\n    x = self.conv1(x)\n    x = self.bn1(x)\n    x = self.relu(x)\n    x = self.maxpool(x)\n\n    x = self.layer1(x)\n    x = self.layer2(x)\n    x = self.layer3(x)\n    x = self.layer4(x)\n\n    x = self.avgpool(x)\n    ft = torch.flatten(x, 1)\n\n    x = self.fc(ft)\n\n    return x.view(-1), ft\n\n\ndef add_ft_extractor_enet(effnet):\n    effnet.forward = lambda x: forward_extract_ft_enet(effnet, x)\n    \n\ndef forward_extract_ft_enet(self, x):\n    x = self.extract_features(x)\n\n    x = self._avg_pooling(x)\n\n    ft = x.flatten(start_dim=1)\n    x = self._dropout(ft)\n    x = self._fc(x)\n\n    return x.view(-1), ft","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision.models import *\nfrom efficientnet_pytorch import EfficientNet\n\n\ndef define_model(name, num_classes=1):\n    if \"resnext\" in name:\n        model = resnext50_32x4d(pretrained=False)\n        model.nb_ft = model.fc.in_features\n        model.fc = nn.Linear(model.nb_ft, num_classes)\n        add_ft_extractor(model)\n    else:\n        model = EfficientNet.from_name(name)\n        model.nb_ft = model._fc.in_features\n        model._fc = nn.Linear(model.nb_ft, num_classes)\n        add_ft_extractor_enet(model)\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2nd level"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RNNModel(nn.Module):\n    def __init__(self, ft_dim=2048, lstm_dim=64, dense_dim=256, use_msd=False, num_classes=9):\n        super().__init__()\n        self.use_msd = use_msd\n        \n        self.mlp = nn.Sequential(\n            nn.Linear(ft_dim, dense_dim * 2),\n            nn.ReLU(),\n            nn.Linear(dense_dim * 2, dense_dim),\n            nn.ReLU(),\n        )\n\n        self.lstm = nn.LSTM(dense_dim, lstm_dim, batch_first=True, bidirectional=True)\n        \n        self.logits_exam = nn.Sequential(\n            nn.Linear(lstm_dim * 4 + dense_dim * 2, lstm_dim),\n            nn.ReLU(),\n            nn.Linear(lstm_dim, num_classes),\n        )\n        \n        self.logits_img = nn.Sequential(\n            nn.Linear(lstm_dim *  2 + dense_dim, lstm_dim),\n            nn.ReLU(),\n            nn.Linear(lstm_dim, 1),\n        )\n        \n        self.high_dropout = nn.Dropout(p=0.5)\n    \n    def forward(self, x):\n        features = self.mlp(x)\n        features2, _ = self.lstm(features)\n        \n        features = torch.cat([features, features2], -1)\n        \n        mean = features.mean(1)\n        max_, _ = features.max(1)\n        pooled = torch.cat([mean, max_], -1)\n        \n        if self.use_msd and self.training:\n            logits_exam = torch.mean(\n                torch.stack(\n                    [self.logits_exam(self.high_dropout(pooled)) for _ in range(5)],\n                    dim=0,\n                    ),\n                dim=0,\n            )\n            \n            logits_img = torch.mean(\n                torch.stack(\n                    [self.logits_img(self.high_dropout(features)) for _ in range(5)],\n                    dim=0,\n                    ),\n                dim=0,\n            )\n        else:\n            logits_exam = self.logits_exam(pooled)\n            logits_img = self.logits_img(features)\n\n        return logits_exam, logits_img.squeeze(-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ensure rules verification"},{"metadata":{"trusted":true},"cell_type":"code","source":"def post_process(exam_pred, image_pred, eps=1e-6):\n    has_pe_image = image_pred.max() > 0.5\n\n    (negative_exam_for_pe, rv_lv_ratio_gte_1, rv_lv_ratio_lt_1, leftsided_pe, \n     chronic_pe, rightsided_pe, acute_and_chronic_pe, central_pe, indeterminate) = exam_pred\n    \n    broken = False\n    \n    if has_pe_image:  # \n        if rv_lv_ratio_gte_1 < 0.5 and rv_lv_ratio_lt_1 < 0.5:  # this means image has no PE\n            broken = True\n            \n        max_ = np.max([leftsided_pe, rightsided_pe, central_pe])  # this means image has no PE\n        if max_ < 0.5:\n            broken = True\n            \n    if not broken and has_pe_image:\n        if rv_lv_ratio_gte_1 > 0.5 and rv_lv_ratio_lt_1 > 0.5:\n            if rv_lv_ratio_gte_1 > rv_lv_ratio_lt_1:\n                rv_lv_ratio_lt_1 = 0.5 - eps\n            else:\n                rv_lv_ratio_gte_1 = 0.5 - eps\n        \n        min_ = np.min([chronic_pe, acute_and_chronic_pe])  # almost never occurs\n        if min_ > 0.5:\n            if chronic_pe > acute_and_chronic_pe:\n                acute_and_chronic_pe = 0.5 - eps\n            else:\n                chronic_pe = 0.5 - eps\n        \n    if broken:\n        image_pred = np.clip(image_pred, 0, 0.5 - eps)\n        assert image_pred.max() < 0.5\n        \n    has_pe_image = image_pred.max() > 0.5\n    \n    if not has_pe_image:   \n        if indeterminate > 0.5 and negative_exam_for_pe > 0.5:\n            if indeterminate > negative_exam_for_pe:\n                negative_exam_for_pe = 0.5 - eps\n            else:\n                indeterminate = 0.5 - eps\n        elif indeterminate < 0.5 and negative_exam_for_pe < 0.5:\n            if indeterminate > negative_exam_for_pe:\n                indeterminate = 0.5 + eps\n            else:\n                negative_exam_for_pe = 0.5 + eps\n        \n        (rv_lv_ratio_gte_1, rv_lv_ratio_lt_1, leftsided_pe, \n         chronic_pe, rightsided_pe, acute_and_chronic_pe, central_pe) = np.clip(\n            exam_pred[[1, 2, 3, 4, 5, 6, 7]],\n            0, \n            0.5 - eps\n        )\n        \n    return np.array([\n        negative_exam_for_pe,\n        rv_lv_ratio_gte_1, \n        rv_lv_ratio_lt_1, \n        leftsided_pe, \n        chronic_pe,\n        rightsided_pe,\n        acute_and_chronic_pe,\n        central_pe,\n        indeterminate, \n    ]), image_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_consistency(sub, test):\n    \n    '''\n    Checks label consistency and returns the errors\n    \n    Args:\n    sub   = submission dataframe (pandas)\n    test  = test.csv dataframe (pandas)\n    '''\n    \n    # EXAM LEVEL\n    for i in test['StudyInstanceUID'].unique():\n        df_tmp = sub.loc[sub.id.str.contains(i, regex = False)].reset_index(drop = True)\n        df_tmp['StudyInstanceUID'] = df_tmp['id'].str.split('_').str[0]\n        df_tmp['label_type']       = df_tmp['id'].str.split('_').str[1:].apply(lambda x: '_'.join(x))\n        del df_tmp['id']\n        if i == test['StudyInstanceUID'].unique()[0]:\n            df = df_tmp.copy()\n        else:\n            df = pd.concat([df, df_tmp], axis = 0)\n    df_exam = df.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')\n    \n    # IMAGE LEVEL\n    df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)\n    df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')\n    df_image.rename(columns = {\"label\": \"pe_present_on_image\"}, inplace = True)\n    del df_image['id']\n    \n    # MERGER\n    df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')\n    ids    = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']\n    labels = [c for c in df.columns if c not in ids]\n    df = df[ids + labels]\n    \n    # SPLIT NEGATIVE AND POSITIVE EXAMS\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n    \n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n    rule1d = df_pos.loc[(df_pos.indeterminate        > 0.5) | \n                        (df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    \n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    \n    # OUTPUT\n    print('Found', len(errors), 'inconsistent predictions')\n    return errors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader\n\ndef extract_features(models, dataset, batch_size=32):\n    fts = np.empty((0, models[0].nb_ft))\n    zs = np.empty(0)\n    preds = np.empty(0)\n\n    loader = DataLoader(\n        dataset, \n        batch_size=batch_size, \n        shuffle=False, \n        num_workers=1,\n        drop_last=False,\n        pin_memory=True\n    )\n    \n    with torch.no_grad():\n        for x, z in loader:\n            fts_tmp = []\n            preds_tmp = []\n            for model in models:\n                pred, ft = model(x.cuda())\n                preds_tmp.append(torch.sigmoid(pred).detach().cpu().numpy())\n                fts_tmp.append(ft.detach().cpu().numpy())\n                \n            preds = np.concatenate([preds, np.mean(preds_tmp, 0)])\n            fts = np.concatenate([fts, np.mean(fts_tmp, 0)])\n            zs = np.concatenate([zs, z.numpy()])\n\n    order = np.argsort(zs)\n    fts = fts[order]\n    preds = preds[order]\n    \n    return preds, fts, order","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inference_second_lvl(model, ft):\n    with torch.no_grad():\n        x = torch.from_numpy(ft).unsqueeze(0).cuda().float()\n        logits_exam, logits_img = model(x)\n        \n        pred_exam = torch.sigmoid(logits_exam).detach().cpu().squeeze().numpy()\n        pred_img = torch.sigmoid(logits_img).detach().cpu().squeeze().numpy()\n        \n    return pred_exam, pred_img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred_to_sub(unique_df, df, pred_exams, pred_imgs, orders):\n    df_dic = {}\n    for i, study in enumerate(unique_df['StudyInstanceUID'].values):\n        reverse_order = np.argsort(orders[i])\n        for t, target in enumerate(EXAM_TARGETS):\n            name = f'{study}_{target}'\n            label = pred_exams[i][t]\n            df_dic[name] = label\n        \n        df_patient = df[df['StudyInstanceUID'] == study]\n        for s, sop in enumerate(df_patient['SOPInstanceUID'].values):\n            df_dic[sop] = pred_imgs[i][reverse_order[s]]\n    \n    sub = pd.DataFrame.from_dict(df_dic, orient='index', columns=['label']).reset_index()\n    return sub.rename({'index': 'id'}, axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main"},{"metadata":{},"cell_type":"markdown","source":"## Paths"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(DATA_PATH + \"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paths_test = [DATA_PATH + f\"test/{study}/{series}/\" for study, series in df_test[['StudyInstanceUID', 'SeriesInstanceUID']].values]\ndf_test['path'] = paths_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_df_test = df_test[['path', 'StudyInstanceUID', 'SeriesInstanceUID']].drop_duplicates().reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pretrained models"},{"metadata":{"trusted":true},"cell_type":"code","source":"CP_PATH = \"../input/peweights/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"efficientnets = [f\"efficientnet-b3__{i}.pt\" for i in range(5)]\n\nrnns_1 = [f'rnn_2_{i}.pt' for i in range(5)]\nrnns_2 = [f'rnn_3_{i}.pt' for i in range(5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_lvl_models = []\n    \nfor weights in efficientnets:\n    model = define_model('efficientnet-b3').cuda().eval()\n    model = load_model_weights(model, CP_PATH + weights)\n    first_lvl_models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"second_lvl_models = []\n\nfor weights in rnns_1:\n    model = RNNModel(\n        ft_dim=1536, \n        lstm_dim=64,\n        use_msd=True,\n    ).cuda().eval()\n    \n    model = load_model_weights(model, CP_PATH + weights)\n    second_lvl_models.append(model)\n    \n# for weights in rnns_2:\n#     model = RNNModel(\n#         ft_dim=1536, \n#         lstm_dim=256,\n#         use_msd=True,\n#     ).cuda().eval()\n    \n#     model = load_model_weights(model, CP_PATH + weights)\n#     second_lvl_models.append(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"IS_PUBLIC = len(unique_df_test) <= 650\nDO_PUBLIC = True\nDO_PRIVATE = True\n\nDO_INFERENCE = (IS_PUBLIC and DO_PUBLIC) or (not IS_PUBLIC and DO_PRIVATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if DO_INFERENCE:\n    all_pred_exams = []\n    all_pred_imgs = []\n    orders = []\n\n    for path, study, series in tqdm(unique_df_test.values):\n        df_patient = df_test[df_test['StudyInstanceUID'] == study].reset_index(drop=True)\n\n        dataset = PatientDataset(df_patient, path, normalizer())\n\n#         print('First level inference. \\n')\n\n        preds_1, fts, order = extract_features(first_lvl_models, dataset, batch_size=16)\n        orders.append(order)\n\n#         print('Second level inference. \\n')\n\n        pred_exams = []\n        pred_imgs = []\n        for model in second_lvl_models:\n            pred_exam, pred_img = inference_second_lvl(model, fts)\n            pred_exams.append(pred_exam)\n            pred_imgs.append(pred_img)\n\n        pred_exams = np.mean(pred_exams, 0)\n        pred_imgs = np.mean(pred_imgs, 0)\n        \n        pred_exams, pred_imgs = post_process(pred_exams, pred_imgs, eps=1e-6)\n\n        all_pred_exams.append(pred_exams)\n        all_pred_imgs.append(pred_imgs)\n        \n    sub = pred_to_sub(unique_df_test, df_test, all_pred_exams, all_pred_imgs, orders)\nelse:\n    sub = pd.read_csv(DATA_PATH + \"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_consistency(sub, df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)\nsub","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}