{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n!pip install ../input/fastai2-offline/timm-0.2.1-py3-none-any.whl","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pydicom\nfrom albumentations import Compose, Normalize\nfrom albumentations.pytorch import ToTensor\nfrom torch.utils.data import Dataset\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.cuda.amp import autocast\nfrom torch.nn.functional import dropout\nimport random\nimport timm\nfrom timm.models.layers.adaptive_avgmax_pool import SelectAdaptivePool2d\nfrom collections import OrderedDict\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert(dicom_path):\n    series = os.listdir(dicom_path)[0]\n    instance = os.listdir(os.path.join(dicom_path, series))\n    stack_image = np.zeros(shape=(len(instance),512,512,3), dtype=np.uint8)\n    stack_zpos = np.zeros(shape=(len(instance),))\n    for i, ins in enumerate(instance):\n        f = pydicom.dcmread(os.path.join(dicom_path, series, ins))\n        image = f.pixel_array\n        image = image.astype(np.int16)\n        image[image <= -1000] = 0\n        _, _, intercept, slope = get_windowing(f)\n        image = image * slope + intercept\n        image_c1 = window(image, -600, 1500)\n        image_c2 = window(image, 100, 700)\n        image_c3 = window(image, 40, 400)\n        image = np.stack([image_c1, image_c2, image_c3], axis=-1)\n        image = image[:,:,::-1]\n        stack_image[i,...] = image\n        z_pos = f.ImagePositionPatient[-1]\n        stack_zpos[i] = z_pos\n    stack_image = stack_image[np.argsort(stack_zpos)]\n    instance = np.array(instance)[np.argsort(stack_zpos)]\n    \n    return stack_image, instance\n\ndef get_first_of_dicom_field_as_int(x):\n    if type(x) == pydicom.multival.MultiValue: return int(x[0])\n    else: return int(x)\n\ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value, #window width\n                    data[('0028','1052')].value, #intercept\n                    data[('0028','1053')].value] #slope\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n\ndef window(img, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    X = (X*255.0).astype('uint8')\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class to_tensor_albu:\n    def __init__(self):\n        transformation = [Normalize(),\n                           ToTensor()]\n        self.transform = Compose(transformation)\n\n    def __call__(self, x):\n        return self.transform(image=x)['image']\n\nclass SeriesDataset(Dataset):\n    def __init__(self, images):\n        self.images = images\n        self.to_tensor = to_tensor_albu()\n        \n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        image = self.images[idx]\n        image = self.to_tensor(image)\n        return image\n\nclass CatEmbeddingDataset(Dataset):\n    def __init__(self, images):\n        self.input = images\n        self.sl = 31\n        self.images = []\n        for i in range(len(images) - self.sl + 1):\n            self.images.append(images[i:i+31].unsqueeze(0))\n        self.images = torch.cat(self.images)\n\n        \n    def __len__(self):\n        return len(self.images - 30)\n\n    def __getitem__(self, idx):\n        image = self.images[idx]\n        return image        \n\ndef batch(iterable, n=1):\n    l = len(iterable)\n    for ndx in range(0, l, n):\n        yield iterable[ndx:min(ndx + n, l)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class NormSoftmax(nn.Module):\n    def __init__(self, in_features, out_features, temperature=1.):\n        super(NormSoftmax, self).__init__()\n        self.weight = nn.Parameter(\n            torch.FloatTensor(in_features, out_features))\n        nn.init.xavier_uniform_(self.weight.data)\n\n        self.ln = nn.LayerNorm(in_features, elementwise_affine=False)\n        self.temperature = nn.Parameter(torch.Tensor([temperature]))\n\n    def forward(self, x):\n        x = self.ln(x)\n        x = torch.matmul(F.normalize(x), F.normalize(self.weight))\n        x = x / self.temperature\n        return x\n\n\nclass EfficientNet(nn.Module):\n    def __init__(self, name):\n        super(EfficientNet, self).__init__()\n\n        backbone = timm.create_model(\n            model_name=name,\n            pretrained=False,\n            in_chans=3,\n        )\n        self.conv_stem = backbone.conv_stem\n        self.bn1 = backbone.bn1\n        self.act1 = backbone.act1\n        ### Original blocks ###\n        for i in range(len((backbone.blocks))):\n            setattr(self, \"block{}\".format(str(i)), backbone.blocks[i])\n        self.conv_head = backbone.conv_head\n        self.bn2 = backbone.bn2\n        self.act2 = backbone.act2\n        self.global_pool = SelectAdaptivePool2d(pool_type=\"avg\")\n        self.num_features = backbone.num_features\n\n        ### Baseline head ###\n        self.fc = nn.Linear(self.num_features, 7)\n        del backbone\n\n    def _features(self, x):\n        x = self.conv_stem(x)\n        x = self.bn1(x)\n        x = self.act1(x)\n        x = self.block0(x)\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = self.block4(x); b4 = x\n        x = self.block5(x); b5 = x\n        x = self.block6(x)\n        x = self.conv_head(x)\n        x = self.bn2(x)\n        x = self.act2(x)\n        return b4,b5,x\n\n    def forward(self, x):\n        with autocast():\n            b4, b5, x = self._features(x)\n            x = self.global_pool(x)\n            x = torch.flatten(x, 1)\n            logits = self.fc(x)\n            return logits\n\n\nclass EmbeddingNet(nn.Module):\n    def __init__(self):\n        super(EmbeddingNet, self).__init__()        \n        self.fw = nn.Sequential(\n            nn.Conv2d(1, 32, 7, 1, 3),\n            nn.Dropout(p=0.2),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(32, 64, 5, 1, 2),\n            nn.Dropout(p=0.2),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(64, 128, 3, 1, 1),\n            nn.Dropout(p=0.2),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(128, 256, 3, 1, 1),\n            nn.Dropout(p=0.2),\n            nn.ReLU(inplace=True),\n            SelectAdaptivePool2d(pool_type=\"avg\"),\n            nn.Flatten(),\n            nn.Linear(256, 7)\n        )\n        self.conv_out = nn.Linear(7, 1)\n\n        self.lstm1 = nn.LSTM(7, 128, bidirectional=True, batch_first=True)\n        self.lstm2 = nn.LSTM(128 * 2, 128, bidirectional=True, batch_first=True)\n        self.linear1 = nn.Linear(128*2, 128*2)\n        self.linear2 = nn.Linear(128*2, 128*2)\n        self.linear = nn.Linear(128*2, 7)\n        self.lstm_out = nn.Linear(7, 1)\n\n    def forward(self, x):\n        with autocast():\n            embedding_vector = x.squeeze(1).float()\n            logits1 = self.fw(x)\n            second_logits1 = self.conv_out((logits1))\n\n            h_lstm1, _ = self.lstm1(embedding_vector)\n            h_lstm2, _ = self.lstm2(h_lstm1)\n            h_conc_linear1  = F.relu(self.linear1(h_lstm1))\n            h_conc_linear2  = F.relu(self.linear2(h_lstm2))\n            hidden = h_lstm1 + h_lstm2 + h_conc_linear1 + h_conc_linear2\n            logits2 = self.linear(hidden)[:,-1,:]\n            second_logits2 = self.lstm_out((logits2))\n\n            return logits1, logits2, second_logits1, second_logits2\n\n\nclass SeriesEmbeddingNet(nn.Module):\n    def __init__(self):\n        super(SeriesEmbeddingNet, self).__init__()\n        \n        self.netA = timm.create_model(\n            model_name=\"tf_efficientnet_b0\",\n            pretrained=False,\n            in_chans=1\n        )\n        self.netA.classifier = NormSoftmax(self.netA.num_features, 9)\n        \n        self.netB = timm.create_model(\n            model_name=\"tf_efficientnet_b0\",\n            pretrained=False,\n            in_chans=1\n        )\n        self.netB.classifier = NormSoftmax(self.netB.num_features, 9)\n        \n        self.netC = timm.create_model(\n            model_name=\"tf_efficientnet_b0\",\n            pretrained=False,\n            in_chans=1\n        )\n        self.netC.classifier = NormSoftmax(self.netC.num_features, 9)\n    \n    def forward(self, x):\n        with autocast():\n            logits1 = self.netA(x)\n            logits2 = self.netB(x)\n            logits3 = self.netC(x)\n            return (logits1 + logits2 + logits3) / 3.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"backbone_b4f0 = EfficientNet(\"tf_efficientnet_b4\").cuda()\ncheckpoint = torch.load(\"../input/rsna-b4-folds/b4_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n    if \"second_\" not in k:\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nbackbone_b4f0.load_state_dict(checkpoint_surged)\nbackbone_b4f0.eval()\ndel checkpoint_surged\nbackbone_b4f1 = EfficientNet(\"tf_efficientnet_b5\").cuda()\ncheckpoint = torch.load(\"../input/rsnastrweights/b5_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n    if \"second_\" not in k:\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nbackbone_b4f1.load_state_dict(checkpoint_surged)\nbackbone_b4f1.eval()\ndel checkpoint_surged\n\nembeddingnet_b4f0 = EmbeddingNet().cuda()\ncheckpoint = torch.load(\"../input/rsna-b4-folds/cnn_lstm_embeddings_b4_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nembeddingnet_b4f0.load_state_dict(checkpoint_surged)\nembeddingnet_b4f0.eval()\ndel checkpoint_surged\nembeddingnet_b4f1 = EmbeddingNet().cuda()\ncheckpoint = torch.load(\"../input/rsnastrweights/cnn_lstm_embeddings_b5_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nembeddingnet_b4f1.load_state_dict(checkpoint_surged)\nembeddingnet_b4f1.eval()\ndel checkpoint_surged\n\nseriesnet_b4f0 = SeriesEmbeddingNet().cuda()\ncheckpoint = torch.load(\"../input/rsna-b4-folds/triple_b0_series_b4_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nseriesnet_b4f0.load_state_dict(checkpoint_surged)\nseriesnet_b4f0.eval()\ndel checkpoint_surged\nseriesnet_b4f1 = SeriesEmbeddingNet().cuda()\ncheckpoint = torch.load(\"../input/rsnastrweights/triple_b0_series_b5_sz512_fold0.pth\", \"cpu\").pop('state_dict')\ncheckpoint_surged = OrderedDict()\nfor k, v in checkpoint.items():\n        checkpoint_surged[k.replace(\"module.\",\"\")] = v\ndel checkpoint\nseriesnet_b4f1.load_state_dict(checkpoint_surged)\nseriesnet_b4f1.eval()\ndel checkpoint_surged","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\ntest_df=pd.read_csv('../input/rsna-str-pulmonary-embolism-detection/test.csv')\nif os.path.exists(\"../input/rsna-str-pulmonary-embolism-detection/train\"):\n    test_df = test_df[test_df[\"StudyInstanceUID\"].isin(test_df[\"StudyInstanceUID\"].unique()[:10])]\nsubmission_csv_data = []\nfor study in tqdm(test_df[\"StudyInstanceUID\"].unique()):\n    stack_image, instance_list = convert(f\"../input/rsna-str-pulmonary-embolism-detection/test/{study}\")\n    stack_image = SeriesDataset(stack_image)\n\n\n    f0_embeddings_out = []\n    f1_embeddings_out = []\n    for images in batch(stack_image, BATCH_SIZE):\n        images = images.permute(1,0,2,3).cuda().half()\n        with autocast():\n            with torch.no_grad():\n                f0_embeddings_out.append(backbone_b4f0(images))\n                f1_embeddings_out.append(backbone_b4f1(images))\n    f0_embeddings_out = torch.cat(f0_embeddings_out)\n    f1_embeddings_out = torch.cat(f1_embeddings_out)\n\n    f0_first_slices = torch.cat([f0_embeddings_out[0,:].unsqueeze(0) for _ in range(15)])\n    f0_last_slices = torch.cat([f0_embeddings_out[-1,:].unsqueeze(0) for _ in range(15)])\n    f0_embeddings_out = torch.cat([f0_first_slices, f0_embeddings_out, f0_last_slices])\n    f0_embeddings_out = CatEmbeddingDataset(f0_embeddings_out)\n    f1_first_slices = torch.cat([f1_embeddings_out[0,:].unsqueeze(0) for _ in range(15)])\n    f1_last_slices = torch.cat([f1_embeddings_out[-1,:].unsqueeze(0) for _ in range(15)])\n    f1_embeddings_out = torch.cat([f1_first_slices, f1_embeddings_out, f1_last_slices])\n    f1_embeddings_out = CatEmbeddingDataset(f1_embeddings_out)\n    \n    f0_embeddings_stage2 = []\n    f1_embeddings_stage2 = []\n    images_output = []\n    for images_f0, images_f1 in zip(batch(f0_embeddings_out, BATCH_SIZE), batch(f1_embeddings_out, BATCH_SIZE)):\n        images_f0 = images_f0.unsqueeze(1).cuda()\n        images_f1 = images_f1.unsqueeze(1).cuda()\n        with autocast():\n            with torch.no_grad():\n                w_output_1_f0, w_output_2_f0, second_w_output_1_f0, second_w_output_2_f0 = embeddingnet_b4f0(images_f0)\n                w_output_3_f0, w_output_4_f0, second_w_output_3_f0, second_w_output_4_f0 = embeddingnet_b4f0(images_f0.flip(2))\n                w_output_1_f1, w_output_2_f1, second_w_output_1_f1, second_w_output_2_f1 = embeddingnet_b4f1(images_f1)\n                w_output_3_f1, w_output_4_f1, second_w_output_3_f1, second_w_output_4_f1 = embeddingnet_b4f1(images_f1.flip(2))\n                ensemble_out = second_w_output_1_f0+second_w_output_2_f0+second_w_output_3_f0+second_w_output_4_f0\n                ensemble_out += second_w_output_1_f1+second_w_output_2_f1+second_w_output_3_f1+second_w_output_4_f1\n                ensemble_out = torch.sigmoid(ensemble_out / 8.)\n                images_output.append(ensemble_out)\n                f0_embeddings_stage2.append(torch.cat([w_output_1_f0,second_w_output_1_f0,w_output_2_f0,second_w_output_2_f0,w_output_3_f0,second_w_output_3_f0,w_output_4_f0,second_w_output_4_f0], dim=1))\n                f1_embeddings_stage2.append(torch.cat([w_output_1_f1,second_w_output_1_f1,w_output_2_f1,second_w_output_2_f1,w_output_3_f1,second_w_output_3_f1,w_output_4_f1,second_w_output_4_f1], dim=1))\n    images_output = torch.cat(images_output).cpu().numpy()\n    for instanceuid,image_out in zip(instance_list, images_output):\n        submission_csv_data.append([instanceuid.replace(\".dcm\",\"\"), image_out.reshape(-1)[0]])\n\n    f0_embeddings_stage2 = torch.cat(f0_embeddings_stage2, dim=0)\n    f1_embeddings_stage2 = torch.cat(f1_embeddings_stage2, dim=0)\n\n\n    seq_len = f0_embeddings_stage2.size()[0]\n    if seq_len < 1024:\n        padsize = 1024 - seq_len\n        before_pad = int(padsize / 2)\n        after_pad = int(padsize / 2)\n        if padsize > (before_pad+after_pad):\n            after_pad += 1\n        f0_embeddings_stage2 = torch.cat([torch.tensor(np.zeros((before_pad,32))), f0_embeddings_stage2.cpu(), torch.tensor(np.zeros((after_pad,32)))])\n        f1_embeddings_stage2 = torch.cat([torch.tensor(np.zeros((before_pad,32))), f1_embeddings_stage2.cpu(), torch.tensor(np.zeros((after_pad,32)))])\n    elif seq_len > 1024:\n        truncate = seq_len - 1024\n        before_truncate = int(truncate / 2)\n        after_truncate = int(truncate / 2)\n        if truncate > (before_truncate+after_truncate):\n            after_truncate += 1\n        f0_embeddings_stage2 = f0_embeddings_stage2[before_truncate:-(after_truncate),:].cpu()\n        f1_embeddings_stage2 = f1_embeddings_stage2[before_truncate:-(after_truncate),:].cpu()\n    else:\n        f0_embeddings_stage2 = f0_embeddings_stage2.cpu()\n        f1_embeddings_stage2 = f1_embeddings_stage2.cpu()\n\n    with autocast():\n        with torch.no_grad():\n            f0_embeddings_stage2 = f0_embeddings_stage2.unsqueeze(0).unsqueeze(0).cuda().half()\n            f1_embeddings_stage2 = f1_embeddings_stage2.unsqueeze(0).unsqueeze(0).cuda().half()\n            out1 = (seriesnet_b4f0(f0_embeddings_stage2) + seriesnet_b4f1(f1_embeddings_stage2)) / 4.\n            out2 = (seriesnet_b4f0(f0_embeddings_stage2.flip(2)) + seriesnet_b4f1(f1_embeddings_stage2.flip(2))) / 4.\n            out = torch.sigmoid((out1+out2).cpu()).squeeze(0).numpy()\n    submission_csv_data.append([study+\"_negative_exam_for_pe\", out[0]])\n    submission_csv_data.append([study+\"_indeterminate\", out[1]])\n    submission_csv_data.append([study+\"_chronic_pe\", out[2]])\n    submission_csv_data.append([study+\"_acute_and_chronic_pe\", out[3]])\n    submission_csv_data.append([study+\"_central_pe\", out[4]])\n    submission_csv_data.append([study+\"_leftsided_pe\", out[5]])\n    submission_csv_data.append([study+\"_rightsided_pe\", out[6]])\n    submission_csv_data.append([study+\"_rv_lv_ratio_gte_1\", out[7]])\n    submission_csv_data.append([study+\"_rv_lv_ratio_lt_1\", out[8]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(data=submission_csv_data, columns=['id', 'label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### https://www.kaggle.com/anthracene/host-confirmed-label-consistency-check\ndef check_consistency(sub, test):\n    \n    '''\n    Checks label consistency and returns the errors\n    \n    Args:\n    sub   = submission dataframe (pandas)\n    test  = test.csv dataframe (pandas)\n    '''\n    \n    # EXAM LEVEL\n    for i in test['StudyInstanceUID'].unique():\n        df_tmp = sub.loc[sub.id.str.contains(i, regex = False)].reset_index(drop = True)\n        df_tmp['StudyInstanceUID'] = df_tmp['id'].str.split('_').str[0]\n        df_tmp['label_type']       = df_tmp['id'].str.split('_').str[1:].apply(lambda x: '_'.join(x))\n        del df_tmp['id']\n        if i == test['StudyInstanceUID'].unique()[0]:\n            df = df_tmp.copy()\n        else:\n            df = pd.concat([df, df_tmp], axis = 0)\n    df_exam = df.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')\n    \n    # IMAGE LEVEL\n    df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)\n    df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')\n    df_image.rename(columns = {\"label\": \"pe_present_on_image\"}, inplace = True)\n    del df_image['id']\n    \n    # MERGER\n    df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')\n    ids    = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']\n    labels = [c for c in df.columns if c not in ids]\n    df = df[ids + labels]\n    \n    # SPLIT NEGATIVE AND POSITIVE EXAMS\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n    \n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n    rule1d = df_pos.loc[(df_pos.indeterminate        > 0.5) | \n                        (df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    \n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    \n    # OUTPUT\n    print('Found', len(errors), 'inconsistent predictions')\n    return errors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def neg_ind_pos(exam_pred):\n    p = exam_pred[0:2].copy()                       ### negative_exam_for_pe, indeterminate prediction\n    if p[0] > 0.5 and p[1] <= 0.5:                  ### negative_exam_for_pe\n        return 0\n    elif p[0] > 0.5 and p[1] > 0.5:\n        if 0.0736196319*p[0] > 0.09202453988*p[1]:\n            return 0                                ### negative_exam_for_pe\n        else:\n            return 1                                ### indeterminate\n    elif p[0] <= 0.5 and p[1] > 0.5:                ### indeterminate\n        return 1\n    else:\n        return 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Post-processing\neps = 1e-6\n\nexam_preds = []\nexam_ids = []\nimage_preds = []\nimage_ids = []\nfor StudyInstanceUID, grp in tqdm(test_df.groupby('StudyInstanceUID')):\n    exam_ids.append(StudyInstanceUID)\n    \n    study_instances = grp['SOPInstanceUID'].values\n    study_df = df[df[\"id\"].isin(study_instances)]\n    preds = study_df[\"label\"].values\n    for instance in study_df[\"id\"].values:\n        image_ids.append(instance)\n    idx = np.argmax(preds)\n    COLUMNS = ['negative_exam_for_pe','indeterminate','rightsided_pe','leftsided_pe','central_pe',\n           'rv_lv_ratio_gte_1','rv_lv_ratio_lt_1','chronic_pe','acute_and_chronic_pe']\n    study_exam_names = [StudyInstanceUID+\"_\"+c for c in COLUMNS]\n    exam_pred = df[df[\"id\"].isin(study_exam_names)].label.values\n    exam_pred = exam_pred[[0,1,6,5,4,7,8,2,3]]\n    c = neg_ind_pos(exam_pred)\n\n    if c == 0:\n        ### negative_exam_for_pe\n        preds = np.where(preds > 0.5, 0.5-eps, preds)\n        for i in [1,2,3,4,5,6,7,8]:\n            if exam_pred[i] > 0.5:\n                exam_pred[i] = 0.5-eps\n    elif c == 1:\n        ### indeterminate\n        preds = np.where(preds > 0.5, 0.5-eps, preds)\n        for i in [0,2,3,4,5,6,7,8]:\n            if exam_pred[i] > 0.5:\n                exam_pred[i] = 0.5-eps\n    else:\n        ### positive_exam_for_pe\n\n        ### pe_present_on_image\n        if preds[idx] <= 0.5:\n            preds[idx] = 0.5+eps\n\n        ### rightsided_pe,leftsided_pe,central_pe\n        ri_le_ce = np.argmax(exam_pred[2:5])                                \n        if exam_pred[ri_le_ce+2] <= 0.5:\n            exam_pred[ri_le_ce+2] = 0.5+eps\n\n        ### rv_lv_ratio_gte_1,rv_lv_ratio_lt_1\n        if exam_pred[5] > 0.5 and exam_pred[6] > 0.5:\n            if exam_pred[5] > exam_pred[6]:\n                exam_pred[6] = 0.5-eps\n            else:\n                exam_pred[5] = 0.5-eps\n        elif exam_pred[5] <= 0.5 and exam_pred[6] <= 0.5:\n            if exam_pred[5] > exam_pred[6]:\n                exam_pred[5] = 0.5+eps\n            else:\n                exam_pred[6] = 0.5+eps\n\n        ### chronic_pe,acute_and_chronic_pe\n        if exam_pred[7] > 0.5 and exam_pred[8] > 0.5:\n            if exam_pred[7] > exam_pred[8]:\n                exam_pred[8] = 0.5-eps\n            else:\n                exam_pred[7] = 0.5-eps\n    image_preds.append(preds)\n    exam_preds.append(exam_pred)\nimage_ids = np.array(image_ids)\nimage_preds = np.concatenate(image_preds).astype(np.float64)\n\nexam_ids = np.array(exam_ids)\nexam_preds = np.array(exam_preds, dtype=np.float64)\n\nids = []\nlabels = []\nfor StudyInstanceUID, preds in zip(exam_ids, exam_preds):\n    for col, pred in zip(COLUMNS, preds):\n        ids.append('{}_{}'.format(StudyInstanceUID, col))\n        labels.append(pred)\nfor SOPInstanceUID, pred in zip(image_ids, image_preds):\n    ids.append(SOPInstanceUID)\n    labels.append(pred)\n\nsub_df = pd.DataFrame()\nsub_df['id'] = np.array(ids)\nsub_df['label'] = np.array(labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"errors = check_consistency(sub_df, test_df)\nif len(errors) == 0:\n    sub_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf gdcm*","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}