{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import gdcm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.medical.imaging import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_columns = 100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"datapath = Path(\"/kaggle/input/rsna-str-pulmonary-embolism-detection/\")\ntest_df = pd.read_csv(datapath/'test.csv')\nsub_df = pd.read_csv(datapath/'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### test gdcm"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dcm_files = get_dicom_files(datapath/'train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pe_window = (700, 100)\n# train_metadata = pd.DataFrame.from_dicoms(train_dcm_files, window=pe_window)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load and export learn for future"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dls(files, size=256, bs=128):\n    tfms = [[PILImage.create, ToTensor, RandomResizedCrop(size, min_scale=0.9)], \n            [lambda o: np.random.choice([0,1]), Categorize()]]\n\n    dsets = Datasets(files, tfms=tfms, splits=RandomSplitter(0.1)(files))\n\n    batch_tfms = [IntToFloatTensor]\n    dls = dsets.dataloaders(bs=bs, after_batch=batch_tfms)\n    return dls","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imagepath = Path(\"/kaggle/input/rsna-str-pe-detection-jpeg-256/\")\nfiles = get_image_files((imagepath/'train-jpegs').ls()[0])\nfiles = files[:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls = get_dls(files, bs=64)\nlearn = cnn_learner(dls, xresnet34, pretrained=False)\nlearn.path = Path(\"/kaggle/input/rsnastrpecnnmodel/\")\nlearn.load('basemodel-ft')\nlearn.to_fp16();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"# RGB windows\nlung_window = (1500, -600)\npe_window = (700, 100)\nmediastinal_window = (400, 40)\nwindows = (lung_window, pe_window, mediastinal_window)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dcm_files = get_dicom_files(datapath/'test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_testdl(test_files, size=256, method='crop', bs=128):\n    \"At inference time we directly read dcm files not jpg images, so we need a diff get dls func\"\n    tfms = [[Path.dcmread, partial(DcmDataset.to_nchan, wins=windows, bins=0), Resize(size, method=method)],\n            [lambda o: 0, Categorize()]]\n    dsets = Datasets(test_files, tfms=tfms, splits=RandomSplitter(0.1)(test_files))\n    batch_tfms = [Normalize.from_stats(*imagenet_stats)]\n    dls = dsets.dataloaders(bs=bs, after_batch=batch_tfms)\n    return dls.test_dl(test_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def return_valid_files(dcm_files):\n    valid_files = []\n    for o in progress_bar(dcm_files):\n        try:\n            o.dcmread().pixel_array\n            valid_files.append(o)\n        except:\n            pass\n    return valid_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"do_full = True\nn = 100\nsubmit_full = True\n\nif Path('../input/rsna-str-pulmonary-embolism-detection/train').exists() and not do_full:\n    test_dl = get_testdl(return_valid_files(test_dcm_files[:n]), size=256, method='squish', bs=32)\nelse:\n    if submit_full:\n        test_dl = get_testdl(return_valid_files(test_dcm_files), size=256, method='squish', bs=32)\n    else:\n        test_dl = get_testdl(return_valid_files(test_dcm_files[:n]), size=256, method='squish', bs=32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inference 1/3 public test data ~40 mins"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds, targs = learn.get_preds(dl=test_dl)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate preds"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds[:,1].min(), preds[:,1].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_ids = [o.parent.parent.stem for o in test_dl.items]\ninstance_ids = [o.stem for o in test_dl.items]\npreds_df = pd.DataFrame({\"StudyInstanceUID\":study_ids, \"SOPInstanceUID\":instance_ids})\npreds_df['pe_present_on_image'] = torch.clamp(preds[:,1], 0.001, 0.999).numpy().astype(np.float64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df['pe_present_on_image'].min(), preds_df['pe_present_on_image'].max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert not preds_df.isna().sum().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mean_pe = 0.2799\nmean_labels = {\n             'negative_exam_for_pe': 0.6763928618101033,\n             'rv_lv_ratio_gte_1': 0.12875001256566257,\n             'rv_lv_ratio_lt_1': 0.17437230326919448,\n             'leftsided_pe': 0.21089872969528548,\n             'chronic_pe': 0.040139752506710064,\n             'rightsided_pe': 0.2575653665766779,\n             'acute_and_chronic_pe': 0.019458347341720122,\n             'central_pe': 0.054468517151291695,\n             'indeterminate': 0.020484822355039723}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_max_pe = (preds_df.groupby(\"StudyInstanceUID\")['pe_present_on_image'].agg([\"max\"]).reset_index())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"study_max_pe.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# max image pe proba for predicted exams\nstudy_max_dict = dict(zip(study_max_pe['StudyInstanceUID'], study_max_pe['max']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract all exam and image ids to predict for\ntest_unique_sids = test_df['StudyInstanceUID'].unique()\ntest_unique_sopids = defaultdict(list)\nfor _,row in test_df.iterrows():\n    sid = row['StudyInstanceUID']\n    sopid = row['SOPInstanceUID']\n    test_unique_sopids[sid].append(sopid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prediction dict for each exam - assuming SOPInstanceUID is unique for a given dcm file across all data\npe_image_preds_dict = dict(zip(preds_df['SOPInstanceUID'], preds_df['pe_present_on_image']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate Submission df"},{"metadata":{"trusted":true},"cell_type":"code","source":"res = []\nfor sid in test_unique_sids:\n    \n    sopids = test_unique_sopids[sid]\n    \n    if sid not in study_max_dict:\n        for l in mean_labels: res.append((sid+\"_\"+l, mean_labels[l]))\n        for sopid in sopids: res.append((sopid, mean_pe))\n        \n    else:\n        max_pe = study_max_dict[sid]\n        \n        if max_pe > 0.5:\n            res.append((f\"{sid}_negative_exam_for_pe\", 1 - mean_labels['negative_exam_for_pe'])) # <=0.5\n            res.append((f\"{sid}_indeterminate\", mean_labels['indeterminate'])) # <=0.5\n            \n            res.append((f\"{sid}_leftsided_pe\", 1 - mean_labels['leftsided_pe'])) # >0.5\n            res.append((f\"{sid}_central_pe\", 1 - mean_labels['central_pe'])) # and/or >0.5\n            res.append((f\"{sid}_rightsided_pe\", 1 - mean_labels['rightsided_pe'])) # and/or >0.5\n            \n            res.append((f\"{sid}_rv_lv_ratio_gte_1\", 1 - mean_labels['rv_lv_ratio_gte_1'])) # >0.5\n            res.append((f\"{sid}_rv_lv_ratio_lt_1\", mean_labels['rv_lv_ratio_lt_1'])) # or >0.5\n            \n            res.append((f\"{sid}_chronic_pe\", mean_labels['chronic_pe'])) # <=0.5 if other >0.5\n            res.append((f\"{sid}_acute_and_chronic_pe\", mean_labels['acute_and_chronic_pe'])) # <=0.5 if other >0.5\n            \n            for sopid in sopids:\n                if sopid in pe_image_preds_dict: res.append((sopid, pe_image_preds_dict[sopid]))\n                else:                            res.append((sopid, mean_pe))\n            \n            \n        else:\n            res.append((f\"{sid}_negative_exam_for_pe\", mean_labels['negative_exam_for_pe'])) # >0.5\n            res.append((f\"{sid}_indeterminate\", mean_labels['indeterminate'])) # or >0.5\n            \n            res.append((f\"{sid}_leftsided_pe\", mean_labels['leftsided_pe'])) # <=0.5\n            res.append((f\"{sid}_central_pe\", mean_labels['central_pe'])) # and <=0.5\n            res.append((f\"{sid}_rightsided_pe\", mean_labels['rightsided_pe'])) # and <=0.5\n            \n            res.append((f\"{sid}_rv_lv_ratio_gte_1\", mean_labels['rv_lv_ratio_gte_1'])) # <=0.5\n            res.append((f\"{sid}_rv_lv_ratio_lt_1\", mean_labels['rv_lv_ratio_lt_1'])) # or <=0.5\n            \n            res.append((f\"{sid}_chronic_pe\", mean_labels['chronic_pe'])) # <=0.5 if other >0.5\n            res.append((f\"{sid}_acute_and_chronic_pe\", mean_labels['acute_and_chronic_pe'])) # <=0.5 if other >0.5\n            \n            for sopid in sopids:\n                if sopid in pe_image_preds_dict: res.append((sopid, pe_image_preds_dict[sopid]))\n                else:                            res.append((sopid, mean_pe))\n        \n        \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_sub_df = pd.DataFrame(res, columns=['id', 'label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(set(sub_df.index).intersection(set(new_sub_df.index))) == len(sub_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_sub_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check Consistency"},{"metadata":{"trusted":true},"cell_type":"code","source":"def check_consistency(sub, test):\n    \n    '''\n    Checks label consistency and returns the errors\n    \n    Args:\n    sub   = submission dataframe (pandas)\n    test  = test.csv dataframe (pandas)\n    '''\n    \n    # EXAM LEVEL\n    for i in test['StudyInstanceUID'].unique():\n        df_tmp = sub.loc[sub.id.str.contains(i, regex = False)].reset_index(drop = True)\n        df_tmp['StudyInstanceUID'] = df_tmp['id'].str.split('_').str[0]\n        df_tmp['label_type']       = df_tmp['id'].str.split('_').str[1:].apply(lambda x: '_'.join(x))\n        del df_tmp['id']\n        if i == test['StudyInstanceUID'].unique()[0]:\n            df = df_tmp.copy()\n        else:\n            df = pd.concat([df, df_tmp], axis = 0)\n    df_exam = df.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')\n    \n    # IMAGE LEVEL\n    df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)\n    df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')\n    df_image.rename(columns = {\"label\": \"pe_present_on_image\"}, inplace = True)\n    del df_image['id']\n    \n    # MERGER\n    df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')\n    ids    = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']\n    labels = [c for c in df.columns if c not in ids]\n    df = df[ids + labels]\n    \n    # SPLIT NEGATIVE AND POSITIVE EXAMS\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n    \n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    \n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule2a, rule2b], axis = 0)\n    \n    # OUTPUT\n    print('Found', len(errors), 'inconsistent predictions')\n    return errors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = check_consistency(new_sub_df, test_df)\nif len(res) == 0:\n    new_sub_df.to_csv(\"submission.csv\", index=False)\nelse:\n    raise(\"not valid submission\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}