{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Importing Libs","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nimport gc\nimport time\nfrom IPython.display import clear_output\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.callbacks import ModelCheckpoint as MC\nfrom tensorflow.keras import backend as K\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Dropout, Conv2D, LeakyReLU","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-02T05:13:26.202351Z","iopub.execute_input":"2021-07-02T05:13:26.20269Z","iopub.status.idle":"2021-07-02T05:13:26.209858Z","shell.execute_reply.started":"2021-07-02T05:13:26.202661Z","shell.execute_reply":"2021-07-02T05:13:26.208755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Viewing Data","metadata":{}},{"cell_type":"code","source":"root = '/kaggle/input/rsna-str-pulmonary-embolism-detection'\nfor item in os.listdir(root):\n    path = os.path.join(root, item)\n    if os.path.isfile(path):\n        print(path)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-02T04:41:15.637215Z","iopub.execute_input":"2021-07-02T04:41:15.63753Z","iopub.status.idle":"2021-07-02T04:41:15.651839Z","shell.execute_reply.started":"2021-07-02T04:41:15.637494Z","shell.execute_reply":"2021-07-02T04:41:15.651042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading Data\n","metadata":{}},{"cell_type":"code","source":"print('Reading train data...')\ntrain = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\nprint(train.shape)\ntrain.head()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-07-02T04:41:28.198778Z","iopub.execute_input":"2021-07-02T04:41:28.199116Z","iopub.status.idle":"2021-07-02T04:41:31.319487Z","shell.execute_reply.started":"2021-07-02T04:41:28.199087Z","shell.execute_reply":"2021-07-02T04:41:31.31885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading test data...')\ntest = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")\nprint(test.shape)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:41:31.321186Z","iopub.execute_input":"2021-07-02T04:41:31.32152Z","iopub.status.idle":"2021-07-02T04:41:31.47845Z","shell.execute_reply.started":"2021-07-02T04:41:31.321483Z","shell.execute_reply":"2021-07-02T04:41:31.47778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Reading sample data...')\nss = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/sample_submission.csv\")\nprint(ss.shape)\nss.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:41:31.479762Z","iopub.execute_input":"2021-07-02T04:41:31.480135Z","iopub.status.idle":"2021-07-02T04:41:31.621912Z","shell.execute_reply.started":"2021-07-02T04:41:31.480097Z","shell.execute_reply":"2021-07-02T04:41:31.621054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking Targets and Input Image\n\nTo make sure that we will be making the correct training targets, we check all ID from the sample submission.","metadata":{}},{"cell_type":"code","source":"ids = ss.id\ncounter = [1 for _ in range(10)]\nmapper = []\nfor i in ids:\n    n = '_'.join(i.split('_')[1:])\n    if n not in mapper:\n        mapper.append(n)\n    else:\n        counter[mapper.index(n)] += 1\nprint(\"List of keys:\")\nprint(mapper, sep='\\n')\nprint()\nprint(\"Count of items per key:\")\nprint(counter)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:42:07.16402Z","iopub.execute_input":"2021-07-02T04:42:07.164351Z","iopub.status.idle":"2021-07-02T04:42:07.334928Z","shell.execute_reply.started":"2021-07-02T04:42:07.16432Z","shell.execute_reply":"2021-07-02T04:42:07.333956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The following fuction will return an image array from a DICOM image [Snipet](https://www.kaggle.com/eladwar/20-seconds-or-less).","metadata":{}},{"cell_type":"code","source":"import vtk\nfrom vtk.util import numpy_support\nimport cv2\n\nreader = vtk.vtkDICOMImageReader()\ndef get_img(path):\n    reader.SetFileName(path)\n    reader.Update()\n    _extent = reader.GetDataExtent()\n    ConstPixelDims = [_extent[1]-_extent[0]+1, _extent[3]-_extent[2]+1, _extent[5]-_extent[4]+1]\n\n    ConstPixelSpacing = reader.GetPixelSpacing()\n    imageData = reader.GetOutput()\n    pointData = imageData.GetPointData()\n    arrayData = pointData.GetArray(0)\n    ArrayDicom = numpy_support.vtk_to_numpy(arrayData)\n    ArrayDicom = ArrayDicom.reshape(ConstPixelDims, order='F')\n    ArrayDicom = cv2.resize(ArrayDicom,(512,512))\n    return ArrayDicom","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:43:59.770128Z","iopub.execute_input":"2021-07-02T04:43:59.770456Z","iopub.status.idle":"2021-07-02T04:44:01.182373Z","shell.execute_reply.started":"2021-07-02T04:43:59.770425Z","shell.execute_reply":"2021-07-02T04:44:01.181615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Testing our Function","metadata":{}},{"cell_type":"code","source":"#test read a dcom file and view it\nfpath = \"../input/rsna-str-pulmonary-embolism-detection/train/0003b3d648eb/d2b2960c2bbf/00ac73cfc372.dcm\"\nds = get_img(fpath)\n\nimport matplotlib.pyplot as plt\n\n#Convert dcom file to 8bit color\nfunc = lambda x: int((2**15 + x)*(255/2**16))\nint16_to_uint8 = np.vectorize(func)\n\ndef show_dicom_images(dcom):\n    f, ax = plt.subplots(1,2, figsize=(16,20))\n    data_row_img = int16_to_uint8(ds)\n    ax[0].imshow(data_row_img, cmap=plt.cm.bone)\n    ax[1].imshow(ds, cmap=plt.cm.bone)\n    #print(data_row_img)\n    ax[0].axis('off')\n    ax[0].set_title('8-bit DICOM Image')\n    ax[1].axis('off')\n    ax[1].set_title('16-bit DICOM Image')\n    plt.show()\n    \nshow_dicom_images(ds)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:44:33.749161Z","iopub.execute_input":"2021-07-02T04:44:33.749475Z","iopub.status.idle":"2021-07-02T04:44:34.171822Z","shell.execute_reply.started":"2021-07-02T04:44:33.749445Z","shell.execute_reply":"2021-07-02T04:44:34.170969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining Model Architecture ","metadata":{}},{"cell_type":"markdown","source":"Here I am using the first couple of layers from the famous pre-trained Xception model which will act like a feature-extractor and the rest of the layers will be custom trained to our dataset.","metadata":{}},{"cell_type":"code","source":"inputs = Input((512, 512, 3))\n#x = Conv2D(3, (1, 1), activation='relu')(inputs)\nbase_model = keras.applications.Xception(\n    include_top=False,\n    weights=\"imagenet\"\n)\n\nbase_model.trainable = False","metadata":{"execution":{"iopub.status.busy":"2021-07-02T04:49:17.402373Z","iopub.execute_input":"2021-07-02T04:49:17.402743Z","iopub.status.idle":"2021-07-02T04:49:23.034911Z","shell.execute_reply.started":"2021-07-02T04:49:17.40271Z","shell.execute_reply":"2021-07-02T04:49:23.034133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### The outer layers ","metadata":{}},{"cell_type":"code","source":"outputs = base_model(inputs, training=False)\noutputs = keras.layers.GlobalAveragePooling2D()(outputs)\noutputs = Dropout(0.25)(outputs)\noutputs = Dense(1024, kernel_initializer='uniform')(outputs)\noutputs = LeakyReLU(alpha=0.3)(outputs)\noutputs = Dense(256)(outputs)\noutputs = LeakyReLU(alpha=0.1)(outputs)\noutputs = Dense(64, activation='relu')(outputs)\nppoi = Dense(1, activation='sigmoid', name='pe_present_on_image')(outputs)\nrlrg1 = Dense(1, activation='sigmoid', name='rv_lv_ratio_gte_1')(outputs)\nrlrl1 = Dense(1, activation='sigmoid', name='rv_lv_ratio_lt_1')(outputs) \nlspe = Dense(1, activation='sigmoid', name='leftsided_pe')(outputs)\ncpe = Dense(1, activation='sigmoid', name='chronic_pe')(outputs)\nrspe = Dense(1, activation='sigmoid', name='rightsided_pe')(outputs)\naacpe = Dense(1, activation='sigmoid', name='acute_and_chronic_pe')(outputs)\ncnpe = Dense(1, activation='sigmoid', name='central_pe')(outputs)\nindt = Dense(1, activation='sigmoid', name='indeterminate')(outputs)\n\nmodel = Model(inputs=inputs, outputs={'pe_present_on_image':ppoi,\n                                      'rv_lv_ratio_gte_1':rlrg1,\n                                      'rv_lv_ratio_lt_1':rlrl1,\n                                      'leftsided_pe':lspe,\n                                      'chronic_pe':cpe,\n                                      'rightsided_pe':rspe,\n                                      'acute_and_chronic_pe':aacpe,\n                                      'central_pe':cnpe,\n                                      'indeterminate':indt})\n\nopt = keras.optimizers.Adam(lr=0.001)\n\nmodel.compile(optimizer=opt,\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()\nmodel.save('pe_detection_model.h5')\ndel model\nK.clear_session()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:14:00.019213Z","iopub.execute_input":"2021-07-02T05:14:00.01952Z","iopub.status.idle":"2021-07-02T05:14:01.279315Z","shell.execute_reply.started":"2021-07-02T05:14:00.019491Z","shell.execute_reply":"2021-07-02T05:14:01.278278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining Callbacks","metadata":{}},{"cell_type":"code","source":"my_callbacks = [\n    tf.keras.callbacks.EarlyStopping(patience=2),\n    tf.keras.callbacks.ModelCheckpoint(filepath='model.{epoch:02d}-{val_loss:.2f}.h5'),\n    tf.keras.callbacks.TensorBoard(log_dir='./logs'),\n    tf.keras.callbacks.ModelCheckpoint(filepath='../working/pe_detection_model.h5', monitor='val_loss', save_best_only=True, save_freq='epoch',verbose=1)\n]","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:06:25.18425Z","iopub.execute_input":"2021-07-02T05:06:25.184565Z","iopub.status.idle":"2021-07-02T05:06:25.191944Z","shell.execute_reply.started":"2021-07-02T05:06:25.184536Z","shell.execute_reply":"2021-07-02T05:06:25.190913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image Genrator","metadata":{}},{"cell_type":"code","source":"def convert_to_rgb(array):\n    array = array.reshape((512, 512, 1))\n    return np.stack([array, array, array], axis=2).reshape((512, 512, 3))\n    \ndef custom_dcom_image_generator(batch_size, dataset, test=False, debug=False):\n    \n    fnames = dataset[['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']]\n    \n    if not test:\n        Y = dataset[['pe_present_on_image', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', 'leftsided_pe',\n                     'chronic_pe', 'rightsided_pe', 'acute_and_chronic_pe', 'central_pe', 'indeterminate'\n                    ]]\n        prefix = 'input/rsna-str-pulmonary-embolism-detection/train'\n        \n    else:\n        prefix = 'input/rsna-str-pulmonary-embolism-detection/test'\n    \n    X = []\n    batch = 0\n    for st, sr, so in fnames.values:\n        if debug:\n            print(f\"Current file: ../{prefix}/{st}/{sr}/{so}.dcm\")\n\n        dicom = get_img(f\"../{prefix}/{st}/{sr}/{so}.dcm\")\n        image = convert_to_rgb(dicom)\n        X.append(image)\n        \n        del st, sr, so\n        \n        if len(X) == batch_size:\n            if test:\n                yield np.array(X)\n                del X\n            else:\n                yield np.array(X), Y[batch*batch_size:(batch+1)*batch_size].values\n                del X\n                \n            gc.collect()\n            X = []\n            batch += 1\n        \n    if test:\n        yield np.array(X)\n    else:\n        yield np.array(X), Y[batch*batch_size:(batch+1)*batch_size].values\n        del Y\n    del X\n    gc.collect()\n    return","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:06:54.130174Z","iopub.execute_input":"2021-07-02T05:06:54.13048Z","iopub.status.idle":"2021-07-02T05:06:54.142905Z","shell.execute_reply.started":"2021-07-02T05:06:54.130452Z","shell.execute_reply":"2021-07-02T05:06:54.142077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training proces","metadata":{}},{"cell_type":"code","source":"history = {}\nstart = time.time()\ndebug = 0\nbatch_size = 1000\ntrain_size = int(batch_size*0.9)\n\nmax_train_time = 3600 * 4 #hours to seconds of training\n\n\nfor n, (x, y) in enumerate(custom_dcom_image_generator(batch_size, train.sample(frac=1), False, debug)):\n    \n    if len(x) < 10: #Tries to filter out empty or short data\n        break\n        \n    clear_output(wait=True)\n    print(\"Training batch: %i - %i\" %(batch_size*n, batch_size*(n+1)))\n    \n    model = load_model('../working/pe_detection_model.h5')\n    hist = model.fit(\n        x[:train_size], #Y values are in a dict as there's more than one target for training output\n        {'pe_present_on_image':y[:train_size, 0],\n         'rv_lv_ratio_gte_1':y[:train_size, 1],\n         'rv_lv_ratio_lt_1':y[:train_size, 2],\n         'leftsided_pe':y[:train_size, 3],\n         'chronic_pe':y[:train_size, 4],\n         'rightsided_pe':y[:train_size, 5],\n         'acute_and_chronic_pe':y[:train_size, 6],\n         'central_pe':y[:train_size, 7],\n         'indeterminate':y[:train_size, 8]},\n\n        callbacks = my_callbacks,\n\n        validation_split=0.2,\n        epochs=3,\n        batch_size=8,\n        verbose=debug\n    )\n    \n    print(\"Metrics for batch validation:\")\n    model.evaluate(x[train_size:],\n                   {'pe_present_on_image':y[train_size:, 0],\n                    'rv_lv_ratio_gte_1':y[train_size:, 1],\n                    'rv_lv_ratio_lt_1':y[train_size:, 2],\n                    'leftsided_pe':y[train_size:, 3],\n                    'chronic_pe':y[train_size:, 4],\n                    'rightsided_pe':y[train_size:, 5],\n                    'acute_and_chronic_pe':y[train_size:, 6],\n                    'central_pe':y[train_size:, 7],\n                    'indeterminate':y[train_size:, 8]\n                   }\n                  )\n    \n    try:\n        for key in hist.history.keys():\n            history[key] = np.concatenate([history[key], hist.history[key]], axis=0)\n    except:\n        for key in hist.history.keys():\n            history[key] = hist.history[key]\n            \n    if time.time() - start >= max_train_time:\n        print(\"Time's up!\")\n        break\n        \n    model.save('pe_detection_model.h5')\n    del model, x, y, hist\n    K.clear_session()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:14:06.223472Z","iopub.execute_input":"2021-07-02T05:14:06.223783Z","iopub.status.idle":"2021-07-02T05:51:03.282916Z","shell.execute_reply.started":"2021-07-02T05:14:06.223753Z","shell.execute_reply":"2021-07-02T05:51:03.280603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in history.keys():\n    if key.startswith('val'):\n        continue\n    else:\n        epoch = range(len(history[key]))\n        plt.plot(epoch, history[key]) #X=epoch, Y=value\n        plt.plot(epoch, history['val_'+key])\n        plt.title(key)\n        if 'accuracy' in key:\n            plt.axis([0, len(history[key]), -0.1, 1.1]) #Xmin, Xmax, Ymin, Ymax\n        plt.legend(['train', 'validation'], loc='upper right')\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:52:34.859875Z","iopub.execute_input":"2021-07-02T05:52:34.860216Z","iopub.status.idle":"2021-07-02T05:52:37.205901Z","shell.execute_reply.started":"2021-07-02T05:52:34.860186Z","shell.execute_reply":"2021-07-02T05:52:37.204721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Viewing Predictions\n","metadata":{}},{"cell_type":"code","source":"predictions = {}\nstopper = 3600 * 4 #4 hours limit for prediction\npred_start_time = time.time()\n\np, c = time.time(), time.time()\nbatch_size = 500\n    \nl = 0\nn = test.shape[0]\n\nfor x in custom_dcom_image_generator(batch_size, test, True, False):\n    clear_output(wait=True)\n    model = load_model(\"../working/pe_detection_model.h5\")\n    preds = model.predict(x, batch_size=8, verbose=1)\n    \n    try:\n        for key in preds.keys():\n            predictions[key] += preds[key].flatten().tolist()\n            \n    except Exception as e:\n        print(e)\n        for key in preds.keys():\n            predictions[key] = preds[key].flatten().tolist()\n            \n    l = (l+batch_size)%n\n    print('Total predicted:', len(predictions['indeterminate']),'/', n)\n    p, c = c, time.time()\n    print(\"One batch time: %.2f seconds\" %(c-p))\n    print(\"ETA: %.2f\" %((n-l)*(c-p)/batch_size))\n    \n    if c - pred_start_time >= stopper:\n        print(\"Time's up!\")\n        break\n    \n    del model\n    K.clear_session()\n    \n    del x, preds\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:54:08.204936Z","iopub.execute_input":"2021-07-02T05:54:08.205254Z","iopub.status.idle":"2021-07-02T05:57:17.779659Z","shell.execute_reply.started":"2021-07-02T05:54:08.205225Z","shell.execute_reply":"2021-07-02T05:57:17.77783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for key in predictions.keys():\n    print(key, np.array(predictions[key]).shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:57:21.041527Z","iopub.execute_input":"2021-07-02T05:57:21.04186Z","iopub.status.idle":"2021-07-02T05:57:21.053907Z","shell.execute_reply.started":"2021-07-02T05:57:21.041823Z","shell.execute_reply":"2021-07-02T05:57:21.053086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_ids = []\nfor v in test.StudyInstanceUID:\n    if v not in test_ids:\n        test_ids.append(v)\n        \ntest_preds = test.copy()\ntest_preds = pd.concat([test_preds, pd.DataFrame(predictions)], axis=1)\ntest_preds.to_csv('test_predictions.csv', index=False)\ntest_preds","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:57:21.491962Z","iopub.execute_input":"2021-07-02T05:57:21.492273Z","iopub.status.idle":"2021-07-02T05:57:24.047176Z","shell.execute_reply.started":"2021-07-02T05:57:21.492247Z","shell.execute_reply":"2021-07-02T05:57:24.046445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.special import softmax\n\nlabel_agg = {key:[] for key in \n             ['id', 'negative_exam_for_pe', 'rv_lv_ratio_gte_1',\n              'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe',\n              'rightsided_pe', 'acute_and_chronic_pe',\n              'central_pe', 'indeterminate']\n            }\n\nfor uid in test_ids:\n    temp = test_preds.loc[test_preds.StudyInstanceUID ==uid]\n    label_agg['id'].append(uid)\n    \n    n = temp.shape[0]\n    #Check for any image level presence of PE of high confidence\n    positive = any(temp.pe_present_on_image >= 0.5) #50% threshhold\n    \n    #Only one from positive, negative and indeterminate should have value>0.5\n    #per exam\n    if positive: \n        label_agg['indeterminate'].append(temp.indeterminate.min()/2)\n        label_agg['negative_exam_for_pe'].append(0)\n    else:\n        if any(temp.indeterminate >= 0.5):\n            label_agg['indeterminate'].append(temp.indeterminate.max())\n            label_agg['negative_exam_for_pe'].append(1)\n        else:\n            label_agg['indeterminate'].append(temp.indeterminate.min()/2)\n            label_agg['negative_exam_for_pe'].append(1)\n    \n    #I decided that the total ratio should be equal to 1, so I used softmax\n    #We modify the weights by multiplying the bigger by 2 and dividing the smaller by 2\n    a, b = temp[['rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1']].mean().values\n    if a > b:\n        a, b = a*2, b/2\n    elif a < b:\n        a, b = a/2, b*2\n    a, b = softmax([a, b])\n    if positive:\n        label_agg['rv_lv_ratio_gte_1'].append(a)\n        label_agg['rv_lv_ratio_lt_1'].append(b)\n    else:\n        label_agg['rv_lv_ratio_gte_1'].append(a/2)\n        label_agg['rv_lv_ratio_lt_1'].append(b/2)\n    \n    #Next is for Chronic (C), Acute-Chronic (AC) and Acute (A) PE\n    #We need to see if we got a high confidence value from either C or AC\n    #If there is, we add it to a 50% based score for high confidence\n    #and half weight for low confidence score\n    if any(temp['acute_and_chronic_pe'] > 0.5): #50% confidence level\n        label_agg['acute_and_chronic_pe'].append(0.5 + temp['acute_and_chronic_pe'].mean()/2)\n        label_agg['chronic_pe'].append(temp['chronic_pe'].mean()/2)\n        \n    elif any(temp['chronic_pe'] > 0.5):\n        label_agg['acute_and_chronic_pe'].append(temp['acute_and_chronic_pe'].mean()/2)\n        label_agg['chronic_pe'].append(0.5 + temp['chronic_pe'].mean()/2)\n        \n    else: #Else, we set both to half values, as we declare the A as the value\n        label_agg['acute_and_chronic_pe'].append(temp['acute_and_chronic_pe'].mean()/2)\n        label_agg['chronic_pe'].append(temp['chronic_pe'].mean()/2)\n    \n    #for right, left, central, we use the same metric above\n    for key in ['leftsided_pe', 'rightsided_pe', 'central_pe']:\n        if positive:\n            label_agg[key].append(0.5 + temp[key].mean()/2)\n        else:\n            label_agg[key].append(temp[key].mean()/2)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:57:37.239884Z","iopub.execute_input":"2021-07-02T05:57:37.240265Z","iopub.status.idle":"2021-07-02T05:57:38.72909Z","shell.execute_reply.started":"2021-07-02T05:57:37.240222Z","shell.execute_reply":"2021-07-02T05:57:38.727507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uid = []\nlabels = []\ndf = pd.DataFrame(label_agg)\nfor key in ['negative_exam_for_pe', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe',\n            'rightsided_pe', 'acute_and_chronic_pe', 'central_pe', 'indeterminate']:\n    for i in df.id:\n        uid.append('_'.join([i, key]))\n        labels.append(df.loc[df.id==i][key].values[0])\ndel df\ngc.collect()\n\nuid += test_preds.SOPInstanceUID.tolist()\nlabels += test_preds['pe_present_on_image'].tolist()\n\nsub = pd.DataFrame({\"id\":uid, 'label':labels})\nsub","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:57:38.730621Z","iopub.status.idle":"2021-07-02T05:57:38.731358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally, we will save our submission into a file. But first, to make sure that we fill up any unpredicted variables (so that there will be no NaN in our values), we will fill up all missing values with 0.2 as a placeholder.","metadata":{}},{"cell_type":"code","source":"sub.fillna(0.2, inplace=True)\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T05:57:17.789051Z","iopub.status.idle":"2021-07-02T05:57:17.78976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Viewing one pred","metadata":{}},{"cell_type":"code","source":"batch_size = 1\n    \nl = 0\nn = test.shape[0]\n\nfor x in custom_dcom_image_generator(batch_size, test, True, False):\n    clear_output(wait=True)\n    model = load_model(\"../working/pe_detection_model.h5\")\n    preds = model.predict(x, batch_size=1, verbose=1)\n    print(preds)\n","metadata":{"execution":{"iopub.status.busy":"2021-07-02T06:00:28.541478Z","iopub.execute_input":"2021-07-02T06:00:28.541825Z","iopub.status.idle":"2021-07-02T06:00:31.051883Z","shell.execute_reply.started":"2021-07-02T06:00:28.541782Z","shell.execute_reply":"2021-07-02T06:00:31.050851Z"},"trusted":true},"execution_count":null,"outputs":[]}]}