{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nfrom PIL import Image\n\n# For data manipulation\nimport numpy as np\nimport pandas as pd\n\n# Pytorch Imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda import amp\n\n# Utils\nimport joblib\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\n# Sklearn Imports\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold, KFold\n\nimport time\n\n# Albumentations for augmentations\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# For colored terminal text\nfrom colorama import Fore, Back, Style\nc_ = Fore.CYAN\nsr_ = Style.RESET_ALL\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# For descriptive error messages\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-12T15:32:54.982668Z","iopub.execute_input":"2021-12-12T15:32:54.983219Z","iopub.status.idle":"2021-12-12T15:32:58.839697Z","shell.execute_reply.started":"2021-12-12T15:32:54.983094Z","shell.execute_reply":"2021-12-12T15:32:58.838802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nimport glob, warnings\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix, classification_report\nimport seaborn as sns\n\nwarnings.filterwarnings('ignore')\nprint('TensorFlow Version ' + tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:32:58.844872Z","iopub.execute_input":"2021-12-12T15:32:58.845684Z","iopub.status.idle":"2021-12-12T15:33:03.853483Z","shell.execute_reply.started":"2021-12-12T15:32:58.845646Z","shell.execute_reply":"2021-12-12T15:33:03.852723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### Setting some important libraries for throughout use \n##########################################################\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n######\nimport os\nimport os.path\nfrom pathlib import Path\n#import pydicom\nimport glob\n######\nfrom PIL import Image\nfrom tensorflow.keras.preprocessing import image\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n######\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler\nfrom tensorflow.keras.utils import to_categorical\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras import regularizers\n######\nfrom sklearn.metrics import confusion_matrix, accuracy_score, classification_report, roc_auc_score, roc_curve\n######\nfrom tensorflow.keras.optimizers import RMSprop,Adam,Optimizer\n######\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization,MaxPooling2D\nfrom tensorflow.keras.layers import BatchNormalization,Permute, TimeDistributed, Bidirectional,GRU, SimpleRNN, LSTM, GlobalAveragePooling2D\nfrom tensorflow.keras import models\nfrom tensorflow.keras import layers\nimport tensorflow as tf\nfrom tensorflow.keras.applications import resnet50\n\n######\nfrom warnings import filterwarnings\n\nfilterwarnings(\"ignore\",category=DeprecationWarning)\nfilterwarnings(\"ignore\", category=FutureWarning) \nfilterwarnings(\"ignore\", category=UserWarning)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:03.85485Z","iopub.execute_input":"2021-12-12T15:33:03.855297Z","iopub.status.idle":"2021-12-12T15:33:03.871745Z","shell.execute_reply.started":"2021-12-12T15:33:03.85526Z","shell.execute_reply":"2021-12-12T15:33:03.871088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### Setting the path and Reading the data\n###################################################\ntrain = pd.read_csv(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv\")\nsub = pd.read_csv(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_sample_submission.csv\")\ntrain_images = os.listdir(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/\")\ntest_images = os.listdir(\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/\")\nprint ('Train:', train.shape[0])\nprint ('Sub:', sub.shape[0])","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:03.874202Z","iopub.execute_input":"2021-12-12T15:33:03.874695Z","iopub.status.idle":"2021-12-12T15:33:32.397644Z","shell.execute_reply.started":"2021-12-12T15:33:03.874659Z","shell.execute_reply":"2021-12-12T15:33:32.396747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### Getting training and testing data\n###################################################\ntrain['type'] = train['ID'].str.split(\"_\", n = 3, expand = True)[2]\ntrain['PatientID'] = train['ID'].str.split(\"_\", n = 3, expand = True)[1]\ntrain['filename'] = train['ID'].apply(lambda st: \"ID_\" + st.split('_')[1] + \".png\")\n# Remove invalid PNGs\ntrain=train.head(150000)\n\n\nsub['filename'] = sub['ID'].apply(lambda st: \"ID_\" + st.split('_')[1] + \".png\")\nsub['type'] = sub['ID'].apply(lambda st: st.split('_')[2])\nsub=sub.head(150000)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:32.400148Z","iopub.execute_input":"2021-12-12T15:33:32.400958Z","iopub.status.idle":"2021-12-12T15:33:59.846767Z","shell.execute_reply.started":"2021-12-12T15:33:32.400919Z","shell.execute_reply":"2021-12-12T15:33:59.845964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### Understanding the unique types of brain hemmorhage\n###################################################\nprint ('Train type =', list(train.type.unique()))\nprint ('Train label =', list(train.Label.unique()))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:59.847864Z","iopub.execute_input":"2021-12-12T15:33:59.848135Z","iopub.status.idle":"2021-12-12T15:33:59.879452Z","shell.execute_reply.started":"2021-12-12T15:33:59.848099Z","shell.execute_reply":"2021-12-12T15:33:59.878722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### No of unique type of patients for each of the category\n##########################################################\nprint ('Number of Patients: ', train.PatientID.nunique())","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:59.880461Z","iopub.execute_input":"2021-12-12T15:33:59.880691Z","iopub.status.idle":"2021-12-12T15:33:59.919559Z","shell.execute_reply.started":"2021-12-12T15:33:59.880657Z","shell.execute_reply":"2021-12-12T15:33:59.918736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################################################\n#### No of unique type of patients for each of the category\n##########################################################\ntrain.type.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:59.920937Z","iopub.execute_input":"2021-12-12T15:33:59.921216Z","iopub.status.idle":"2021-12-12T15:33:59.963627Z","shell.execute_reply.started":"2021-12-12T15:33:59.92118Z","shell.execute_reply":"2021-12-12T15:33:59.96286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################\n#### Images with label of 0 and 1 such as hammerhoage or not\n#############################################################\nprint(train.Label.value_counts())\nsns.countplot(x='Label', data=train)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:33:59.965687Z","iopub.execute_input":"2021-12-12T15:33:59.967203Z","iopub.status.idle":"2021-12-12T15:34:00.18551Z","shell.execute_reply.started":"2021-12-12T15:33:59.967159Z","shell.execute_reply":"2021-12-12T15:34:00.184848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Understanding the no of unique type of patients for each of the category in 0 and 1\n######################################################################################\ntrain.groupby('type').Label.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:00.188391Z","iopub.execute_input":"2021-12-12T15:34:00.188596Z","iopub.status.idle":"2021-12-12T15:34:00.23381Z","shell.execute_reply.started":"2021-12-12T15:34:00.188572Z","shell.execute_reply":"2021-12-12T15:34:00.233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Understanding the no of unique type of patients for each of the category in 0 and 1\n#### in terms of barplot\n#####################################################################################\n\nsns.countplot(x=\"Label\", hue=\"type\", data=train)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:00.235374Z","iopub.execute_input":"2021-12-12T15:34:00.235651Z","iopub.status.idle":"2021-12-12T15:34:00.690418Z","shell.execute_reply.started":"2021-12-12T15:34:00.235617Z","shell.execute_reply":"2021-12-12T15:34:00.689729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Understanding the no of unique type of patients for each of the category in 0 and 1\n#### in terms of Pie-plot\n######################################################################################\ntrain.type.value_counts().plot.pie(figsize=(6,6))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:00.691498Z","iopub.execute_input":"2021-12-12T15:34:00.69221Z","iopub.status.idle":"2021-12-12T15:34:00.826181Z","shell.execute_reply.started":"2021-12-12T15:34:00.692173Z","shell.execute_reply":"2021-12-12T15:34:00.825508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Here, we are setting the image' windows (3types), ROI and skull removal  \n######################################################################################\nTRAIN_IMG_PATH = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/\"\nTEST_IMG_PATH = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test/\"\nBASE_PATH = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'\nTRAIN_DIR = 'stage_2_train/'\nTEST_DIR = 'stage_2_test/'\n\ndef window_image(img, window_center,window_width, intercept, slope, rescale=True):\n\n    img = (img*slope +intercept)\n    img_min = window_center - window_width//2\n    img_max = window_center + window_width//2\n    img[img<img_min] = img_min\n    img[img>img_max] = img_max\n    \n    if rescale:\n        # Extra rescaling to 0-1, not in the original notebook\n        img = (img - img_min) / (img_max - img_min)\n    \n    return img\n    \ndef get_first_of_dicom_field_as_int(x):\n    #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n    if type(x) == pydicom.multival.MultiValue:\n        return int(x[0])\n    else:\n        return int(x)\n\ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value, #window width\n                    data[('0028','1052')].value, #intercept\n                    data[('0028','1053')].value] #slope\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n\n    \n    \ndef view_images(images, title = '', aug = None):\n    width = 5\n    height = 2\n    fig, axs = plt.subplots(height, width, figsize=(15,5))\n    \n    for im in range(0, height * width):\n        data = pydicom.read_file(os.path.join(TRAIN_IMG_PATH,'ID_'+images[im]+ '.dcm'))\n        image = data.pixel_array\n        window_center , window_width, intercept, slope = get_windowing(data)\n        image_windowed = window_image(image, window_center, window_width, intercept, slope)\n\n\n        i = im // width\n        j = im % width\n        axs[i,j].imshow(image_windowed, cmap=plt.cm.bone) \n        axs[i,j].axis('off')\n        \n    plt.suptitle(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:00.827273Z","iopub.execute_input":"2021-12-12T15:34:00.82786Z","iopub.status.idle":"2021-12-12T15:34:00.842584Z","shell.execute_reply.started":"2021-12-12T15:34:00.827823Z","shell.execute_reply":"2021-12-12T15:34:00.841736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Demonstration of single image and its complete meta-data information\n######################################################################################\nimport pydicom\ncase = 8\ndata = pydicom.dcmread(TRAIN_IMG_PATH+train_images[case])\n\nprint(\"data\",data)\nwindow_center , window_width, intercept, slope = get_windowing(data)\n\n\n#displaying the image\nimg = pydicom.read_file(TRAIN_IMG_PATH+train_images[case]).pixel_array\n\nimg = window_image(img, window_center, window_width, intercept, slope)\nplt.imshow(img, cmap=plt.cm.bone)\nplt.grid(False)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:00.843774Z","iopub.execute_input":"2021-12-12T15:34:00.844703Z","iopub.status.idle":"2021-12-12T15:34:01.192934Z","shell.execute_reply.started":"2021-12-12T15:34:00.844661Z","shell.execute_reply":"2021-12-12T15:34:01.192232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Demonstration of some epidural images with label 1\n######################################################################################\n\nview_images(train[(train['type'] == 'epidural') & (train['Label'] == 1)][:10].PatientID.values, title = 'Images with epidural')","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:01.19393Z","iopub.execute_input":"2021-12-12T15:34:01.194175Z","iopub.status.idle":"2021-12-12T15:34:02.120419Z","shell.execute_reply.started":"2021-12-12T15:34:01.194146Z","shell.execute_reply":"2021-12-12T15:34:02.116533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Demonstration of some intraparenchymal images with label 1\n######################################################################################\nview_images(train[(train['type'] == 'intraparenchymal') & (train['Label'] == 1)][:10].PatientID.values, title = 'Images with intraparenchymal')","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:02.121973Z","iopub.execute_input":"2021-12-12T15:34:02.122467Z","iopub.status.idle":"2021-12-12T15:34:02.973748Z","shell.execute_reply.started":"2021-12-12T15:34:02.12243Z","shell.execute_reply":"2021-12-12T15:34:02.972944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Demonstration of some subarachnoid images with label 1\n######################################################################################\nview_images(train[(train['type'] == 'subarachnoid') & (train['Label'] == 1)][:10].PatientID.values, title = 'Images with subarachnoid')","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:02.975147Z","iopub.execute_input":"2021-12-12T15:34:02.975915Z","iopub.status.idle":"2021-12-12T15:34:03.799617Z","shell.execute_reply.started":"2021-12-12T15:34:02.975872Z","shell.execute_reply":"2021-12-12T15:34:03.798948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Demonstration of some subdural images with label 1\n######################################################################################\nview_images(train[(train['type'] == 'subdural') & (train['Label'] == 1)][:10].PatientID.values, title = 'Images with subdural')","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:03.800843Z","iopub.execute_input":"2021-12-12T15:34:03.801593Z","iopub.status.idle":"2021-12-12T15:34:04.585751Z","shell.execute_reply.started":"2021-12-12T15:34:03.801553Z","shell.execute_reply":"2021-12-12T15:34:04.585076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#######################################################################################\n#### Setting and demonstration of testing data\n######################################################################################\n\ntest = pd.DataFrame(sub.filename.unique(), columns=['filename'])\nprint ('Test:', test.shape[0])\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:04.586955Z","iopub.execute_input":"2021-12-12T15:34:04.587412Z","iopub.status.idle":"2021-12-12T15:34:04.619962Z","shell.execute_reply.started":"2021-12-12T15:34:04.58737Z","shell.execute_reply":"2021-12-12T15:34:04.619267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################################\n#### Getting some random images for training and testing to reduce the computational complexity\n########################################################################################################\nnp.random.seed(1234)\nsample_files = np.random.choice(os.listdir(TRAIN_IMG_PATH), 150000)\nsample_df = train[train.filename.apply(lambda x: x.replace('.png', '.dcm')).isin(sample_files)]\nprint(sample_df.shape)\n\npivot_df = sample_df[['Label', 'filename', 'type']].drop_duplicates().pivot(\n    index='filename', columns='type', values='Label').reset_index()\nprint(pivot_df.shape)\npivot_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:04.621295Z","iopub.execute_input":"2021-12-12T15:34:04.621719Z","iopub.status.idle":"2021-12-12T15:34:06.096581Z","shell.execute_reply.started":"2021-12-12T15:34:04.621682Z","shell.execute_reply":"2021-12-12T15:34:06.095823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################################\n#### Getting some random images for training and testing to reduce the computational complexity\n########################################################################################################\nnp.random.seed(1234)\nsample_files_test = np.random.choice(os.listdir(TEST_IMG_PATH), 150000)\nsample_files_test_df = sub[sub.filename.apply(lambda x: x.replace('.png', '.dcm')).isin(sample_files_test)]\nprint(sample_files_test_df.shape)\n\npivot_test_df = sample_files_test_df[['Label', 'filename', 'type']].drop_duplicates().pivot(\n    index='filename', columns='type', values='Label').reset_index()\nprint(pivot_test_df.shape)\npivot_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:06.098063Z","iopub.execute_input":"2021-12-12T15:34:06.098524Z","iopub.status.idle":"2021-12-12T15:34:06.491654Z","shell.execute_reply.started":"2021-12-12T15:34:06.098488Z","shell.execute_reply":"2021-12-12T15:34:06.490939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################################\n#### Getting and storing completly processed images (ROI, windowing, skull removal in directory)\n########################################################################################################\ndef save_and_resize(filenames, load_dir):    \n    save_dir = '/kaggle/tmp/'\n    if not os.path.exists(save_dir):\n        os.makedirs(save_dir)\n\n    for filename in tqdm(filenames):\n        path = load_dir + filename\n        new_path = save_dir + filename.replace('.dcm', '.png')\n        \n        dcm = pydicom.dcmread(path)\n        window_center , window_width, intercept, slope = get_windowing(dcm)\n        img = dcm.pixel_array\n        img = window_image(img, window_center, window_width, intercept, slope)\n        \n        resized = cv2.resize(img, (299, 299))\n        res = cv2.imwrite(new_path, resized)\n        if not res:\n            print('Failed')","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:06.493156Z","iopub.execute_input":"2021-12-12T15:34:06.49341Z","iopub.status.idle":"2021-12-12T15:34:06.501733Z","shell.execute_reply.started":"2021-12-12T15:34:06.493377Z","shell.execute_reply":"2021-12-12T15:34:06.500912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################################\n#### Getting and storing completly processed images (ROI, windowing, skull removal in directory)\n########################################################################################################\n\nfrom tqdm import tqdm\nimport json\nimport cv2\n\nsave_and_resize(filenames=sample_files, load_dir=BASE_PATH + TRAIN_DIR)\nsave_and_resize(filenames=sample_files_test, load_dir=BASE_PATH + TEST_DIR)\n#save_and_resize(filenames=os.listdir(BASE_PATH + TEST_DIR), load_dir=BASE_PATH + TEST_DIR)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T15:34:06.503436Z","iopub.execute_input":"2021-12-12T15:34:06.504022Z","iopub.status.idle":"2021-12-12T16:52:09.96795Z","shell.execute_reply.started":"2021-12-12T15:34:06.503949Z","shell.execute_reply":"2021-12-12T16:52:09.965301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#########################################################################################################\n#### Making batches of data and the apply augmentation on the data\n########################################################################################################\n\nBATCH_SIZE = 16\n\ndef create_datagen():\n    return ImageDataGenerator(\n        zoom_range=0.1,  # set range for random zoom\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images,\n        validation_split=0.25\n    )\n\ndef create_test_gen():\n    return ImageDataGenerator().flow_from_dataframe(\n        test,\n        directory='/kaggle/tmp/',\n        x_col='filename',\n        class_mode=None,\n        target_size=(299, 299),\n        batch_size=BATCH_SIZE,\n        shuffle=False\n    )\n\ndef create_flow(datagen, subset):\n    return datagen.flow_from_dataframe(\n        pivot_df, \n        directory='/kaggle/tmp/',\n        x_col='filename', \n        y_col=['any', 'epidural', 'intraparenchymal', \n               'intraventricular', 'subarachnoid', 'subdural'],\n        class_mode='other',\n        target_size=(299, 299),\n        batch_size=BATCH_SIZE,\n        subset=subset\n    )\n\n# Using original generator\ndata_generator = create_datagen()\ntrain_gen = create_flow(data_generator, 'training')\nval_gen = create_flow(data_generator, 'validation')\ntest_gen = create_test_gen()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:09.972809Z","iopub.execute_input":"2021-12-12T16:52:09.975098Z","iopub.status.idle":"2021-12-12T16:52:10.710406Z","shell.execute_reply.started":"2021-12-12T16:52:09.975024Z","shell.execute_reply":"2021-12-12T16:52:10.709654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##################################################################\n####Here we are checking the training set batch and its output\n##################################################################\nfor data_batch,label_batch in train_gen:\n    print(\"DATA SHAPE: \",data_batch.shape)\n    print(\"LABEL SHAPE: \",label_batch.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:10.711742Z","iopub.execute_input":"2021-12-12T16:52:10.711983Z","iopub.status.idle":"2021-12-12T16:52:11.339917Z","shell.execute_reply.started":"2021-12-12T16:52:10.71195Z","shell.execute_reply":"2021-12-12T16:52:11.339214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################\n####Here we are checking the validation or testing set batch and its output\n#############################################################################\nfor data_batch,label_batch in val_gen:\n    print(\"DATA SHAPE: \",data_batch.shape)\n    print(\"LABEL SHAPE: \",label_batch.shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.341142Z","iopub.execute_input":"2021-12-12T16:52:11.341913Z","iopub.status.idle":"2021-12-12T16:52:11.702465Z","shell.execute_reply.started":"2021-12-12T16:52:11.341875Z","shell.execute_reply":"2021-12-12T16:52:11.70173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import numpy as np\n#from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.703542Z","iopub.execute_input":"2021-12-12T16:52:11.704237Z","iopub.status.idle":"2021-12-12T16:52:11.707919Z","shell.execute_reply.started":"2021-12-12T16:52:11.7042Z","shell.execute_reply":"2021-12-12T16:52:11.706825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train, X_test, y_train, y_test = train_test_split(\n# data_batch, label_batch, test_size=0.33, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.714324Z","iopub.execute_input":"2021-12-12T16:52:11.714528Z","iopub.status.idle":"2021-12-12T16:52:11.718635Z","shell.execute_reply.started":"2021-12-12T16:52:11.7145Z","shell.execute_reply":"2021-12-12T16:52:11.717761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.719802Z","iopub.execute_input":"2021-12-12T16:52:11.72069Z","iopub.status.idle":"2021-12-12T16:52:11.727463Z","shell.execute_reply.started":"2021-12-12T16:52:11.720655Z","shell.execute_reply":"2021-12-12T16:52:11.726524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.applications import xception","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.730282Z","iopub.execute_input":"2021-12-12T16:52:11.730605Z","iopub.status.idle":"2021-12-12T16:52:11.736726Z","shell.execute_reply.started":"2021-12-12T16:52:11.730569Z","shell.execute_reply":"2021-12-12T16:52:11.73603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################\n####Here we are setting the branch 1 of the double cnn-rf model\n#############################################################################\nfrom tensorflow.keras.models import Model\n#from tensorflow.keras.applications import mobilenet_v2\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n#tf.keras.applications.MobileNetV2()\n#branch1 = mobilenet_v2.MobileNetV2()\ntf.keras.applications.Xception()\nbranch1 = xception.Xception()\n#branch1 = resnet50.ResNet50()\nbranch1.layers.pop()\nfor layer in branch1.layers:\n  layer.trainable=False\nlast = branch1.layers[-2].output####################### getting last layer of pooling and it give 2046 features#######\nx = Dense(6, activation=\"softmax\")(last)#########last layer for classification for experiment no 01#########\nfinetuned_model_1= Model(branch1.input, x, name=\"Branch 1\")\nfinetuned_model_1.summary()\nfinetuned_model_1.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=['accuracy','mse', 'mae', 'mape', 'cosine'])\n\n\ncheckpoint = ModelCheckpoint(\n    'model.h5', \n    monitor='val_loss', \n    verbose=0, \n    save_best_only=True, \n    save_weights_only=False,\n    mode='auto'\n)\n'''\nResnet_Model1 = finetuned_model_1.fit_generator(\n    train_gen,\n    steps_per_epoch=200,\n    validation_data=val_gen,\n    validation_steps=100,\n    callbacks=[checkpoint],\n    epochs=30\n)\nPrediction_branch1 = finetuned_model_1.predict(test_gen)\n'''\n\n\n##############################################################################\n###########################################################################\n## if you want to perform experiment no 1, which is based on the output of bracnch 1. \n##Simply if you want to perform classisfication through branch 1, so please remove the comments. Experiment #1 described in Section 3.\n###########################################################################\n'''\n###################\n########parameters, including the number of epochs (60),were chosen experimentally.\n########## The training was terminated automatically if the validation loss did notdecrease for ten epochs (patience 10).#########\n############as mentioned in paper#########\n'''","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:11.738778Z","iopub.execute_input":"2021-12-12T16:52:11.738973Z","iopub.status.idle":"2021-12-12T16:52:20.647552Z","shell.execute_reply.started":"2021-12-12T16:52:11.738949Z","shell.execute_reply":"2021-12-12T16:52:20.646872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################\n####Here we are setting the branch 2 of the double cnn-rf model\n#############################################################################\nfrom tensorflow.keras.models import Model\n#from tensorflow.keras.applications import mobilenet_v2\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n#tf.keras.applications.MobileNetV2()\n#branch2 = mobilenet_v2.MobileNetV2()\n#branch2 = resnet50.ResNet50()\n#branch2 = resnet50.ResNet50()\ntf.keras.applications.Xception()\nbranch2 = xception.Xception()\nbranch2.layers.pop()\nfor layer in branch2.layers:\n  layer.trainable=False\nlast = branch2.layers[-2].output####################### getting last layer of pooling and it give 2046 features#######\nx = Dense(6, activation=\"softmax\")(last)#########last layer for classification for experiment no 01#########\nfinetuned_model_2= Model(branch2.input, x, name=\"Branch 2\")\nfinetuned_model_2.summary()\nfinetuned_model_2.compile(optimizer=Adam(lr=0.0001), loss='binary_crossentropy', metrics=['accuracy','mse', 'mae', 'mape', 'cosine'])\n\n\ncheckpoint = ModelCheckpoint(\n    'model.h5', \n    monitor='val_loss', \n    verbose=0, \n    save_best_only=True, \n    save_weights_only=False,\n    mode='auto'\n)\n'''\nResnet_Model2 = finetuned_model_2.fit_generator(\n    train_gen,\n    steps_per_epoch=300,\n    validation_data=val_gen,\n    validation_steps=200,\n    callbacks=[checkpoint],\n    epochs=60\n)\nPrediction_branch2 = finetuned_model_2.predict(test_gen)\n'''\n\n\n##############################################################################\n###########################################################################\n## if you want to perform experiment no 1, which is based on the output of bracnch 1. \n##Simply if you want to perform classisfication through branch 1, so please remove the comments. Experiment #1 described in Section 3.\n###########################################################################\n\n###################\n########parameters, including the number of epochs (60),were chosen experimentally.\n########## The training was terminated automatically if the validation loss did notdecrease for ten epochs (patience 10).#########\n############as mentioned in paper#########\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:20.648756Z","iopub.execute_input":"2021-12-12T16:52:20.649249Z","iopub.status.idle":"2021-12-12T16:52:23.166763Z","shell.execute_reply.started":"2021-12-12T16:52:20.649212Z","shell.execute_reply":"2021-12-12T16:52:23.166091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import concatenate\n###########################\n###########################we have taken the features extracted by either \n########branch after the average pooling layer as mentioned in paper###########################\n'''\nAs mentioned in the paper, after the training, features from the last block preceding the ResNet-50’s fully connected layer\nwere taken from either branch and concatenated. The joint feature vector containing 4096 elements\nwas subjected to the classification process. As we can check that in output, we are getting right same features 4096.\n'''\nprint('Getting training features and concatenation---------------start')\nmodel1= Model(branch1.input, branch1.layers[-2].output)\nmodel1_features=model1.predict(train_gen)\nmodel1_features=pd.DataFrame(model1_features)\nprint(\"Branch1_features\", model1_features.shape)\n\nmodel2= Model(branch2.input, branch2.layers[-2].output)\nmodel2_features=model2.predict(train_gen)\nmodel2_features=pd.DataFrame(model2_features)\nprint(\"Branch2_features\", model2_features.shape)\n\n####################################Concatenation of features###########################\nconcatenated_features=pd.concat([model1_features,model2_features], axis=1)\nprint(\"Combined_features\", concatenated_features.shape)\n\nprint('Getting validation features and concatenation---------------start')\nmodel1= Model(branch1.input, branch1.layers[-2].output)\nmodel1_val_features=model1.predict(val_gen)\nmodel1_val_features=pd.DataFrame(model1_val_features)\nprint(\"Branch1_val_features\", model1_val_features.shape)\n\nmodel2= Model(branch2.input, branch2.layers[-2].output)\nmodel2_val_features=model2.predict(val_gen)\nmodel2_val_features=pd.DataFrame(model2_val_features)\nprint(\"Branch2_val_features\", model2_val_features.shape)\n\n####################################Concatenation of features###########################\nconcatenated_val_features=pd.concat([model1_val_features,model2_val_features], axis=1)\nprint(\"Combined_val_features\", concatenated_val_features.shape)\n#############################################################################\n####Here, we have obtained the fatures from branch1 and branch 2 \n#############################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:52:23.168675Z","iopub.execute_input":"2021-12-12T16:52:23.168922Z","iopub.status.idle":"2021-12-12T16:56:15.854987Z","shell.execute_reply.started":"2021-12-12T16:52:23.168887Z","shell.execute_reply":"2021-12-12T16:56:15.85422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#############################################################################\n####Setting labels for combined training and testing data\n#############################################################################\n####################################Training Labels of features###########################\nfeature_train_labels=pd.DataFrame(train_gen.labels)\nprint(feature_train_labels)\n\n####################################Test Labels###########################\nfeature_test_labels=pd.DataFrame(val_gen.labels)\nlen(feature_test_labels.shape)\n\n","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:15.856598Z","iopub.execute_input":"2021-12-12T16:56:15.857125Z","iopub.status.idle":"2021-12-12T16:56:15.869517Z","shell.execute_reply.started":"2021-12-12T16:56:15.857087Z","shell.execute_reply":"2021-12-12T16:56:15.868707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################################We used the RF classifier with same hyperparamters##################\n###########################as mentioned in the paper#################################\n###################################################################################################\nfrom sklearn.ensemble import RandomForestClassifier\nRf_classifier=RandomForestClassifier(bootstrap=True, n_estimators=200, max_depth=4,min_samples_split=30)\nRf_classifier.fit(concatenated_features,feature_train_labels)\nRf_classifier.fit(concatenated_val_features,feature_test_labels)\nprint(\"done tarining\")","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:15.87076Z","iopub.execute_input":"2021-12-12T16:56:15.87146Z","iopub.status.idle":"2021-12-12T16:56:33.328292Z","shell.execute_reply.started":"2021-12-12T16:56:15.871404Z","shell.execute_reply":"2021-12-12T16:56:33.327505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_RF=Rf_classifier.predict(concatenated_val_features)\nprint(\"pred_RF\",pred_RF.shape)\n####################################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:33.329602Z","iopub.execute_input":"2021-12-12T16:56:33.329984Z","iopub.status.idle":"2021-12-12T16:56:33.454275Z","shell.execute_reply.started":"2021-12-12T16:56:33.329948Z","shell.execute_reply":"2021-12-12T16:56:33.453575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"################################We used the RF classifier with same hyperparamters##################\n###########################as mentioned in the paper#################################\n###################################################################################################\nfrom sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(n_neighbors=7)\n#from sklearn.ensemble import RandomForestClassifier\n#knn=LinearSVC(multi_class='crammer_singer')\nknn.fit(concatenated_features,feature_train_labels)\nknn.fit(concatenated_val_features,feature_test_labels)\nprint(\"done tarining\")","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:33.455533Z","iopub.execute_input":"2021-12-12T16:56:33.455782Z","iopub.status.idle":"2021-12-12T16:56:39.347416Z","shell.execute_reply.started":"2021-12-12T16:56:33.455749Z","shell.execute_reply":"2021-12-12T16:56:39.345834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_knn= knn.predict(concatenated_val_features)\nprint(\"pred_KNN\",pred_knn.shape)\n####################################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:39.348841Z","iopub.execute_input":"2021-12-12T16:56:39.349114Z","iopub.status.idle":"2021-12-12T16:56:52.833902Z","shell.execute_reply.started":"2021-12-12T16:56:39.349078Z","shell.execute_reply":"2021-12-12T16:56:52.833121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\nfrom sklearn import metrics\naccuracy=metrics.accuracy_score(feature_test_labels,pred_knn.round())\nprint(\"Accuracy of combined model with KNN: {0:0.4f}\".format(accuracy*100))\n\nfrom sklearn.metrics import f1_score\nf1score=f1_score(pred_knn,feature_test_labels, average='weighted')\nprint(\"F1score of combined model with KNN: {0:0.4f}\".format( f1score*100))\n\nfrom sklearn.metrics import recall_score\nrecall = recall_score(feature_test_labels,pred_knn, average='weighted')\nprint('Recall score of combined model with KNN: {0:0.4f}'.format(recall*100))\n\nfrom sklearn.metrics import precision_score\nprecision = precision_score(pred_knn.round(),feature_test_labels,average='weighted')\nprint('Precision of combined model with KNN: {0:0.4f}'.format(precision*100))\n\n####################################################################################\n####################################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.835392Z","iopub.execute_input":"2021-12-12T16:56:52.835797Z","iopub.status.idle":"2021-12-12T16:56:52.862086Z","shell.execute_reply.started":"2021-12-12T16:56:52.835758Z","shell.execute_reply":"2021-12-12T16:56:52.861429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfrom sklearn.metrics import multilabel_confusion_matrix\ncm1 = multilabel_confusion_matrix(feature_test_labels,pred_knn)\nprint(cm1)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.863204Z","iopub.execute_input":"2021-12-12T16:56:52.863435Z","iopub.status.idle":"2021-12-12T16:56:52.871252Z","shell.execute_reply.started":"2021-12-12T16:56:52.863404Z","shell.execute_reply":"2021-12-12T16:56:52.870381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfirst_cat=cm1[0]\ntn, fp, fn, tp = first_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_1=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for any is :\",fpr*100)\nprint(\"fnr for any is :  \",fnr*100)\nprint(\"TPR for any is : \",TPR*100)\nprint(\"TNR for any is : \",TNR*100)\nprint(\"Accuracy for any is \",acc_1*100)\n\nsecond_cat=cm1[1]\ntn, fp, fn, tp = second_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_2=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for epidural is :  \",fpr*100)\nprint(\"fnr for epidural is : \",fnr*100)\nprint(\"TPR for epidural is : \",TPR*100)\nprint(\"TNR for epidural is : \",TNR*100)\nprint(\"Accuracy for epidural is : \",acc_2*100)\n\nthird_cat=cm1[2]\ntn, fp, fn, tp = third_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_3=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for intraparenchymal is : \",fpr*100)\nprint(\"fnr for intraparenchymal is : \",fnr*100)\nprint(\"TPR for intraparenchymal is : \",TPR*100)\nprint(\"TNR for intraparenchymal is : \",TNR*100)\nprint(\"Accuracy for intraparenchymal is \",acc_3*100)\n\nfourth_cat=cm1[3]\ntn, fp, fn, tp = fourth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_4=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for intraventricular is : \",fpr*100)\nprint(\"fnr for intraventricular is :  \",fnr*100)\nprint(\"TPR for intraventricular is : \",TPR*100)\nprint(\"TNR for intraventricular is : \",TNR*100)\nprint(\"Accuracy for intraventricular : is \",acc_4*100)\n\nfifth_cat=cm1[4]\ntn, fp, fn, tp = fourth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_5=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for subarachnoid is  : \",fpr*100)\nprint(\"fnr for subarachnoid is :  \",fnr*100)\nprint(\"TPR for subarachnoid is : \",TPR*100)\nprint(\"TNR for subarachnoid is : \",TNR*100)\nprint(\"Accuracy for subarachnoid is :  \",acc_5*100)\n\nsixth_cat=cm1[5]\ntn, fp, fn, tp = sixth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_6=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for subdural is : \",fpr*100)\nprint(\"fnr for subdural is : \",fnr*100)\nprint(\"TPR for subdural is : \",TPR*100)\nprint(\"TNR for subdural is :\",TNR*100)\nprint(\"Accuracy for Subdural is \",acc_6*100)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.872843Z","iopub.execute_input":"2021-12-12T16:56:52.873444Z","iopub.status.idle":"2021-12-12T16:56:52.904344Z","shell.execute_reply.started":"2021-12-12T16:56:52.873295Z","shell.execute_reply":"2021-12-12T16:56:52.903695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\nfrom sklearn import metrics\naccuracy=metrics.accuracy_score(feature_test_labels,pred_knn.round())\nprint(\"Accuracy of combined model with KNN: {0:0.4f}\".format(accuracy*100))\n\nfrom sklearn.metrics import f1_score\nf1score=f1_score(pred_knn,feature_test_labels, average='weighted')\nprint(\"F1score of combined model with KNN: {0:0.4f}\".format( f1score*100))\n\nfrom sklearn.metrics import recall_score\nrecall = recall_score(feature_test_labels,pred_knn, average='weighted')\nprint('Recall score of combined model with KNN: {0:0.4f}'.format(recall*100))\n\nfrom sklearn.metrics import precision_score\nprecision = precision_score(pred_knn.round(),feature_test_labels,average='weighted')\nprint('Precision of combined model with KNN: {0:0.4f}'.format(precision*100))\n\n####################################################################################\n####################################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.905321Z","iopub.execute_input":"2021-12-12T16:56:52.905627Z","iopub.status.idle":"2021-12-12T16:56:52.924911Z","shell.execute_reply.started":"2021-12-12T16:56:52.905595Z","shell.execute_reply":"2021-12-12T16:56:52.924183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\nfrom sklearn import metrics\naccuracy=metrics.accuracy_score(feature_test_labels,pred_RF.round())\nprint(\"Accuracy of combined model with RF: {0:0.4f}\".format(accuracy*100))\n\nfrom sklearn.metrics import f1_score\nf1score=f1_score(pred_RF,feature_test_labels, average='weighted')\nprint(\"F1score of combined model with RF: {0:0.4f}\".format( f1score*100))\n\nfrom sklearn.metrics import recall_score\nrecall = recall_score(feature_test_labels,pred_RF, average='weighted')\nprint('Recall score of combined model with RF: {0:0.4f}'.format(recall*100))\n\nfrom sklearn.metrics import precision_score\nprecision = precision_score(pred_RF.round(),feature_test_labels,average='weighted')\nprint('Precision of combined model with RF: {0:0.4f}'.format(precision*100))\n\n####################################################################################\n####################################################################################","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.926075Z","iopub.execute_input":"2021-12-12T16:56:52.926762Z","iopub.status.idle":"2021-12-12T16:56:52.945256Z","shell.execute_reply.started":"2021-12-12T16:56:52.926729Z","shell.execute_reply":"2021-12-12T16:56:52.94451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1score","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.946365Z","iopub.execute_input":"2021-12-12T16:56:52.947013Z","iopub.status.idle":"2021-12-12T16:56:52.95203Z","shell.execute_reply.started":"2021-12-12T16:56:52.946979Z","shell.execute_reply":"2021-12-12T16:56:52.951339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfrom sklearn.metrics import classification_report\nlabe=['any', 'epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural']\nprint( classification_report(feature_test_labels,pred_svm, target_names=labe))","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.953341Z","iopub.execute_input":"2021-12-12T16:56:52.954277Z","iopub.status.idle":"2021-12-12T16:56:52.971716Z","shell.execute_reply.started":"2021-12-12T16:56:52.95424Z","shell.execute_reply":"2021-12-12T16:56:52.97109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfrom sklearn.metrics import roc_auc_score\n# Generate class membership probabilities\nROC_auc_score=roc_auc_score( feature_test_labels,pred_RF, average=\"weighted\", multi_class=\"ovr\")\nprint(\"ROC_auc_score is \", ROC_auc_score)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.972838Z","iopub.execute_input":"2021-12-12T16:56:52.97417Z","iopub.status.idle":"2021-12-12T16:56:52.984953Z","shell.execute_reply.started":"2021-12-12T16:56:52.974134Z","shell.execute_reply":"2021-12-12T16:56:52.984223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfrom sklearn.metrics import multilabel_confusion_matrix\ncm = multilabel_confusion_matrix(feature_test_labels,pred_RF)\nprint(cm)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.986913Z","iopub.execute_input":"2021-12-12T16:56:52.987447Z","iopub.status.idle":"2021-12-12T16:56:52.995591Z","shell.execute_reply.started":"2021-12-12T16:56:52.987422Z","shell.execute_reply":"2021-12-12T16:56:52.994892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################################################################################\n######Performance evaluation of the combined double-branch convolutional neural network (CNN) \n############################based on the ResNet-50 architecture with Random forest#######################\n########################################################################################\n\nfirst_cat=cm[0]\ntn, fp, fn, tp = first_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_1=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for any is :\",fpr*100)\nprint(\"fnr for any is :  \",fnr*100)\nprint(\"TPR for any is : \",TPR*100)\nprint(\"TNR for any is : \",TNR*100)\nprint(\"Accuracy for any is \",acc_1*100)\n\nsecond_cat=cm[1]\ntn, fp, fn, tp = second_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_2=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for epidural is :  \",fpr*100)\nprint(\"fnr for epidural is : \",fnr*100)\nprint(\"TPR for epidural is : \",TPR*100)\nprint(\"TNR for epidural is : \",TNR*100)\nprint(\"Accuracy for epidural is : \",acc_2*100)\n\nthird_cat=cm[2]\ntn, fp, fn, tp = third_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_3=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for intraparenchymal is : \",fpr*100)\nprint(\"fnr for intraparenchymal is : \",fnr*100)\nprint(\"TPR for intraparenchymal is : \",TPR*100)\nprint(\"TNR for intraparenchymal is : \",TNR*100)\nprint(\"Accuracy for intraparenchymal is \",acc_3*100)\n\nfourth_cat=cm[3]\ntn, fp, fn, tp = fourth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_4=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for intraventricular is : \",fpr*100)\nprint(\"fnr for intraventricular is :  \",fnr*100)\nprint(\"TPR for intraventricular is : \",TPR*100)\nprint(\"TNR for intraventricular is : \",TNR*100)\nprint(\"Accuracy for intraventricular : is \",acc_4*100)\n\nfifth_cat=cm[4]\ntn, fp, fn, tp = fourth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_5=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for subarachnoid is  : \",fpr*100)\nprint(\"fnr for subarachnoid is :  \",fnr*100)\nprint(\"TPR for subarachnoid is : \",TPR*100)\nprint(\"TNR for subarachnoid is : \",TNR*100)\nprint(\"Accuracy for subarachnoid is :  \",acc_5*100)\n\nsixth_cat=cm[5]\ntn, fp, fn, tp = sixth_cat.ravel()\nfpr = fp / (tn + fp)\nfnr = fn / (tp + fn)\nTPR = tp/(tp+fn)\nTNR = tn/(tn+fp)\nacc_6=(tp+tn)/(tp+tn+fp+fn)\nprint(\"fpr for subdural is : \",fpr*100)\nprint(\"fnr for subdural is : \",fnr*100)\nprint(\"TPR for subdural is : \",TPR*100)\nprint(\"TNR for subdural is :\",TNR*100)\nprint(\"Accuracy for Subdural is \",acc_6*100)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T16:56:52.997218Z","iopub.execute_input":"2021-12-12T16:56:52.998075Z","iopub.status.idle":"2021-12-12T16:56:53.028115Z","shell.execute_reply.started":"2021-12-12T16:56:52.998008Z","shell.execute_reply":"2021-12-12T16:56:53.027436Z"},"trusted":true},"execution_count":null,"outputs":[]}]}