{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np \nimport pandas as pd \nimport os\nfrom pydicom import dcmread # for dcm files\nimport pydicom\nimport cv2\nimport csv\n\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nfrom plotly.offline import iplot, init_notebook_mode\nfrom keras.losses import categorical_crossentropy\nfrom keras.optimizers import Adadelta\nimport plotly.graph_objs as go\nfrom matplotlib.pyplot import cm\nfrom keras.models import Model\nimport numpy as np\nimport keras\nimport h5py\nimport gdcm\nimport PIL\nimport tensorflow as tf\nfrom skimage import measure, morphology\nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nimage_paths = []\nSOPs_train = []\nimport os\nfor dirname, _, filenames in os.walk('../input/rsna-str-pulmonary-embolism-detection/train'):\n    for filename in filenames:\n        temp = os.path.join(dirname, filename)\n        image_paths.append(os.path.join(dirname, filename))\n        SOPs_train.append(temp[len(temp)-16:len(temp)-4])\n        \n\nimage_paths = np.asarray(image_paths)\nSOPs_train = np.asarray(SOPs_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"with open ('../input/rsna-str-pulmonary-embolism-detection/train.csv') as csvfile:  \n    readCSVFeatures = csv.reader(csvfile, delimiter=',')\n    train_data = list(csv.reader(csvfile))\ntrain_data = train_data[1:len(train_data )]    \ntrain_data = np.asarray(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SOPs_train_index= np.argsort(SOPs_train)\nSOPs_train= SOPs_train[SOPs_train_index]\nimage_paths = image_paths[SOPs_train_index]\n\n\nSOPs_in_traincsv_index =  np.argsort(train_data[:,2])\nSOPs_in_traincsv = train_data[:,2][SOPs_in_traincsv_index]\ny_train_images = train_data[:,3][SOPs_in_traincsv_index]\n\nprint(len(SOPs_train))\nprint(len(train_data))\nprint(len(y_train_images))\n\nprint(SOPs_in_traincsv[0])\nprint(image_paths[0])\nprint(SOPs_train[0])\nprint(SOPs_in_traincsv[len(SOPs_in_traincsv)-1])\nprint(SOPs_train[len(SOPs_train)-1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PREPROCESS IMAGES"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef preprocess_image(path):\n    dataset = dcmread(path)\n    image_dene = np.asarray(dataset.pixel_array)\n    preprocessed_image= cv2.resize(image_dene , (150, 150),  interpolation = cv2.INTER_AREA)\n    preprocessed_image = cv2.merge((preprocessed_image,preprocessed_image,preprocessed_image))\n    preprocessed_image = tf.keras.applications.xception.preprocess_input(preprocessed_image) \n\n    return preprocessed_image\n    \n    \n      \n    \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_images = y_train_images.astype(np.float)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CREATE 2D MODEL FOR IMAGES"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nMETRICS = [\n      #keras.metrics.TruePositives(name='tp'),\n      #keras.metrics.FalsePositives(name='fp'),\n      #keras.metrics.TrueNegatives(name='tn'),\n      #keras.metrics.FalseNegatives(name='fn'), \n      #keras.metrics.BinaryAccuracy(name='accuracy'),\n      keras.metrics.Precision(name='precision'),\n      #keras.metrics.Recall(name='recall'),\n      #keras.metrics.AUC(name='auc'),\n]\nbase_model = tf.keras.applications.Xception(\n    weights='imagenet',  # Load weights pre-trained on ImageNet.\n    input_shape=(150, 150, 3),\n    include_top=False) \nbase_model.trainable = False\ninputs = keras.Input(shape=(150, 150, 3))\n\nx = base_model(inputs, training=False)\n\nx = tf.keras.layers.GlobalAveragePooling2D()(x)\nprint(x.shape)\n#x = tf.keras.layers.Dense(64, activation='relu')(x)\n#x = tf.keras.layers.Dropout(0.5)(x)\nprint(x.shape)\noutputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\nmodel_for_pe_on_image = tf.keras.Model(inputs, outputs)\nmodel_for_pe_on_image.compile(loss='binary_crossentropy',\n              optimizer='Adam',\n              metrics=['accuracy'])\n\n#model_for_pe_on_image = keras.models.load_model('../input/model2d500/model_2d.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CREATE BALANCED DATASET"},{"metadata":{"trusted":true},"cell_type":"code","source":"for times in range(4):\n    print(\"times :\",times)\n    for count in range(0,30):\n        no_of_data_to_be_trained = 50000\n        print(\"No of data that is trained\",count*50000)\n        start = count* no_of_data_to_be_trained \n        end = start + no_of_data_to_be_trained \n        train_labels_to_be_used = y_train_images[start:end]\n        image_paths_to_be_used = image_paths[start:end]\n        indexes_0 = np.where(train_labels_to_be_used== 0.0)[0]\n        indexes_1 = np.where(train_labels_to_be_used == 1.0)[0]\n\n        indexes_0_shuffled = np.random.shuffle(indexes_0)\n        indexes_1_shuffled = np.random.shuffle(indexes_1)\n        indexes_0_shuffled = indexes_0[0:2*int(len(indexes_1))]\n        concat_indexes_shuffled = np.concatenate((indexes_0_shuffled,indexes_1),axis=None)\n        np.random.shuffle(concat_indexes_shuffled)\n        print(indexes_0.shape)\n        print(indexes_1.shape)\n        print(concat_indexes_shuffled.shape)\n        shuffled_image_paths = image_paths_to_be_used[concat_indexes_shuffled]\n        shuffled_y_train_images = train_labels_to_be_used[concat_indexes_shuffled]\n\n        x_train_images = np.ones((len(shuffled_image_paths), 150, 150, 3),  dtype=np.int16)\n        for i in range(0,len(shuffled_image_paths)):\n            if i % 1000 == 0 :\n                print(i) \n            x_train_images[i]= preprocess_image(shuffled_image_paths[i])\n\n        model_for_pe_on_image.fit(x=x_train_images, y=shuffled_y_train_images, batch_size=64, epochs=1)\n        model_for_pe_on_image.save('model_last.h5')  # creates a HDF5 file 'my_model.h5'\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nfor count in range(4,8):\n    print(\"No of data that is trained\",count*100000)\n    no_of_data_to_be_trained = 50000\n    start = count*50000\n    end = start + no_of_data_to_be_trained \n    train_labels_to_be_used = y_train_images[start:end]\n    image_paths_to_be_used = image_paths[start:end]\n    indexes_0 = np.where(train_labels_to_be_used== 0.0)[0]\n    indexes_1 = np.where(train_labels_to_be_used == 1.0)[0]\n\n    indexes_0_shuffled = np.random.shuffle(indexes_0)\n    indexes_1_shuffled = np.random.shuffle(indexes_1)\n    indexes_0_shuffled = indexes_0[0:int(3*len(indexes_1))]\n    concat_indexes_shuffled = np.concatenate((indexes_0_shuffled,indexes_1),axis=None)\n    np.random.shuffle(concat_indexes_shuffled)\n    print(indexes_0.shape)\n    print(indexes_1.shape)\n    print(concat_indexes_shuffled.shape)\n    shuffled_image_paths = image_paths_to_be_used[concat_indexes_shuffled]\n    shuffled_y_train_images = train_labels_to_be_used[concat_indexes_shuffled]\n\n    x_train_images = np.ones((len(shuffled_image_paths), 150, 150, 3),  dtype=np.int16)\n    for i in range(0,len(shuffled_image_paths)):\n        if i % 1000 == 0 :\n            print(i) \n        x_train_images[i]= preprocess_image(shuffled_image_paths[i])\n\n    model_for_pe_on_image.fit(x=x_train_images, y=shuffled_y_train_images, batch_size=64, epochs=5, validation_split=0.2)\nmodel_for_pe_on_image.save('model_second.h5')  # creates a HDF5 file 'my_model.h5'\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_images = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from keras.models import load_model\n#model_for_pe_on_image.save('model_first.h5')  # creates a HDF5 file 'my_model.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras.layers import Activation, Dropout, Flatten, Dense\n\nmodel_2d = Sequential()\nmodel_2d.add(Conv2D(32, (3, 3), input_shape=(64, 64, 3)))\nmodel_2d.add(Activation('relu'))\nmodel_2d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d.add(Conv2D(32, (3, 3)))\nmodel_2d.add(Activation('relu'))\nmodel_2d.add(MaxPooling2D(pool_size=(2, 2)))\n\nmodel_2d.add(Conv2D(64, (3, 3)))\nmodel_2d.add(Activation('relu'))\nmodel_2d.add(MaxPooling2D(pool_size=(2, 2)))\n\n\nmodel_2d.add(Flatten())  # this converts our 3D feature maps to 1D feature vectors\nmodel_2d.add(Dense(64))\nmodel_2d.add(Activation('relu'))\nmodel_2d.add(Dropout(0.5))\nmodel_2d.add(Dense(1))\nmodel_2d.add(Activation('sigmoid'))\n\nmodel_2d.compile(loss='binary_crossentropy',\n              optimizer='rmsprop',\n              metrics=['accuracy'])\n\n\nmodel_2d.fit(x=x_train_images, y=y_train_images, batch_size=64, epochs=50, validation_split=0.2)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SIMPLY TRY DATA"},{"metadata":{"trusted":true},"cell_type":"code","source":"no_of_train_test = 2000\nx_train_test = np.ones((no_of_train_test , 150, 150, 3),  dtype=np.int16)\nfor i in range(0,no_of_train_test ):\n    if i % 1000 == 0 :\n        print(i) \n    x_train_test[i]= preprocess_image(image_paths[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model_for_pe_on_image.predict(x_train_test)\nresults = model_for_pe_on_image.evaluate(x_train_test, y_train_images[0:no_of_train_test].astype(np.float), batch_size=64)\npred_binary = np.ones(no_of_train_test)\nprint(len(np.where(pred>0.5)[0]))\nprint(len(np.where(pred<=0.5)[0]))\n\npred_binary[np.where(pred>0.6)[0]] = 1\npred_binary[np.where(pred<=0.6)[0]] = 0\nunique, counts = np.unique(pred_binary, return_counts=True)\nprint(unique, counts)\n\nimport sklearn \nprint(sklearn.metrics.confusion_matrix(y_train_images[0:no_of_train_test],pred_binary,labels =[0,1]))\nprint(pred_binary)\nprint(y_train_images[0:no_of_train_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_binary[np.where(pred>0.5)[0]] = 1\npred_binary[np.where(pred<=0.5)[0]] = 0\nunique, counts = np.unique(pred_binary, return_counts=True)\nprint(unique, counts)\n\nimport sklearn \nprint(sklearn.metrics.confusion_matrix(y_train_images[0:no_of_train_test],pred_binary,labels =[0,1]))\nprint(pred_binary)\nprint(y_train_images[0:no_of_train_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label = y_train_images[0:no_of_train_test]\nfor i in range(0,2000):\n\n    print(\"i :\",i,\" pred: \",pred[i],\"true label :\",label[i])\n#print(y_train_images[0:no_of_train_test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#unique, indexes, counts = np.unique(y_train_images[0:1000],return_index=True, return_counts=True,)\n#print(unique, counts)\n#print(indexes)\n#indexes_0 = np.where(y_train_images[0:1000] == 0.0)[0]\n#indexes_1 = np.where(y_train_images[0:1000] == 1.0)[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# SAVE MODEL"},{"metadata":{"trusted":true},"cell_type":"code","source":"#from keras.models import load_model\n#model_for_pe_on_image.save('model_2d.h5')  # creates a HDF5 file 'my_model.h5'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}