{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/vbookshelf/cnn-how-to-use-160-000-images-without-crashing\nfrom numpy.random import seed\nseed(101)\n#from tensorflow import set_random_seed\n#set_random_seed(101)\n\nimport tensorflow as tf\ntf.compat.v1.set_random_seed(101)\n\n\nimport pandas as pd\nimport numpy as np\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Activation\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\n\nimport os\nimport cv2\n\nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport itertools\nimport shutil\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IMAGE_SIZE = 96\nIMAGE_CHANNELS = 3\n\nSAMPLE_SIZE = 80000 # the number of images we use from each of the two classes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('/kaggle/input/rsna-str-pulmonary-embolism-detection/train')))\nprint(len(os.listdir('/kaggle/input/rsna-str-pulmonary-embolism-detection/test')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create a Dataframe containing all images\n\ndf_data = pd.read_csv('/kaggle/input/rsna-str-pulmonary-embolism-detection/train.csv')\n\nprint(df_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.sample(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add the path column with filenames in the dataframe\n\ndf_data['path'] = '/kaggle/input/rsna-str-pulmonary-embolism-detection/train/' + df_data['StudyInstanceUID'] + '/' + df_data['SeriesInstanceUID'] + '/' + df_data['SOPInstanceUID'] + '.dcm'\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.sample()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.iloc[1621755,].path","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nfrom pydicom import dcmread\nds = pydicom.read_file('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/b4548bee81e8/ac1aea5d7662/cc96a7a2e72c.dcm')\nds = dcmread('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/b4548bee81e8/ac1aea5d7662/cc96a7a2e72c.dcm')\nsample_image_arr = ds.pixel_array\nsample_image_arr.shape\n#show the sample image\n\nimport matplotlib.pyplot as plt\nplt.imshow(sample_image_arr, cmap=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_image_arr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pydicom\nfrom pydicom import dcmread\nds = dcmread(df_data.iloc[1621756,].path)\nsample_image_arr = ds.pixel_array\nsample_image_arr.shape\n#show the sample image\n\nimport matplotlib.pyplot as plt\nplt.imshow(sample_image_arr, cmap=\"gray\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_image_arr.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ImageDataGenerator cannot process .dcm file so we need to change it to .PNG\n# initially we do it for a random sample of 1000 images and save them on /kaggle/tmp i.e. ../tmp\nsample_size = 1000\ndf_data_1 = df_data.sample(sample_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data_1.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pip install png","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport png, os, pydicom\n\nsource_folder = r'path\\to\\source'\noutput_folder = r'path\\to\\output\\folder'\n\n\ndef dicom2png(source_folder, output_folder):\n    list_of_files = os.listdir(source_folder)\n    for file in list_of_files:\n        try:\n            ds = pydicom.dcmread(os.path.join(source_folder,file))\n            shape = ds.pixel_array.shape\n\n            # Convert to float to avoid overflow or underflow losses.\n            image_2d = ds.pixel_array.astype(float)\n\n            # Rescaling grey scale between 0-255\n            image_2d_scaled = (np.maximum(image_2d,0) / image_2d.max()) * 255.0\n\n            # Convert to uint\n            image_2d_scaled = np.uint8(image_2d_scaled)\n\n            # Write the PNG file\n            with open(os.path.join(output_folder,file)+'.png' , 'wb') as png_file:\n                w = png.Writer(shape[1], shape[0], greyscale=True)\n                w.write(png_file, image_2d_scaled)\n        except:\n            print('Could not convert: ', file)\n\n\ncv2.imwrite(outdir + f.replace('.dcm','.png'),img)dicom2png(source_folder, output_folder)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf_data['path_new'] = df_data['SOPInstanceUID'] + '.png'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data['path_new']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cv2\noutdir = '/kaggle/tmp/'\n#os.mkdir(outdir)\n\n\nfor i in range(len(df_data)):\n    ds = dcmread(df_data.iloc[i,].path)\n    image_arr = ds.pixel_array\n    image_arr.shape\n    cv2.imwrite(outdir + df_data.iloc[i,].SOPInstanceUID + '.png',image_arr)\n    print(i)\n    \n#show the sample image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n!ls /kaggle/tmp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data_1.path_new","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/tmp'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a data generator\ndatagen = ImageDataGenerator(rescale=1./255,validation_split=0.20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load and iterate training dataset\ntrain_it = datagen.flow_from_dataframe(df_data,directory='/kaggle/tmp/',x_col = 'path_new',y_col = 'pe_present_on_image',class_mode = 'raw',batch_size = 64,validate_filenames=False, color_mode = 'grayscale',target_size = (512,512))\n# load and iterate validation dataset\n#val_it = datagen.flow_from_directory('/kaggle/input/rsna-str-pulmonary-embolism-detection/test/', class_mode='categorical', batch_size=64)\n# load and iterate test dataset\n#test_it = datagen.flow_from_directory('/kaggle/input/rsna-str-pulmonary-embolism-detection/test/', class_mode='categorical', batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Activation, Dropout\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten\nfrom tensorflow.keras.utils import to_categorical, plot_model\n\n# network parameters\ninput_shape = (512,512,1)\nbatch_size = 128\nkernel_size = (3,3)\npool_size = 2\nfilters = 64\ndropout = 0.2\nnum_labels = 2\n\nmodel = Sequential()\nmodel.add(Conv2D(filters = filters, kernel_size = kernel_size, activation = 'relu', input_shape = input_shape))\nmodel.add(MaxPooling2D(pool_size))\nmodel.add(Conv2D(filters = filters, kernel_size = kernel_size, activation = 'relu'))\nmodel.add(MaxPooling2D(pool_size))\nmodel.add(Conv2D(filters = filters, kernel_size = kernel_size, activation = 'relu'))\nmodel.add(Flatten())\nmodel.add(Dropout(dropout))\nmodel.add(Dense(num_labels))\nmodel.add(Activation('softmax'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_model(model,to_file='cnn-mnist.png',show_shapes = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss = 'categorical_crossentropy',optimizer = 'adam', metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit_generator(generator=train_it,\n                    steps_per_epoch=8,\n                    epochs=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data['pe_present_on_image'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_data['pe_present_on_image'].isnull().count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}