{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Table of contents\n1. [Import libraries and datasets](#Import)\n2. [Exploratory analysis](#Exploratory-analysis)\n\n    * [Number of samples](#Number-of-samples)\n    * [Data Visualization](#Data-visualization) \n    * [Preprocess images](#Prepocess-Images)\n\n\n3. [Model](#Model) \n4. [Submission](#Submission)\n"},{"metadata":{},"cell_type":"markdown","source":"## Import"},{"metadata":{"_uuid":"40777d85-bbdb-4e1a-90d9-a0af291d3e5f","_cell_guid":"fd914ff4-3eb4-4220-a18d-4c76e9ed1998","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport pydicom\nimport cv2\nfrom tqdm import tqdm\nimport json\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = '../input/rsna-intracranial-hemorrhage-detection/'\ndir_train = '../input/rsna-intracranial-hemorrhage-detection/stage_1_train_images/'\ndir_test = '../input/rsna-intracranial-hemorrhage-detection/stage_1_test_images/'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e4347554-cf3f-4a0c-a9f8-ad7fc4b55694","_cell_guid":"b6a16557-de46-4b53-93ee-65ccc7fc385f","trusted":true},"cell_type":"code","source":"train_df =  pd.read_csv(directory+'stage_1_train.csv')\ntest_df = pd.read_csv(directory+'stage_1_sample_submission.csv')\ntrain_images = os.listdir(dir_train)\ntest_images = os.listdir(dir_test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef1dd07e-eb8a-4bef-a3dc-d0a0f9a5b7c5","_cell_guid":"d4030392-b359-4c6b-8ec5-aca22210a555","trusted":true},"cell_type":"markdown","source":"## Exploratory analysis"},{"metadata":{},"cell_type":"markdown","source":"### Number of samples"},{"metadata":{"_uuid":"998bce1d-24b6-4c5e-944d-e92a597eb567","_cell_guid":"04d3f4cb-4feb-40f9-870e-529661230f8d","trusted":true},"cell_type":"code","source":"print(\"Train CSV :\",train_df.shape)\nprint(\"Test CSV :\",test_df.shape)\nprint(\"Train Images:\",len(train_images))\nprint(\"Test Images:\",len(test_images))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b6340f26-b648-4c88-b40b-106c4e96d27a","_cell_guid":"6930bbac-a998-426d-bb5d-b5133d56cf39","trusted":true},"cell_type":"code","source":"display(train_df.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"66541287-b72f-4cb8-9913-7efac1e23800","_cell_guid":"b90dcdf0-3880-49c6-8953-d683c34d8aaa","trusted":true},"cell_type":"code","source":"display(train_df.tail())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb64538f-06d3-4134-9663-7c3191a5a24f","_cell_guid":"729861ee-39d1-40c3-850c-af03d8c59bba","trusted":true},"cell_type":"code","source":"display(train_df.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1ad8b8f-3608-4eba-804f-db02878dd1ce","_cell_guid":"7b0e9e61-3c4f-4035-8fdf-5be0fcb55a3e","trusted":true},"cell_type":"code","source":"print(\"Train: \\n\",train_df.count())\nprint(\"Test: \\n\",test_df.count())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51483d88-4fe0-45b2-95c2-03a4ee2b9824","_cell_guid":"3deef1f0-17e1-46ba-9d8f-46bb5496968b","trusted":true},"cell_type":"code","source":"train_df['Image_ID'] = train_df['ID'].str.rsplit(pat='_',n=1,expand=True)[0]\ntrain_df['Hemorrhage'] = train_df['ID'].str.rsplit(pat='_',n=1,expand=True)[1]\ntrain_df = train_df[['Image_ID','Hemorrhage','Label']]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf1df178-e54f-4266-ad18-55eddcf6c249","_cell_guid":"18e990ba-d3da-4e08-9284-e84f7ac1aef4","trusted":true},"cell_type":"code","source":"#Nombre d'images uniques\nprint(\"Number of images :\",train_df['Image_ID'].nunique())\nprint(\"Number of Hemorraghes :\",train_df['Hemorrhage'].nunique())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dddb59d8-fc75-4f8a-be87-ed432abd16dd","_cell_guid":"45ff2f6a-4d37-416e-85a6-d80aaca4ac0e","trusted":true},"cell_type":"code","source":"pd.DataFrame(train_df['Image_ID'].value_counts()).reset_index().head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['Image_ID'] = test_df['ID'].str.rsplit(pat='_',n=1,expand=True)[0]\ntest_df['Image_ID'] = test_df['Image_ID']+\".png\"\ntest_df = test_df['Image_ID'].drop_duplicates().reset_index()[['Image_ID']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display(test_df.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4bd98d52-3f67-4584-bb11-8c9f45a7b3a7","_cell_guid":"6e64ea4d-86c4-4ee3-b80f-754941ce11b7","trusted":true},"cell_type":"markdown","source":"### Data visualization"},{"metadata":{"_uuid":"8e8865bd-c427-49a3-9f24-638d84550828","_cell_guid":"d21261b6-4bab-47c1-a625-08d3e0f0e245","trusted":true},"cell_type":"markdown","source":"##### An additional label for any, which should always be true if any of the sub-type labels is true. We could know the number of images that have any kind of hemorrhage with this variable"},{"metadata":{"_uuid":"8df2619c-a135-424b-83e2-7f97eeeb4b3f","_cell_guid":"35ae9c12-1d28-4dcf-a5c9-4c4168d6ffe9","trusted":true},"cell_type":"code","source":"def graph_hemorrhage():\n    percentage = train_df[(train_df['Hemorrhage']=='any')&(train_df['Label']==1)]['Image_ID'].count()/train_df['Image_ID'].nunique()*100\n    print(\"percentage of images with hemorrhage : \",round(percentage,2),'%')\n\n    pd.DataFrame([percentage,100-percentage],columns=['percentage']).plot(kind='pie',\n                                                                          y='percentage',\n                                                                          labels=['Hemorrhage','Non_Hemorrhage'],\n                                                                          title='Hemorraghe distribution',\n                                                                          autopct='%.1f%%',\n                                                                          legend=False,\n                                                                          figsize=(6,6),\n                                                                          shadow=True,\n                                                                          startangle=90)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5cb7bf85-0ece-4000-85e2-2f12ef4e9310","_cell_guid":"19300a4f-79f4-4b89-b1ca-7e645073a598","trusted":true},"cell_type":"code","source":"Hemorrhage = pd.DataFrame(train_df[(train_df['Label']==1)&(train_df['Hemorrhage']!='any')]['Hemorrhage'].value_counts()).reset_index()\nHemorrhage.columns = ['Hemorrhage','Number_Pictures']\n\ndisplay(Hemorrhage)\n\nHemorrhage.plot(kind='pie',y='Number_Pictures',labels=Hemorrhage['Hemorrhage'].unique(),title='Hemorrhage distribution',\\\n               legend=False,autopct='%.1f%%',figsize=(6,6),shadow=True, startangle=90,fontsize=11)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"graph_hemorrhage()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c22fb6ea-0ca6-4c59-8733-2f06a579beed","_cell_guid":"7d4bdd63-bb42-4cde-bf4b-f83a76191852","trusted":true},"cell_type":"code","source":"display(train_df.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Transform dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove image with damaged pixels\ntrain_df = train_df[train_df['Image_ID']!='ID_6431af929']\ntrain_images.remove('ID_6431af929.dcm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def undersample(dataset): \n    ID_hemorrhage = train_df[(train_df['Hemorrhage']=='any')&(train_df['Label']==1)][['Image_ID']].reset_index(drop=True)\n    n_samples = len(ID_hemorrhage)\n    ID_Healthy = train_df[(train_df['Hemorrhage']=='any')&(train_df['Label']==0)][['Image_ID']].reset_index(drop=True).sample(n=n_samples,random_state=1)\n    \n    if dataset =='train_df':\n        result = pd.concat([train_df.merge(ID_hemorrhage,how='inner',on='Image_ID'),\\\n                            train_df.merge(ID_Healthy,how='inner',on='Image_ID')]).sample(frac=1).\\\n                            sort_values('Image_ID').reset_index(drop=True)\n    \n    elif dataset == 'train_images':\n        data_frame = pd.DataFrame(train_images,columns=['Image_ID'])\n        result = pd.concat([data_frame.merge(ID_hemorrhage+'.dcm',how='inner',on='Image_ID'),\\\n                            data_frame.merge(ID_Healthy+'.dcm',how='inner',on='Image_ID')])['Image_ID'].tolist()\n    return(result)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = undersample('train_df')\ntrain_images = undersample('train_images')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"graph_hemorrhage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pivot_df = train_df.drop_duplicates().pivot(index='Image_ID', columns='Hemorrhage', values='Label').reset_index()\npivot_df['Image_ID'] = pivot_df['Image_ID']+'.png'\ndisplay(pivot_df.head())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Prepocess Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_first_of_dicom_field_as_int(x):\n    if type(x) == pydicom.multival.MultiValue:\n        return int(x[0])\n    else:\n        return int(x)\n\ndef get_metadata(image):\n    metadata = {\n        \"window_center\": image.WindowCenter,\n        \"window_width\": image.WindowWidth,\n        \"intercept\": image.RescaleIntercept,\n        \"slope\": image.RescaleSlope\n    }\n    return {k: get_first_of_dicom_field_as_int(v) for k, v in metadata.items()}\n\ndef window_image(img, window_center, window_width, intercept, slope):\n    img = img * slope + intercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img[img < img_min] = img_min\n    img[img > img_max] = img_max\n    return img\n\ndef normalize(image):\n    min_image = image.min()\n    max_image = image.max()\n    return (image - min_image) / (max_image - min_image)\n\ndef resize(image,width,weight):\n    resized = cv2.resize(image, (width, weight))\n    return resized\n\ndef save(directory,image,image_normalized_resized):\n    save_dir = '/kaggle/tmp/'\n    if not os.path.exists(save_dir):\n        os.makedirs(save_dir)\n    \n    path = directory+image\n    new_path = save_dir + image.replace('.dcm', '.png')        \n    res = cv2.imwrite(new_path, image_normalized_resized)\n    \ndef normalize_resize_save(dataset,width,weight,directory):\n    for i in tqdm(dataset):\n        image=pydicom.read_file(directory+i)\n        image_windowed = window_image(image.pixel_array, ** get_metadata(image))\n        image_normalized_resized = resize(normalize(image_windowed),width,weight)\n        save(directory,i,image_normalized_resized)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Visualize first image in the Data Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"image=pydicom.read_file(dir_train+train_df['Image_ID'][0]+\".dcm\")\nimage_windowed = window_image(image.pixel_array, ** get_metadata(image))\n\ndisplay(image)\nplt.imshow(image_windowed, cmap=plt.cm.bone)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Visualize images with hemorraghes"},{"metadata":{"_uuid":"f69c5803-b6df-4ccd-aa1a-136912ff8793","_cell_guid":"ac567fbb-8cbf-46cc-bc27-09ed46c45e94","trusted":true},"cell_type":"code","source":"def view_images(data_frame,hemorraghe):\n    width = 5\n    height = 1\n    fig, axs = plt.subplots(height, width, figsize=(20,5))\n\n    list_hem = pd.DataFrame(train_df[(train_df['Label']==1)&(train_df['Hemorrhage']==hemorraghe)][['Image_ID']].\\\n                            head(width*height)+\".dcm\").reset_index()\n    \n    for i in range(0,width*height):\n        image=pydicom.read_file(dir_train+list_hem['Image_ID'][i])\n        image_windowed = window_image(image.pixel_array, ** get_metadata(image))\n        fig.add_subplot(height,width, i+1)\n        axs[i].set_title(list_hem['Image_ID'][i])\n        plt.imshow(image_windowed, cmap=plt.cm.bone)\n        \n    plt.suptitle(\"Images with \"+hemorraghe,fontsize = 20)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c557759-682d-4b30-8b04-33a3d55ed9c9","_cell_guid":"9da1efc6-428b-46f2-8e96-510086184bf6","trusted":true},"cell_type":"code","source":"for i in train_df['Hemorrhage'].unique():\n    view_images(train_df,i)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Normalize, resize and save new images in png format[](http://)"},{"metadata":{"trusted":true},"cell_type":"code","source":"normalize_resize_save(train_images,224,224,dir_train)\nnormalize_resize_save(test_images,224,224,dir_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Number of files resized and saved: {}\".format(len(os.listdir(\"/kaggle/tmp/\"))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ebc7a7e4-f4bf-43e9-bbec-f3da45102076","_cell_guid":"9dc8ba97-0a29-4436-ad4e-81e387b3a530","trusted":true},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import layers\nimport tensorflow as tf\nfrom keras.applications import DenseNet121\n# from keras.preprocessing.image import ImageDataGenerator\nfrom keras_preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nfrom keras_preprocessing.image import ImageDataGenerator\nimport torch\nimport keras\n\nEPOCHS = 11\nBATCH_SIZE = 32\nmodel_nn='densenet'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pick_nn(neu_net):\n    if neu_net == 'densenet':\n        neural_network = DenseNet121(\n            weights='../input/densenet-keras/DenseNet-BC-121-32-no-top.h5',\n            include_top=False,\n            input_shape=(224,224,3)\n        )\n        \n        \n    elif neu_net=='resnet':\n        neural_network = ResNet50 (\n            weights = 'imagenet',\n            include_top=False,\n            input_shape =(224,224,3)\n        )\n        \n    return(neural_network)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DataGenerator():\n    return(ImageDataGenerator(zoom_range=0.1,  # set range for random zoom\n        # set mode for filling points outside the input boundaries\n        fill_mode='constant',\n        cval=0.,  # value used for fill_mode = \"constant\"\n        horizontal_flip=True,  # randomly flip images\n        vertical_flip=True,  # randomly flip images,\n        validation_split=0.1))\n\ndef create_generator(dataset,batch_size):\n    if dataset == 'test':\n        generator = DataGenerator().flow_from_dataframe(\n                        test_df,\n                        directory='/kaggle/tmp/',\n                        x_col='Image_ID',\n                        class_mode=None,\n                        target_size=(224, 224),\n                        batch_size=batch_size,\n                        shuffle=False\n                    )\n    else:\n        generator = DataGenerator().flow_from_dataframe(dataframe=pivot_df, \n                                            directory=\"/kaggle/tmp/\",\n                                            x_col=\"Image_ID\",\n                                            y_col=['any', 'epidural', 'intraparenchymal', \n                                           'intraventricular', 'subarachnoid', 'subdural'],\n                                            class_mode='other',\n                                            target_size=(224,224),\n                                            batch_size=batch_size,\n                                            subset = dataset)\n    \n    return(generator)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(nn):\n    model = Sequential()\n    \n    model.add(pick_nn(neu_net=nn))\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Dense(6, activation='sigmoid'))\n    \n    model.compile(\n        loss='binary_crossentropy',\n        optimizer=Adam(lr=0.001),\n        metrics=['accuracy']\n    )\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(model_nn)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = ModelCheckpoint(\n    'model.h5', \n    monitor='val_loss', \n    verbose=0, \n    save_best_only=True, \n    save_weights_only=False,\n    mode='auto'\n)\n\n\nhistory = model.fit_generator(\n    create_generator(dataset = 'training', batch_size=BATCH_SIZE),\n    steps_per_epoch=3500,\n    validation_data=create_generator(dataset = 'validation', batch_size=BATCH_SIZE),\n    validation_steps=3000,\n    callbacks=[checkpoint],\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\n\nplt.plot(history_df[['loss', 'val_loss']] )\nplt.title('Value of loss')\nplt.xlabel('Number of epochs')\nplt.ylabel('Loss')\nplt.legend(('Loss_train', 'Loss_Val'))\nplt.grid()\nplt.show()\n\nplt.plot(history_df[['accuracy', 'val_accuracy']])\nplt.title('Value of accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Number of epochs')\nplt.legend(('Acc', 'Acc_Val'))\nplt.grid()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model.h5')\ncreate_data_test = create_generator(dataset = 'test',batch_size=BATCH_SIZE)\ny_test = model.predict_generator(create_data_test,\n    steps=len(create_data_test),\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = test_df.join(pd.DataFrame(y_test, columns = ['any', 'epidural', 'intraparenchymal', \n         'intraventricular', 'subarachnoid', 'subdural']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = test_df.melt(id_vars=['Image_ID'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df['ID'] = test_df.Image_ID.apply(lambda x: x.replace('.png', '')) + '_' + test_df.variable\ntest_df['Label'] = test_df['value']\n\ntest_df[['ID', 'Label']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[['ID', 'Label']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import HTML\nimport pandas as pd\nimport numpy as np\n\ndef create_download_link(title = \"Download CSV file\", filename = \"data.csv\"):  \n    html = '<a href={filename}>{title}</a>'\n    html = html.format(title=title,filename=filename)\n    return HTML(html)\n\n# create a link to download the dataframe which was saved with .to_csv method\ncreate_download_link(filename='submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"#### References\n\n[Data_Visualisation + Model ](https://www.kaggle.com/jesucristo/rsna-introduction-eda-models) <br>\n[Data_Visualisation ](https://www.kaggle.com/marcovasquez/basic-eda-data-visualization)\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}