{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%reload_ext autoreload\n%autoreload 2\n%matplotlib inline\nimport pydicom\nimport os\nfrom os import listdir\nfrom os.path import isfile, join\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot, cm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\n\n\nimport fastai\nfrom fastai.vision import *\n\nfrom IPython.display import FileLink\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, let's take a look at the training data file. "},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '../input/rsna-intracranial-hemorrhage-detection/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw = pd.read_csv(path + 'stage_1_train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"According to the dataset information, the ID column includes the patient's ID and the probability (0-1) of each of the 6 types of intracranial hemorrhages occuring. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw[0:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see the types cycle every 6 rows. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's create a new dataframe that formats this training data more intuitively. The first step to this is splitting up the ID column using the underscores. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_raw['Sub_type'] = train_raw['ID'].str.split(\"_\", n = 3, expand = True)[2]\ntrain_raw['PatientID'] = train_raw['ID'].str.split(\"_\", n = 3, expand = True)[1]\ntrain_raw['PatientID'] = 'ID_' + train_raw['PatientID']\ntrain_raw","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The new DataFrame will have a column of probabilities for each type of hemorrhage for each patient. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.DataFrame()\ntrain = train_raw.drop_duplicates()\ntrain = train.pivot(index = 'PatientID', columns = 'Sub_type', values = 'Label')\ntrain.reset_index(level=0, inplace=True)\ntrain.columns.name = None\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['labels'] = ''\ntrain.loc[train['any'] == 1,'labels'] += 'any'\ntrain.loc[train['epidural'] == 1,'labels'] += ',epidural'\ntrain.loc[train['intraparenchymal'] == 1,'labels'] += ',intraparenchymal'\ntrain.loc[train['intraventricular'] == 1,'labels'] += ',intraventricular'\ntrain.loc[train['subarachnoid'] == 1,'labels'] += ',subarachnoid'\ntrain.loc[train['subdural'] == 1,'labels'] += ',subdural'\ntrain","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's create a bar plot to see the frequency of different types of hemorrhages appearing. "},{"metadata":{"trusted":true},"cell_type":"code","source":"#nonzero = np.count_nonzero(train, axis=0)\ng = sns.barplot(train.columns, np.count_nonzero(train, axis = 0))\ng.set_xticklabels(g.get_xticklabels(), rotation=40, ha=\"right\")\nplt.title('Frequncies of Hemorrhage Types')\nplt.xlabel('Hemorrhage Type')\nplt.ylabel('Nonzero Instances')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, let's take a look at the images themselves. "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images_path = (path + 'stage_1_train_images/')\ntrain_images = [f for f in listdir(train_images_path) if isfile(join(train_images_path, f))]\ntest_images_path = (path + 'stage_1_test_images/')\ntest_images = [f for f in listdir(test_images_path) if isfile(join(test_images_path, f))]\nprint('5 Training images', train_images[:5]) # Print the first 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Number of train images:', len(train_images))\nprint('Number of test images:', len(test_images))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image = pydicom.dcmread(train_images_path + 'ID_ffff922b9.dcm')\nplt.imshow(image.pixel_array)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now that we're able to plot one image, let's make a function to plot multiple images side by side (capped at 12). "},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_dcm(IDs):\n    if (type(IDs) == str):\n        IDs = [IDs]\n    if len(IDs) > 12:\n        raise Exception('Number of images should not exceed 12. The number of images was: {}'.format(len(IDs)))\n    fig = plt.figure(figsize = (15, 10))\n    columns = 4\n    rows = 3\n    index = 1\n    for ID in IDs: \n        if (not ID.endswith('.dcm')):\n            ID = ID + '.dcm'\n        image = pydicom.dcmread(train_images_path + ID)\n        fig.add_subplot(rows, columns, index)\n        plt.imshow(image.pixel_array)\n        fig.add_subplot\n        index += 1\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"First, we'll just plot the random first ten images."},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_dcm(train_images[12:24])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, let's plot some of each kind of hemorrhage. "},{"metadata":{"trusted":true},"cell_type":"code","source":"certain_cases = train.sort_values('epidural', ascending = False)\nplot_dcm(certain_cases.PatientID[0:12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"certain_cases = train.sort_values('subdural', ascending = False)\nplot_dcm(certain_cases.PatientID[0:12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"certain_cases = train.sort_values('subarachnoid', ascending = False)\nplot_dcm(certain_cases.PatientID[0:12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"certain_cases = train.sort_values('intraventricular', ascending = False)\nplot_dcm(certain_cases.PatientID[0:12])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"certain_cases = train.sort_values('intraparenchymal', ascending = False)\nplot_dcm(certain_cases.PatientID[0:12])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As we can see, the difference in hemorrhage types is difficult to discern for the untrained eye. This is because we're looking at the raw images instead of the Hounsfield Units which the dicom images are scaled with. "},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/omission/eda-view-dicom-images-with-correct-windowing\n\ndef window_image(img, window_center,window_width, intercept, slope):\n\n    img = (img*slope +intercept)\n    img_min = window_center - window_width//2\n    img_max = window_center + window_width//2\n    img[img<img_min] = img_min\n    img[img>img_max] = img_max\n    return img\n\ndef get_first_of_dicom_field_as_int(x):\n    #get x[0] as in int is x is a 'pydicom.multival.MultiValue', otherwise get int(x)\n    if type(x) == pydicom.multival.MultiValue:\n        return int(x[0])\n    else:\n        return int(x)\n\ndef get_windowing(data):\n    dicom_fields = [data[('0028','1050')].value, #window center\n                    data[('0028','1051')].value, #window width\n                    data[('0028','1052')].value, #intercept\n                    data[('0028','1053')].value] #slope\n    return [get_first_of_dicom_field_as_int(x) for x in dicom_fields]\n\ndef new_open_image(path, div=True, convert_mode=None, after_open=None):\n    dcm = pydicom.dcmread(str(path))\n    window_center, window_width, intercept, slope = get_windowing(dcm)\n    im = window_image(dcm.pixel_array, window_center, window_width, intercept, slope)\n    im = np.stack((im,)*3, axis=-1)\n    im -= im.min()\n    im_max = im.max()\n    if im_max != 0: im = im / im.max()\n    x = Image(pil2tensor(im, dtype=np.float32))\n    #if div: x.div_(2048)  # ??\n    return x\n\n\nvision.data.open_image = new_open_image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"certainly_affected = train.sort_values('any', ascending = False).filter(['PatientID', 'labels'])[:10000]\ncertainly_affected.reset_index(drop = True, inplace=True)\n\ncertainly_unaffected = train.sort_values('any', ascending = True).filter(['PatientID', 'labels'])[:10000]#.PatientID[:10000]\ncertainly_unaffected.reset_index(drop = True, inplace=True)\n\ntrain_subset = pd.concat([certainly_affected, certainly_unaffected], axis = 0)\ntrain_subset['PatientID'] += '.dcm'\ntrain_subset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\nim_list = ImageList.from_df(train_subset, path = (path + \"stage_1_train_images\"))\ntest_fnames = pd.DataFrame(\"ID_\" + pd.read_csv(path + \"stage_1_sample_submission.csv\")[\"ID\"].str.split(\"_\", n = 2, expand = True)[1].unique() + \".dcm\")\ntest_im_list = ImageList.from_df(test_fnames, path = (path + \"stage_1_test_images\"))\n\ntfms = get_transforms(do_flip = False)\n\ndata = (im_list.split_by_rand_pct(0.2)\n               .label_from_df(label_delim=\",\")\n               .transform(tfms, size = 512)\n               .add_test(test_im_list)\n               .databunch(bs = batch_size, num_workers = 0)\n               .normalize())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc_02 = partial(accuracy_thresh, thresh=0.2)\nf_score = partial(fbeta, thresh=0.2)\n\nlearn = cnn_learner(data, models.resnet18, metrics = [acc_02, f_score, accuracy_thresh], model_dir = '/kaggle/working/models')\n\n#models_path = Path(\"/kaggle/working/models\")\n#if not models_path.exists(): models_path.mkdir()\n    \n#learn.model_dir = models_path\n#learn.metrics = [accuracy_thresh]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()\nlearn.recorder.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-3\nlearn.freeze()\nlearn.fit_one_cycle(3, slice(lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/rsna-intracranial-hemorrhage-detection/stage_1_sample_submission.csv')\nsubmission['fn'] = submission.ID.apply(lambda x: '_'.join(x.split('_')[:2]) + '.dcm')\nsubmission['label'] = submission.ID.apply(lambda x: x.split('_')[-1])\n\npivot_test = submission.pivot(index='fn', columns='label', values='Label')\npivot_test.reset_index(inplace=True)\npivot_test['MultiLabel'] = \" \"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test_path = \"../input/rsna-intracranial-hemorrhage-detection/stage_1_test_images/\"\ndata_test = (ImageList\n.from_df(path=test_images_path,df= pivot_test[['fn', 'MultiLabel']])\n.split_by_rand_pct(valid_pct= 0)\n.label_from_df(cols=1, label_delim = \" \")\n.transform(size=(128,128))\n.databunch()\n.normalize(imagenet_stats))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_classes = learn.data.classes\nlearn.data = data_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_predict, _ = learn.get_preds(ds_type=DatasetType.Fix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert len(y_predict) == pivot_test.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pivot_test['MultiLabel']  = y_predict","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, col in enumerate(data_classes):\n    print(col, end= \", \")\n    pivot_test[col] = pivot_test['MultiLabel'].apply(lambda x: x[i].numpy())\n    \ncols_to_consider = [col for col in pivot_test.columns if not col in ['None', 'MultiLabel']]\nprint(cols_to_consider)\ndf_temp = pd.melt(pivot_test[cols_to_consider], id_vars= ['fn'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_temp['ID'] = df_temp['fn'].apply(lambda x: x[:-4])\ndf_temp['ID'] = df_temp[['ID', 'label']].apply(lambda x: \"_\".join(x), axis = 1)\n\nassert len(set(submission.ID.unique()).intersection(set(df_temp.ID.unique()))) == submission.shape[0]\n\ndf_temp.rename(columns={'value':'Label'}, inplace = True)\ndf_temp[['ID', 'Label']].to_csv(\"submission1.gz\", compression = 'gzip' , index = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink('submission1.gz')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}