{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pydicom\nimport os\nimport matplotlib.pyplot as plt\nimport collections\nfrom tqdm import tqdm_notebook as tqdm\nfrom datetime import datetime\n\nfrom math import ceil, floor, log\nimport cv2\nfrom PIL import Image\nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport sys\n\n# from keras_applications.resnet import ResNet50\nfrom keras.applications.inception_v3 import InceptionV3\n\nfrom sklearn.model_selection import ShuffleSplit\n\ninput_path = \"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/\"\ntest_images_dir = input_path + 'stage_2_test/'\ntrain_images_dir = input_path + 'stage_2_train/'","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:23:50.272266Z","iopub.execute_input":"2023-07-28T20:23:50.272551Z","iopub.status.idle":"2023-07-28T20:23:56.267484Z","shell.execute_reply.started":"2023-07-28T20:23:50.272525Z","shell.execute_reply":"2023-07-28T20:23:56.266343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def correct_dcm(dcm):\n    x = dcm.pixel_array + 1000\n    px_mode = 4096\n    x[x>=px_mode] = x[x>=px_mode] - px_mode\n    dcm.PixelData = x.tobytes()\n    dcm.RescaleIntercept = -1000\n\ndef window_image(dcm, window_center, window_width):\n    \n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        correct_dcm(dcm)\n    \n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef bsb_window(dcm,extract_bone = False):\n    brain_img = window_image(dcm, 40, 80)\n    subdural_img = window_image(dcm, 80, 200)\n    soft_img = window_image(dcm, 40, 380)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    \n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n    return bsb_img\n\ndef matrix2image(mat, shape = (224,224)): \n    \"\"\"\n    Gets a 2D np.array with float values.\n    Maps the values between 0 - 255.\n    \"\"\"\n    epsilon = 10e-8\n    mat -= mat.min()\n    mat = mat / (mat.max() + epsilon)\n    mat *= 255\n    mat = np.round(mat, decimals=0)\n    mat = np.array(mat, dtype=\"uint8\")\n    \n    \n    resized = cv2.resize(mat,shape)\n    out_img = Image.fromarray(resized)\n    return out_img\n# Sanity Check\n# Example dicoms: ID_2669954a7, ID_5c8b5d701, ID_52c9913b1\n\ndicom = pydicom.dcmread(os.path.join(train_images_dir,'ID_00005679d.dcm'))\n#                                     ID  Label\n# 4045566          ID_5c8b5d701_epidural      0\n# 4045567  ID_5c8b5d701_intraparenchymal      1\n# 4045568  ID_5c8b5d701_intraventricular      0\n# 4045569      ID_5c8b5d701_subarachnoid      1\n# 4045570          ID_5c8b5d701_subdural      1\n# 4045571               ID_5c8b5d701_any      1\nplt.imshow(bsb_window(dicom), cmap=plt.cm.bone);\n","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:23:56.269367Z","iopub.execute_input":"2023-07-28T20:23:56.269731Z","iopub.status.idle":"2023-07-28T20:23:56.514298Z","shell.execute_reply.started":"2023-07-28T20:23:56.269695Z","shell.execute_reply":"2023-07-28T20:23:56.51358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def window_with_correction(dcm, window_center, window_width):\n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        correct_dcm(dcm)\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\ndef window_without_correction(dcm, window_center, window_width):\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept\n    img_min = window_center - window_width // 2\n    img_max = window_center + window_width // 2\n    img = np.clip(img, img_min, img_max)\n    return img\n\n##################################################\ndef window_testing(img, window):\n    brain_img = window(img, 40, 80)\n    subdural_img = window(img, 80, 200)\n    soft_img = window(img, 40, 380)\n    \n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1,2,0)\n\n    return bsb_img\n'''\n##################################################\n# example of a \"bad data point\" (i.e. (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100) == True)\ndicom = pydicom.dcmread(train_images_dir + \"ID_036db39b7\" + \".dcm\")\nprint(dicom)\nfig, ax = plt.subplots(1, 2)\n\nax[0].imshow(window_testing(dicom, window_without_correction), cmap=plt.cm.bone);\nax[0].set_title(\"original\")\nax[1].imshow(window_testing(dicom, window_with_correction), cmap=plt.cm.bone);\nax[1].set_title(\"corrected\");\n#############################\n'''","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:23:56.515337Z","iopub.execute_input":"2023-07-28T20:23:56.51566Z","iopub.status.idle":"2023-07-28T20:23:56.529043Z","shell.execute_reply.started":"2023-07-28T20:23:56.515619Z","shell.execute_reply":"2023-07-28T20:23:56.5281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_dir = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv'\ntrain = pd.read_csv('../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:23:56.530217Z","iopub.execute_input":"2023-07-28T20:23:56.530523Z","iopub.status.idle":"2023-07-28T20:24:00.583986Z","shell.execute_reply.started":"2023-07-28T20:23:56.530492Z","shell.execute_reply":"2023-07-28T20:24:00.582802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_jpeg(path,shape =(224,224)):\n    dcm = pydicom.dcmread(path)\n    array = bsb_window(dcm)\n    img = matrix2image(array, shape = shape)\n    return img\nfrom tqdm import tqdm\nimport random\n\npath = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'\n\ndef get_brains(path, data_frame,number_of_brain):\n    dcm_ids = dict()\n    \n    sampled_data = data_frame.sample(number_of_brain)\n    random_sampled_data_ids = sampled_data.id\n    for image_id in tqdm(random_sampled_data_ids):\n        dcm = pydicom.dcmread(os.path.join(path,image_id+'.dcm'))\n        array = bsb_window(dcm)\n        dcm_ids[image_id] = array.sum()\n    return dcm_ids\n        \n","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:00.586192Z","iopub.execute_input":"2023-07-28T20:24:00.586416Z","iopub.status.idle":"2023-07-28T20:24:00.594351Z","shell.execute_reply.started":"2023-07-28T20:24:00.586392Z","shell.execute_reply":"2023-07-28T20:24:00.593497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\ndef visualize(dcm_id,shape =(224,224)):\n    path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train'\n    dcm = pydicom.dcmread(os.path.join(path,dcm_id+'.dcm'))\n    array = bsb_window(dcm)\n    img = matrix2image(array, shape = shape)\n    plt.show(img)\n    return img\nvisualize('ID_d193c4638')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:00.596011Z","iopub.execute_input":"2023-07-28T20:24:00.596391Z","iopub.status.idle":"2023-07-28T20:24:00.669039Z","shell.execute_reply.started":"2023-07-28T20:24:00.596359Z","shell.execute_reply":"2023-07-28T20:24:00.66783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\ndef get_training_jpegs(df,label = 'healthy', image_size = (224,224)):\n    if label not in os.listdir('./'):\n        os.mkdir(f'./{label}')\n    dcm_ids = df.id\n    for ids in tqdm(dcm_ids):\n        path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'+str(ids)+'.dcm'\n        img = get_jpeg(path , shape = image_size)\n        img_name = f\"./{label}/{ids}.jpeg\"\n        img.save(img_name)\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:00.670145Z","iopub.execute_input":"2023-07-28T20:24:00.670352Z","iopub.status.idle":"2023-07-28T20:24:00.676957Z","shell.execute_reply.started":"2023-07-28T20:24:00.670329Z","shell.execute_reply":"2023-07-28T20:24:00.675509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"get_training_jpegs(df_sampled_healthy_brains, label = 'healthy')\n","metadata":{"execution":{"iopub.status.busy":"2023-07-02T16:25:27.856797Z","iopub.status.idle":"2023-07-02T16:25:27.857334Z"}}},{"cell_type":"markdown","source":"!tar -zcvf ctscans.tar.gz /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2022-04-03T13:56:42.699116Z","iopub.execute_input":"2022-04-03T13:56:42.699726Z"}}},{"cell_type":"code","source":"def get_jpeg(path,shape =(299,299)):\n    try:\n        dcm = pydicom.dcmread(path)\n        array = bsb_window(dcm)\n        img = matrix2image(array, shape = shape)\n    except:\n        return \n    return img","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:00.678563Z","iopub.execute_input":"2023-07-28T20:24:00.678912Z","iopub.status.idle":"2023-07-28T20:24:00.688521Z","shell.execute_reply.started":"2023-07-28T20:24:00.678875Z","shell.execute_reply":"2023-07-28T20:24:00.687887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import os\ndef good_slice_data(number_of_images = 10 ,labels = labels, shape = (299,299),directories = False):   \n    if not directories:\n        os.makedirs('./train/1')\n        os.makedirs('./train/0')\n    for label in labels:\n        for i in range(number_of_images):\n            subtype = str(label['any'].iloc[i])\n            ID = label.Image.iloc[i]\n            path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'+ID+'.dcm'\n            img = get_jpeg(path , shape = shape)\n            img_name = f\"./train/{subtype}/{ID}.jpeg\"\n            #print(img_name)\n            img.save(img_name)\n            if i % 10000 == 0:\n                print(i)\n        print('done converting ',subtype)\n","metadata":{"execution":{"iopub.status.busy":"2022-04-03T12:32:27.842648Z","iopub.status.idle":"2022-04-03T12:32:27.843014Z"}}},{"cell_type":"markdown","source":"good_slice_data(number_of_images = 50000 ,labels = labels, shape = (299,299),directories = False)","metadata":{"execution":{"iopub.status.busy":"2022-04-03T12:32:27.843898Z","iopub.status.idle":"2022-04-03T12:32:27.844267Z"}}},{"cell_type":"code","source":"df = pd.read_pickle(\"/kaggle/input/dataset/df_one_hot.pkl\")\ndf","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:00.689631Z","iopub.execute_input":"2023-07-28T20:24:00.689971Z","iopub.status.idle":"2023-07-28T20:24:02.361021Z","shell.execute_reply.started":"2023-07-28T20:24:00.689931Z","shell.execute_reply":"2023-07-28T20:24:02.360001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"types = ['epidural','intraparenchymal','intraventricular','subarachnoid','subdural','no_hemorrhage']","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:02.361945Z","iopub.execute_input":"2023-07-28T20:24:02.362178Z","iopub.status.idle":"2023-07-28T20:24:02.367098Z","shell.execute_reply.started":"2023-07-28T20:24:02.362154Z","shell.execute_reply":"2023-07-28T20:24:02.365465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"import os\n\nfrom tqdm import tqdm\ndef create_training_images(bone = True ,df = df,types = types, shape = (224,224),create_directories = True):   \n    \n    if create_directories:\n        #Create jpegs from given data frames \n        #ex:\n        #labels = [intraventricular,subdural,intraparenchymal,subarachnoid] \n        #label name should be same as the hemorrhage type\n#         os.makedirs('./train/epidural')\n#         os.makedirs('./train/intraparenchymal')\n#         os.makedirs('./train/intraventricular')\n#         os.makedirs('./train/subdural')\n#         os.makedirs('./train/subarachnoid')\n#         os.makedirs('./train/no_hemorrhage')\n        os.makedirs('./train/no_hemorrhage')\n        os.makedirs('./train/any_hemorrhage')\n    for type_ in types:\n        temp = df[df[type_] ==  1] \n        for i in tqdm(temp.index):\n            ID = temp.Image.loc[i]\n            path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'+ID+'.dcm'\n            img = get_jpeg(path , shape = shape)\n            img_name = f\"./train/{type_}/{ID}.jpeg\"\n            #print(img_name)\n            img.save(img_name)\n            if i % 10000 == 0:\n                print(i)\n        print('done converting ',type_)\n    print('done!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-25T13:51:37.666306Z","iopub.execute_input":"2023-07-25T13:51:37.666788Z","iopub.status.idle":"2023-07-25T13:51:37.681623Z","shell.execute_reply.started":"2023-07-25T13:51:37.66675Z","shell.execute_reply":"2023-07-25T13:51:37.680179Z"}}},{"cell_type":"code","source":"import os\n\nfrom tqdm import tqdm\ndef create_binary_training_images(bone = True ,df = df, shape = (299,299),create_directories = True):   \n    \n    if create_directories:\n        #Create jpegs from given data frames \n        #ex:\n        #labels = [intraventricular,subdural,intraparenchymal,subarachnoid] \n        #label name should be same as the hemorrhage type\n#         os.makedirs('./train/epidural')\n#         os.makedirs('./train/intraparenchymal')\n#         os.makedirs('./train/intraventricular')\n#         os.makedirs('./train/subdural')\n#         os.makedirs('./train/subarachnoid')\n#         os.makedirs('./train/no_hemorrhage')\n        os.makedirs('./train/0')\n        os.makedirs('./train/1')\n    for type_ in range(2):\n        temp = df[df['any'] ==  type_] \n        for i in tqdm(temp.index):\n            ID = temp.Image.loc[i]\n            path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/'+ID+'.dcm'\n            \n            img = get_jpeg(path , shape = shape)\n            if img is None:\n                continue\n            img_name = f\"./train/{type_}/{ID}.jpeg\"\n            #print(img_name)\n            img.save(img_name)\n            if i % 100000 == 0: \n                print(i)\n        print('done converting ',type_)\n    print('done!!!')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:02.368167Z","iopub.execute_input":"2023-07-28T20:24:02.368436Z","iopub.status.idle":"2023-07-28T20:24:02.377474Z","shell.execute_reply.started":"2023-07-28T20:24:02.368413Z","shell.execute_reply":"2023-07-28T20:24:02.376876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_binary_training_images(create_directories = True)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T20:24:02.37853Z","iopub.execute_input":"2023-07-28T20:24:02.378797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tar -zcvf ctscans.tar.gz /kaggle/working/train/","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}