{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# RSNA : EDA ","metadata":{"papermill":{"duration":0.024092,"end_time":"2022-10-27T03:55:38.15713","exception":false,"start_time":"2022-10-27T03:55:38.133038","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nos.chdir(\"../input/rsna-2022-cervical-spine-fracture-detection\")\n! pip install -qU \"python-gdcm\" pydicom pylibjpeg \"opencv-python-headless\"","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":24.092165,"end_time":"2022-10-27T03:56:02.312749","exception":false,"start_time":"2022-10-27T03:55:38.220584","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:30:38.145202Z","iopub.execute_input":"2023-02-07T11:30:38.145677Z","iopub.status.idle":"2023-02-07T11:30:58.7982Z","shell.execute_reply.started":"2023-02-07T11:30:38.145587Z","shell.execute_reply":"2023-02-07T11:30:58.796333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib import animation\nfrom IPython.display import HTML\nimport seaborn as sns\nsns.set_theme(style=\"dark\")\nimport cv2\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly.offline import init_notebook_mode, iplot, plot\nplt.ion()\n","metadata":{"papermill":{"duration":2.328788,"end_time":"2022-10-27T03:56:04.662552","exception":false,"start_time":"2022-10-27T03:56:02.333764","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:34:53.155576Z","iopub.execute_input":"2023-02-07T11:34:53.156035Z","iopub.status.idle":"2023-02-07T11:34:55.099291Z","shell.execute_reply.started":"2023-02-07T11:34:53.155986Z","shell.execute_reply":"2023-02-07T11:34:55.097809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# For rotating images\nfrom skimage.transform import rotate\nimport matplotlib.patches as patches","metadata":{"papermill":{"duration":0.772739,"end_time":"2022-10-27T03:56:05.457106","exception":false,"start_time":"2022-10-27T03:56:04.684367","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:35:07.832147Z","iopub.execute_input":"2023-02-07T11:35:07.832586Z","iopub.status.idle":"2023-02-07T11:35:08.057443Z","shell.execute_reply.started":"2023-02-07T11:35:07.832552Z","shell.execute_reply":"2023-02-07T11:35:08.056216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport random\n\n\n# Files lecture\nimport pydicom as dcm\nfrom pydicom.data import get_testdata_file\nfrom pydicom import dcmread\n","metadata":{"papermill":{"duration":0.032678,"end_time":"2022-10-27T03:56:05.512441","exception":false,"start_time":"2022-10-27T03:56:05.479763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:35:13.1788Z","iopub.execute_input":"2023-02-07T11:35:13.17918Z","iopub.status.idle":"2023-02-07T11:35:13.403891Z","shell.execute_reply.started":"2023-02-07T11:35:13.179149Z","shell.execute_reply":"2023-02-07T11:35:13.402697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding Dataframe","metadata":{"papermill":{"duration":0.020648,"end_time":"2022-10-27T03:56:05.721595","exception":false,"start_time":"2022-10-27T03:56:05.700947","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trainDF = pd.read_csv('/kaggle/input/rsna-2022-cervical-spine-fracture-detection/train.csv')\nsegmentationsFilesPath = \"segmentations\"\ntrainFilesPath = \"train_images\"\ntestFilesPath = \"test_images\"","metadata":{"papermill":{"duration":0.047715,"end_time":"2022-10-27T03:56:05.790319","exception":false,"start_time":"2022-10-27T03:56:05.742604","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:35:52.893353Z","iopub.execute_input":"2023-02-07T11:35:52.8938Z","iopub.status.idle":"2023-02-07T11:35:52.906304Z","shell.execute_reply.started":"2023-02-07T11:35:52.893765Z","shell.execute_reply":"2023-02-07T11:35:52.905179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now having the data into a dataframe, we are going to analize the situation","metadata":{"papermill":{"duration":0.021039,"end_time":"2022-10-27T03:56:05.832668","exception":false,"start_time":"2022-10-27T03:56:05.811629","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trainDF.tail()","metadata":{"papermill":{"duration":0.046561,"end_time":"2022-10-27T03:56:05.901184","exception":false,"start_time":"2022-10-27T03:56:05.854623","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:35:56.079846Z","iopub.execute_input":"2023-02-07T11:35:56.080603Z","iopub.status.idle":"2023-02-07T11:35:56.10556Z","shell.execute_reply.started":"2023-02-07T11:35:56.080554Z","shell.execute_reply":"2023-02-07T11:35:56.104526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Columns understanding\n\n\n**StudyInstanceUID**\nId generated for each patient\n\n**Patient Overall**\nThis column is going to say us if some vertebrae is damaged\n\n**Vertebraes columns:**\nc1 - c7","metadata":{"papermill":{"duration":0.022285,"end_time":"2022-10-27T03:56:06.121828","exception":false,"start_time":"2022-10-27T03:56:06.099543","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Seeing data","metadata":{"papermill":{"duration":0.021505,"end_time":"2022-10-27T03:56:06.165746","exception":false,"start_time":"2022-10-27T03:56:06.144241","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trainDF.describe()","metadata":{"papermill":{"duration":0.075307,"end_time":"2022-10-27T03:56:06.262578","exception":false,"start_time":"2022-10-27T03:56:06.187271","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:36:04.07122Z","iopub.execute_input":"2023-02-07T11:36:04.071677Z","iopub.status.idle":"2023-02-07T11:36:04.126575Z","shell.execute_reply.started":"2023-02-07T11:36:04.071639Z","shell.execute_reply":"2023-02-07T11:36:04.125307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this case we have the data from 2019 patients","metadata":{"papermill":{"duration":0.021464,"end_time":"2022-10-27T03:56:06.305774","exception":false,"start_time":"2022-10-27T03:56:06.28431","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trainDF.duplicated()","metadata":{"papermill":{"duration":0.039822,"end_time":"2022-10-27T03:56:06.367486","exception":false,"start_time":"2022-10-27T03:56:06.327664","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:36:10.553654Z","iopub.execute_input":"2023-02-07T11:36:10.554362Z","iopub.status.idle":"2023-02-07T11:36:10.567343Z","shell.execute_reply.started":"2023-02-07T11:36:10.554325Z","shell.execute_reply":"2023-02-07T11:36:10.566064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We don't have any duplicated data.","metadata":{"papermill":{"duration":0.021811,"end_time":"2022-10-27T03:56:06.411952","exception":false,"start_time":"2022-10-27T03:56:06.390141","status":"completed"},"tags":[]}},{"cell_type":"code","source":"trainDF.info()","metadata":{"papermill":{"duration":0.042069,"end_time":"2022-10-27T03:56:06.476015","exception":false,"start_time":"2022-10-27T03:56:06.433946","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:36:17.555827Z","iopub.execute_input":"2023-02-07T11:36:17.556264Z","iopub.status.idle":"2023-02-07T11:36:17.577297Z","shell.execute_reply.started":"2023-02-07T11:36:17.556211Z","shell.execute_reply":"2023-02-07T11:36:17.57574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,10))\nsns.heatmap(trainDF.corr(), annot=True, cmap=\"viridis\", vmax=1)","metadata":{"papermill":{"duration":0.722842,"end_time":"2022-10-27T03:56:07.221591","exception":false,"start_time":"2022-10-27T03:56:06.498749","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:37:06.684Z","iopub.execute_input":"2023-02-07T11:37:06.684458Z","iopub.status.idle":"2023-02-07T11:37:07.271408Z","shell.execute_reply.started":"2023-02-07T11:37:06.684421Z","shell.execute_reply":"2023-02-07T11:37:07.270506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are not any interesting correlation between the data but we can see the most frecuent fractures when it exists in a patient, in this case:\n* C7\n* C6\n* C2","metadata":{"papermill":{"duration":0.022425,"end_time":"2022-10-27T03:56:07.267443","exception":false,"start_time":"2022-10-27T03:56:07.245018","status":"completed"},"tags":[]}},{"cell_type":"code","source":"labels = ['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']","metadata":{"papermill":{"duration":0.032984,"end_time":"2022-10-27T03:56:07.324089","exception":false,"start_time":"2022-10-27T03:56:07.291105","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:37:12.385812Z","iopub.execute_input":"2023-02-07T11:37:12.386937Z","iopub.status.idle":"2023-02-07T11:37:12.392346Z","shell.execute_reply.started":"2023-02-07T11:37:12.386886Z","shell.execute_reply":"2023-02-07T11:37:12.39074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meltDF = pd.melt(trainDF, id_vars=[\"StudyInstanceUID\"], var_name=\"vertebrae\", value_name=\"fractured\")","metadata":{"papermill":{"duration":0.037069,"end_time":"2022-10-27T03:56:07.383697","exception":false,"start_time":"2022-10-27T03:56:07.346628","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:37:18.906285Z","iopub.execute_input":"2023-02-07T11:37:18.907292Z","iopub.status.idle":"2023-02-07T11:37:18.916889Z","shell.execute_reply.started":"2023-02-07T11:37:18.907251Z","shell.execute_reply":"2023-02-07T11:37:18.915783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meltDF","metadata":{"execution":{"iopub.status.busy":"2023-02-07T11:37:35.286166Z","iopub.execute_input":"2023-02-07T11:37:35.286647Z","iopub.status.idle":"2023-02-07T11:37:35.30384Z","shell.execute_reply.started":"2023-02-07T11:37:35.28661Z","shell.execute_reply":"2023-02-07T11:37:35.302391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots( figsize=(24, 12))\nsns.countplot(x=\"vertebrae\",hue=\"fractured\", data= meltDF, ax=ax)\nax.legend(title='Labels', loc='upper right', labels=[\"Healthy\", \"With Fractures\"])\nax.set_title(\"Data\");\nax.bar_label(ax.containers[0])\nax.bar_label(ax.containers[1])\n","metadata":{"papermill":{"duration":0.529411,"end_time":"2022-10-27T03:56:07.935218","exception":false,"start_time":"2022-10-27T03:56:07.405807","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:41:35.900639Z","iopub.execute_input":"2023-02-07T11:41:35.901029Z","iopub.status.idle":"2023-02-07T11:41:36.284358Z","shell.execute_reply.started":"2023-02-07T11:41:35.900998Z","shell.execute_reply":"2023-02-07T11:41:36.283118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this case we have more healthy vertebraes examples than with fractures:\nThe first bar group show us that we have 961 patients with fractures and 1058 without fractures.","metadata":{"papermill":{"duration":0.023574,"end_time":"2022-10-27T03:56:07.982999","exception":false,"start_time":"2022-10-27T03:56:07.959425","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Understanding what are the files DCM(train folders) and NII(segmentations folder)","metadata":{"papermill":{"duration":0.022809,"end_time":"2022-10-27T03:56:08.028994","exception":false,"start_time":"2022-10-27T03:56:08.006185","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"According to the page FileInfo DCM is equals to DICOM and means Digital Imaging and Communications in Medicine and it contains an image and another descriptions about the patient. [(DCM File Extension, 2022)](https://fileinfo.com/extension/dcm)\n\nSome of the important keys in the object are:\n\n* ImagePositionPatient\n* SliceLocation\n* pixel_array\n* RescaleSlope\n* RescaleIntercept\n* ImageOrientationPatient\n\nInto the file we can see the information below:\n","metadata":{"papermill":{"duration":0.023623,"end_time":"2022-10-27T03:56:08.076631","exception":false,"start_time":"2022-10-27T03:56:08.053008","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ds = dcmread(\"train_images/1.2.826.0.1.3680043.10001/101.dcm\")\nprint(ds)","metadata":{"papermill":{"duration":0.050207,"end_time":"2022-10-27T03:56:08.150492","exception":false,"start_time":"2022-10-27T03:56:08.100285","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:42:13.100188Z","iopub.execute_input":"2023-02-07T11:42:13.100662Z","iopub.status.idle":"2023-02-07T11:42:13.125059Z","shell.execute_reply.started":"2023-02-07T11:42:13.100627Z","shell.execute_reply":"2023-02-07T11:42:13.12381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Adding Path Files\ntrainDF[\"patientPathFiles\"] = trainDF[\"StudyInstanceUID\"].apply(lambda x: os.path.join( trainFilesPath, x))\n\n## Counting Paths per patient\ntrainDF[\"patientPathFilesLen\"] = trainDF[\"patientPathFiles\"].apply(lambda x: len(os.listdir(x)))\ntrainDF.head()","metadata":{"papermill":{"duration":112.658421,"end_time":"2022-10-27T03:58:00.83272","exception":false,"start_time":"2022-10-27T03:56:08.174299","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:42:21.908107Z","iopub.execute_input":"2023-02-07T11:42:21.908555Z","iopub.status.idle":"2023-02-07T11:45:47.829839Z","shell.execute_reply.started":"2023-02-07T11:42:21.908519Z","shell.execute_reply":"2023-02-07T11:45:47.828561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.histogram(trainDF, x = \"patientPathFilesLen\")","metadata":{"papermill":{"duration":1.118028,"end_time":"2022-10-27T03:58:01.976328","exception":false,"start_time":"2022-10-27T03:58:00.8583","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:46:54.027468Z","iopub.execute_input":"2023-02-07T11:46:54.027968Z","iopub.status.idle":"2023-02-07T11:46:55.287197Z","shell.execute_reply.started":"2023-02-07T11:46:54.027932Z","shell.execute_reply":"2023-02-07T11:46:55.286121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In the previous shown chart we can see the distribution of number of slices per patient, it is so important to understand and take measures about the preprocessing data before training.","metadata":{"papermill":{"duration":0.024136,"end_time":"2022-10-27T03:58:02.024596","exception":false,"start_time":"2022-10-27T03:58:02.00046","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"### Let's plot random random slices","metadata":{"papermill":{"duration":0.02377,"end_time":"2022-10-27T03:58:02.072863","exception":false,"start_time":"2022-10-27T03:58:02.049093","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"For ploting the next images is neccesary to install two librarys, ","metadata":{"papermill":{"duration":0.023868,"end_time":"2022-10-27T03:58:02.121276","exception":false,"start_time":"2022-10-27T03:58:02.097408","status":"completed"},"tags":[]}},{"cell_type":"code","source":"plt.figure(figsize= (20, 20))\nfor i in range (1, 7):\n    \n    ## Choosing a random example\n    number = random.randint(0, 2018)\n    RandomPatient = trainDF[\"patientPathFiles\"][number]\n    ChosenFile = random.choice(os.listdir(RandomPatient))\n    Path = os.path.join( RandomPatient, ChosenFile)\n    randomSliceFile = dcmread(Path)\n    \n    #ploting images\n    plt.subplot(6,3,i)\n    plt.style.use('default')\n    plt.imshow(randomSliceFile.pixel_array)\n    plt.title(randomSliceFile.pixel_array.shape)\n\nplt.show()","metadata":{"papermill":{"duration":1.063595,"end_time":"2022-10-27T03:58:03.208777","exception":false,"start_time":"2022-10-27T03:58:02.145182","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:47:24.78679Z","iopub.execute_input":"2023-02-07T11:47:24.787217Z","iopub.status.idle":"2023-02-07T11:47:25.887762Z","shell.execute_reply.started":"2023-02-07T11:47:24.787174Z","shell.execute_reply":"2023-02-07T11:47:25.886059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Obtaining Sagital and Coronal View","metadata":{"papermill":{"duration":0.026747,"end_time":"2022-10-27T03:58:03.263429","exception":false,"start_time":"2022-10-27T03:58:03.236682","status":"completed"},"tags":[]}},{"cell_type":"code","source":"path=\"train_images/1.2.826.0.1.3680043.10001\"\ndcmListDir=os.listdir(path)\n\ndcmSlices = [dcm.read_file(path+'/'+s,force=True) for s in dcmListDir]\n#print(slices)\ndcmSlices = sorted(dcmSlices,key=lambda x:x.ImagePositionPatient[2])\n\npixelSpacing = dcmSlices[0].PixelSpacing\nsliceThickness = dcmSlices[0].SliceThickness\n\nsagitalAspectRatio = pixelSpacing[1]/sliceThickness\ncoronalAspectRatio = sliceThickness/pixelSpacing[0]\n\nimgShape = list(dcmSlices[0].pixel_array.shape)\nimgShape.append(len(dcmSlices))\nvolume3d=np.zeros(imgShape)\n\nfor i,s in enumerate(dcmSlices):\n    array2D=s.pixel_array\n    volume3d[:,:,i]= array2D\n\nsagital=plt.subplot(1,2,1)\nplt.title(\"Sagital\")\nplt.imshow(volume3d[:,imgShape[1]//2,:])\nsagital.set_aspect(sagitalAspectRatio)\n\ncoronal = plt.subplot(1,2,2)\nplt.title(\"Coronal\")\nplt.imshow(volume3d[imgShape[0]//2,:,:].T)\ncoronal.set_aspect(coronalAspectRatio)\n\n\nplt.show()\n\n\nprint(array2D.shape)\nprint(volume3d.shape)    ","metadata":{"papermill":{"duration":5.477824,"end_time":"2022-10-27T03:58:08.768634","exception":false,"start_time":"2022-10-27T03:58:03.29081","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:47:32.253626Z","iopub.execute_input":"2023-02-07T11:47:32.254035Z","iopub.status.idle":"2023-02-07T11:47:37.505484Z","shell.execute_reply.started":"2023-02-07T11:47:32.254003Z","shell.execute_reply":"2023-02-07T11:47:37.504312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Classifing images","metadata":{"papermill":{"duration":0.033688,"end_time":"2022-10-27T03:58:49.958711","exception":false,"start_time":"2022-10-27T03:58:49.925023","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Before continue we need to understand what we have, in segmentations we have near of 87 files and each of them correspond to a single patient,and all them correspond to one only to train files set.","metadata":{"papermill":{"duration":0.034317,"end_time":"2022-10-27T03:58:50.027381","exception":false,"start_time":"2022-10-27T03:58:49.993064","status":"completed"},"tags":[]}},{"cell_type":"code","source":"segmentationsIDs = [ os.listdir(segmentationsFilesPath) ]\nsegmentationsIDs = [ ID[:-4] for ID in segmentationsIDs[0] ]\ntrainIDs = [ os.listdir(trainFilesPath) ]\ntrainIDs = [ ID for ID in segmentationsIDs ]\nIDs = [ ID for ID in trainIDs if ID in segmentationsIDs ]\nlen( segmentationsIDs )","metadata":{"papermill":{"duration":0.146788,"end_time":"2022-10-27T03:58:50.208602","exception":false,"start_time":"2022-10-27T03:58:50.061814","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-07T11:47:44.342022Z","iopub.execute_input":"2023-02-07T11:47:44.343011Z","iopub.status.idle":"2023-02-07T11:47:44.477155Z","shell.execute_reply.started":"2023-02-07T11:47:44.342966Z","shell.execute_reply":"2023-02-07T11:47:44.476263Z"},"trusted":true},"execution_count":null,"outputs":[]}]}