{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of Contents\n* [Training Data - CSV File](#1)\n* [Training Data - Bounding Boxes](#2)\n* [Test Data](#3)\n","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\n\n# plot\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# statistics\nimport phik\n\n# images\nimport glob\nimport pydicom as dicom","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-29T07:50:57.278551Z","iopub.execute_input":"2022-07-29T07:50:57.279625Z","iopub.status.idle":"2022-07-29T07:50:57.414655Z","shell.execute_reply.started":"2022-07-29T07:50:57.279586Z","shell.execute_reply":"2022-07-29T07:50:57.413309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show files\n!ls -l '../input/rsna-2022-cervical-spine-fracture-detection'","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:51.602333Z","iopub.execute_input":"2022-07-29T07:29:51.603317Z","iopub.status.idle":"2022-07-29T07:29:52.524105Z","shell.execute_reply.started":"2022-07-29T07:29:51.603128Z","shell.execute_reply":"2022-07-29T07:29:52.522821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='1'></a>\n# Training Data - CSV File","metadata":{}},{"cell_type":"code","source":"# import and overview\ndf_train = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/train.csv')\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:52.526115Z","iopub.execute_input":"2022-07-29T07:29:52.527421Z","iopub.status.idle":"2022-07-29T07:29:52.564984Z","shell.execute_reply.started":"2022-07-29T07:29:52.527381Z","shell.execute_reply":"2022-07-29T07:29:52.563683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overall patient level outcome\nprint(df_train.patient_overall.value_counts())\ndf_train.patient_overall.value_counts().plot(kind='bar')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T08:15:02.728443Z","iopub.execute_input":"2022-07-29T08:15:02.728837Z","iopub.status.idle":"2022-07-29T08:15:02.894156Z","shell.execute_reply.started":"2022-07-29T08:15:02.728804Z","shell.execute_reply":"2022-07-29T08:15:02.893126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# other individual targets\nyy = ['C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:52.810753Z","iopub.execute_input":"2022-07-29T07:29:52.811484Z","iopub.status.idle":"2022-07-29T07:29:52.817027Z","shell.execute_reply.started":"2022-07-29T07:29:52.811433Z","shell.execute_reply":"2022-07-29T07:29:52.816113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# count frequencies by column\ndf_train[yy].sum()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T08:16:48.253242Z","iopub.execute_input":"2022-07-29T08:16:48.254336Z","iopub.status.idle":"2022-07-29T08:16:48.264537Z","shell.execute_reply.started":"2022-07-29T08:16:48.254247Z","shell.execute_reply":"2022-07-29T08:16:48.263347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show frequency plots\nfor f in yy:\n    df_train[f].value_counts().plot(kind='bar')\n    plt.title(f)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:52.818161Z","iopub.execute_input":"2022-07-29T07:29:52.818956Z","iopub.status.idle":"2022-07-29T07:29:54.023616Z","shell.execute_reply.started":"2022-07-29T07:29:52.818917Z","shell.execute_reply":"2022-07-29T07:29:54.02235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Measure correlation of (categorical) C-targets using Phi_K - correlation \n(see https://phik.readthedocs.io/en/latest/)","metadata":{}},{"cell_type":"code","source":"phiK_mat = df_train[yy].phik_matrix()\nphiK_mat","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:39:40.594255Z","iopub.execute_input":"2022-07-29T07:39:40.594661Z","iopub.status.idle":"2022-07-29T07:39:40.724297Z","shell.execute_reply.started":"2022-07-29T07:39:40.594631Z","shell.execute_reply":"2022-07-29T07:39:40.72296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize phi_K matrix\nplt.figure(figsize=(7,6))\nsns.heatmap(phiK_mat, annot=True, cmap='RdYlGn', vmin=-1, vmax=+1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T08:11:05.052253Z","iopub.execute_input":"2022-07-29T08:11:05.05357Z","iopub.status.idle":"2022-07-29T08:11:05.48776Z","shell.execute_reply.started":"2022-07-29T08:11:05.053525Z","shell.execute_reply":"2022-07-29T08:11:05.486525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Add sum of C-targets","metadata":{}},{"cell_type":"code","source":"df_train['C_sum'] = df_train[yy].sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:54.027643Z","iopub.execute_input":"2022-07-29T07:29:54.028605Z","iopub.status.idle":"2022-07-29T07:29:54.038911Z","shell.execute_reply.started":"2022-07-29T07:29:54.028559Z","shell.execute_reply":"2022-07-29T07:29:54.037676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.C_sum.value_counts())\ndf_train.C_sum.value_counts().plot(kind='bar')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:54.040883Z","iopub.execute_input":"2022-07-29T07:29:54.041786Z","iopub.status.idle":"2022-07-29T07:29:54.237477Z","shell.execute_reply.started":"2022-07-29T07:29:54.041736Z","shell.execute_reply":"2022-07-29T07:29:54.236333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Look at two extreme cases having sum 6:","metadata":{}},{"cell_type":"code","source":"df_train[df_train.C_sum==6]","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:54.239096Z","iopub.execute_input":"2022-07-29T07:29:54.240171Z","iopub.status.idle":"2022-07-29T07:29:54.258706Z","shell.execute_reply.started":"2022-07-29T07:29:54.240122Z","shell.execute_reply":"2022-07-29T07:29:54.257596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Credits to https://www.kaggle.com/code/mohammaddehghan/rsna2022-eda-w-b for the following code snippets to get and display the images","metadata":{}},{"cell_type":"code","source":"# get access to image files for first of the extreme cases\nimage_file = glob.glob('../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.25071/*.dcm')\n\n# we have a lot of images here\nprint('Number of images:', len(image_file))","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:55:47.154439Z","iopub.execute_input":"2022-07-29T07:55:47.154891Z","iopub.status.idle":"2022-07-29T07:55:47.163158Z","shell.execute_reply.started":"2022-07-29T07:55:47.154854Z","shell.execute_reply":"2022-07-29T07:55:47.161814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show a few of them\nplt.figure(figsize=(80, 80))\nfor i in range(16):\n    ax = plt.subplot(4, 4, i + 1)\n    ds = dicom.dcmread(image_file[i])\n    plt.axis('off')    \n    plt.imshow(ds.pixel_array)","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:53:03.231957Z","iopub.execute_input":"2022-07-29T07:53:03.232413Z","iopub.status.idle":"2022-07-29T07:53:13.290562Z","shell.execute_reply.started":"2022-07-29T07:53:03.232368Z","shell.execute_reply":"2022-07-29T07:53:13.288785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='2'></a>\n# Training Data - Bounding Boxes","metadata":{}},{"cell_type":"code","source":"# import and overview\ndf_train_bound = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv')\ndf_train_bound.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:54.26053Z","iopub.execute_input":"2022-07-29T07:29:54.260874Z","iopub.status.idle":"2022-07-29T07:29:54.294053Z","shell.execute_reply.started":"2022-07-29T07:29:54.260842Z","shell.execute_reply":"2022-07-29T07:29:54.292784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# slice number - distribution\nplt.figure(figsize=(15,4))\ndf_train_bound.slice_number.value_counts().plot(kind='bar')\nplt.title('slice_number - frequencies')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T08:10:24.359846Z","iopub.execute_input":"2022-07-29T08:10:24.360304Z","iopub.status.idle":"2022-07-29T08:10:29.300655Z","shell.execute_reply.started":"2022-07-29T08:10:24.360253Z","shell.execute_reply":"2022-07-29T08:10:29.299533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show only top 50\nplt.figure(figsize=(15,4))\ndf_train_bound.slice_number.value_counts()[0:50].plot(kind='bar')\nplt.title('slice_number - frequencies (top 50)')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:35:11.020658Z","iopub.execute_input":"2022-07-29T07:35:11.021116Z","iopub.status.idle":"2022-07-29T07:35:11.635317Z","shell.execute_reply.started":"2022-07-29T07:35:11.021077Z","shell.execute_reply":"2022-07-29T07:35:11.634138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display as histogram\ndf_train_bound.slice_number.plot(kind='hist', bins=100)\nplt.title('slice_number - histogram')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T08:10:16.880967Z","iopub.execute_input":"2022-07-29T08:10:16.881376Z","iopub.status.idle":"2022-07-29T08:10:17.230783Z","shell.execute_reply.started":"2022-07-29T08:10:16.88134Z","shell.execute_reply":"2022-07-29T08:10:17.229169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate numerical features","metadata":{}},{"cell_type":"code","source":"xx_bound_num = ['x','y','width','height']","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:59.103357Z","iopub.execute_input":"2022-07-29T07:29:59.103724Z","iopub.status.idle":"2022-07-29T07:29:59.109286Z","shell.execute_reply.started":"2022-07-29T07:29:59.103691Z","shell.execute_reply":"2022-07-29T07:29:59.108021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(df_train_bound[xx_bound_num])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:29:59.111405Z","iopub.execute_input":"2022-07-29T07:29:59.112475Z","iopub.status.idle":"2022-07-29T07:30:03.132816Z","shell.execute_reply.started":"2022-07-29T07:29:59.112419Z","shell.execute_reply":"2022-07-29T07:30:03.131998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats\ndf_train_bound[xx_bound_num].describe(percentiles=[0.05,0.1,0.25,0.5,0.75,0.9,0.95])","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:30:03.134131Z","iopub.execute_input":"2022-07-29T07:30:03.134497Z","iopub.status.idle":"2022-07-29T07:30:03.164056Z","shell.execute_reply.started":"2022-07-29T07:30:03.134463Z","shell.execute_reply":"2022-07-29T07:30:03.162866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='3'></a>\n# Test Data - CSV","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/test.csv')\ndf_test.info()","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:30:03.16561Z","iopub.execute_input":"2022-07-29T07:30:03.166493Z","iopub.status.idle":"2022-07-29T07:30:03.181094Z","shell.execute_reply.started":"2022-07-29T07:30:03.166457Z","shell.execute_reply":"2022-07-29T07:30:03.180241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:30:03.18265Z","iopub.execute_input":"2022-07-29T07:30:03.18355Z","iopub.status.idle":"2022-07-29T07:30:03.195567Z","shell.execute_reply.started":"2022-07-29T07:30:03.183509Z","shell.execute_reply":"2022-07-29T07:30:03.194353Z"},"trusted":true},"execution_count":null,"outputs":[]}]}