{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1 | Introduction\nThis notebook creates a dataframe from all the metadata in the provided DICOM files. This is a time consuming job, but only needs to be once. The result is stored in a file, so it can be consumed later from other notebooks.","metadata":{}},{"cell_type":"code","source":"!pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom python-gdcm\n\n!pip install -U pylibjpeg pylibjpeg-openjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:10.681004Z","iopub.execute_input":"2022-12-31T13:32:10.681723Z","iopub.status.idle":"2022-12-31T13:32:28.308119Z","shell.execute_reply.started":"2022-12-31T13:32:10.681545Z","shell.execute_reply":"2022-12-31T13:32:28.30688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tell the notebook to reload any changes made to any libraries used.\n# Also ensure that any graphs are plotted are shown in this notebook\n%reload_ext autoreload\n%autoreload 2\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-31T13:32:39.62868Z","iopub.execute_input":"2022-12-31T13:32:39.629051Z","iopub.status.idle":"2022-12-31T13:32:39.686952Z","shell.execute_reply.started":"2022-12-31T13:32:39.629012Z","shell.execute_reply":"2022-12-31T13:32:39.685994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import clear_output, display_html\n\nimport os\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nimport seaborn as sns # statistical data visualization\nfrom tqdm import tqdm # progress\n\nfrom fastai.imports import *\nfrom fastai.metrics import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:39.689966Z","iopub.execute_input":"2022-12-31T13:32:39.690402Z","iopub.status.idle":"2022-12-31T13:32:43.381893Z","shell.execute_reply.started":"2022-12-31T13:32:39.690359Z","shell.execute_reply":"2022-12-31T13:32:43.380683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2 | Explore the datasets\nLets start by printing some basic information about the datasets we are given\n","metadata":{}},{"cell_type":"code","source":"def load_data():\n    \n    data_dir = Path(\"../input/rsna-breast-cancer-detection\")\n    train = pd.read_csv(data_dir / \"train.csv\")\n    test = pd.read_csv(data_dir / \"test.csv\")\n    sample_submission = pd.read_csv(data_dir / 'sample_submission.csv')\n    return train, test, sample_submission","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:43.38324Z","iopub.execute_input":"2022-12-31T13:32:43.383789Z","iopub.status.idle":"2022-12-31T13:32:43.433036Z","shell.execute_reply.started":"2022-12-31T13:32:43.383754Z","shell.execute_reply":"2022-12-31T13:32:43.431806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_data_info(csv, name=\"Train\"):\n           \n    print(\"\\n\")\n    print('Data set: ', name)\n    print('Shape: ', csv.shape)\n    print('NaN values: ', csv.isnull().sum().sum())\n    print(\"\\n\")\n    \n    display_html(csv.head())\n\ntrain, test, sample_submission = load_data()","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:43.434272Z","iopub.execute_input":"2022-12-31T13:32:43.434624Z","iopub.status.idle":"2022-12-31T13:32:43.691557Z","shell.execute_reply.started":"2022-12-31T13:32:43.43459Z","shell.execute_reply":"2022-12-31T13:32:43.690715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"names = [\"Train\", \"Test\", \"Sample Submission\"]\nfor i, df in enumerate([train, test, sample_submission]): \n    print_data_info(df, names[i])","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:43.692912Z","iopub.execute_input":"2022-12-31T13:32:43.693201Z","iopub.status.idle":"2022-12-31T13:32:43.776908Z","shell.execute_reply.started":"2022-12-31T13:32:43.693174Z","shell.execute_reply":"2022-12-31T13:32:43.775858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3 | Create a meta data dataframe \nLet's load every metadata from the images into a dataframe.\n\nWe start by getting get all the dcm files.","metadata":{}},{"cell_type":"code","source":"dcms = get_dicom_files('/kaggle/input/rsna-breast-cancer-detection/train_images/')\nprint(f'There are {len(dcms)} images')","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:32:43.778339Z","iopub.execute_input":"2022-12-31T13:32:43.778897Z","iopub.status.idle":"2022-12-31T13:33:27.946866Z","shell.execute_reply.started":"2022-12-31T13:32:43.778858Z","shell.execute_reply":"2022-12-31T13:33:27.945916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we grab a file and take a look inside it using the dcmread method that fastai v2 provides.","metadata":{}},{"cell_type":"code","source":"TEST_DCM = dcms[0]\ndcm = TEST_DCM.dcmread()\ndcm","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:33:27.948244Z","iopub.execute_input":"2022-12-31T13:33:27.94937Z","iopub.status.idle":"2022-12-31T13:33:28.053934Z","shell.execute_reply.started":"2022-12-31T13:33:27.949308Z","shell.execute_reply":"2022-12-31T13:33:28.052719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(dcm)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:33:28.057686Z","iopub.execute_input":"2022-12-31T13:33:28.059913Z","iopub.status.idle":"2022-12-31T13:33:28.108406Z","shell.execute_reply.started":"2022-12-31T13:33:28.059877Z","shell.execute_reply":"2022-12-31T13:33:28.107539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now it's time to turn the DICOM files metadata into a DataFrame. Note: this takes a very long time (so could be usefuly to split in chunks). \n","metadata":{}},{"cell_type":"code","source":"%time dfmeta = pd.DataFrame.from_dicoms(dcms, px_summ=True)","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:47:18.518882Z","iopub.execute_input":"2022-12-31T13:47:18.519291Z","iopub.status.idle":"2022-12-31T13:49:14.115215Z","shell.execute_reply.started":"2022-12-31T13:47:18.519258Z","shell.execute_reply":"2022-12-31T13:49:14.114008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4 | Store data frame for later exploration\nPersist it using the feather format (which is lightning fast!) so we can read and explore it later (from another notebook).","metadata":{}},{"cell_type":"code","source":"dfmeta.to_feather('/kaggle/working/dfmeta.fth')","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:53:06.019219Z","iopub.execute_input":"2022-12-31T13:53:06.019647Z","iopub.status.idle":"2022-12-31T13:53:06.078605Z","shell.execute_reply.started":"2022-12-31T13:53:06.019614Z","shell.execute_reply":"2022-12-31T13:53:06.077111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just a peak to see what we've got..","metadata":{}},{"cell_type":"code","source":"dfmeta.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:53:12.599878Z","iopub.execute_input":"2022-12-31T13:53:12.600276Z","iopub.status.idle":"2022-12-31T13:53:12.674474Z","shell.execute_reply.started":"2022-12-31T13:53:12.600244Z","shell.execute_reply":"2022-12-31T13:53:12.6734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There's not much RAM on these kaggle kernel instances, so we'll clean up as we go.","metadata":{}},{"cell_type":"code","source":"#del(dfmeta)\n#import gc; gc.collect();","metadata":{"execution":{"iopub.status.busy":"2022-12-31T13:33:30.241642Z","iopub.execute_input":"2022-12-31T13:33:30.242394Z","iopub.status.idle":"2022-12-31T13:33:30.289806Z","shell.execute_reply.started":"2022-12-31T13:33:30.24236Z","shell.execute_reply":"2022-12-31T13:33:30.288854Z"},"trusted":true},"execution_count":null,"outputs":[]}]}