{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The goal of this notebook is to share some data exploration I have made on the dataset and the observation.\n\nI hope this can bring some insight in the process of building the best model for Breast cancer detection.","metadata":{}},{"cell_type":"markdown","source":"Here others interesting notebooks about the dataset that have inspired me:\n\n[Dicom -> Cropped & Resized PNG/JPG](https://www.kaggle.com/code/fabiendaniel/dicom-cropped-resized-png-jpg) \n\n[Dicom -> Resized PNG/JPG](https://www.kaggle.com/code/theoviel/dicom-resized-png-jpg)\n","metadata":{}},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-14T17:15:47.634363Z","iopub.execute_input":"2023-02-14T17:15:47.635283Z","iopub.status.idle":"2023-02-14T17:15:48.922685Z","shell.execute_reply.started":"2023-02-14T17:15:47.634781Z","shell.execute_reply":"2023-02-14T17:15:48.921444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The train csv file\ntrain_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\nprint(\"Lenght of train set: \",len(train_df))\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T17:16:52.452618Z","iopub.execute_input":"2023-02-14T17:16:52.453012Z","iopub.status.idle":"2023-02-14T17:16:52.552816Z","shell.execute_reply.started":"2023-02-14T17:16:52.45298Z","shell.execute_reply":"2023-02-14T17:16:52.55149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T17:16:34.606118Z","iopub.execute_input":"2023-02-14T17:16:34.606502Z","iopub.status.idle":"2023-02-14T17:16:34.626637Z","shell.execute_reply.started":"2023-02-14T17:16:34.606471Z","shell.execute_reply":"2023-02-14T17:16:34.625558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T17:16:37.594682Z","iopub.execute_input":"2023-02-14T17:16:37.595126Z","iopub.status.idle":"2023-02-14T17:16:37.634971Z","shell.execute_reply.started":"2023-02-14T17:16:37.595087Z","shell.execute_reply":"2023-02-14T17:16:37.633442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The test csv file\ntest_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\nprint(\"lenght of test set: \",len(test_df))\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T17:16:58.004152Z","iopub.execute_input":"2023-02-14T17:16:58.004564Z","iopub.status.idle":"2023-02-14T17:16:58.028988Z","shell.execute_reply.started":"2023-02-14T17:16:58.004505Z","shell.execute_reply":"2023-02-14T17:16:58.027606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:55.523367Z","iopub.execute_input":"2023-02-14T15:18:55.523819Z","iopub.status.idle":"2023-02-14T15:18:55.537273Z","shell.execute_reply.started":"2023-02-14T15:18:55.523785Z","shell.execute_reply":"2023-02-14T15:18:55.536331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:55.538413Z","iopub.execute_input":"2023-02-14T15:18:55.539539Z","iopub.status.idle":"2023-02-14T15:18:55.557267Z","shell.execute_reply.started":"2023-02-14T15:18:55.539505Z","shell.execute_reply":"2023-02-14T15:18:55.556007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of patient: \",train_df['patient_id'].nunique())\nprint(\"Number of images: \",len(train_df['image_id']))\nprint(\"Number of unique images: \",train_df['image_id'].nunique())\nprint(\"Number of site: \",train_df['site_id'].nunique())\nprint(\"Number of unique laterality: \",train_df['laterality'].nunique())\nprint(\"Number of unique view: \",train_df['view'].nunique())","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:55.55879Z","iopub.execute_input":"2023-02-14T15:18:55.559325Z","iopub.status.idle":"2023-02-14T15:18:55.581905Z","shell.execute_reply.started":"2023-02-14T15:18:55.559286Z","shell.execute_reply":"2023-02-14T15:18:55.58094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Cancer\n97.9% of the train images are cancer free. A big class imbalance.","metadata":{}},{"cell_type":"code","source":"print(train_df['cancer'].value_counts(normalize=True))\nprint(train_df['cancer'].value_counts())\nsns.countplot(data=train_df, x='cancer')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:55.583216Z","iopub.execute_input":"2023-02-14T15:18:55.583772Z","iopub.status.idle":"2023-02-14T15:18:55.778529Z","shell.execute_reply.started":"2023-02-14T15:18:55.583739Z","shell.execute_reply":"2023-02-14T15:18:55.777258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### laterality\nThe number of images in the train set are approximatively equal for each kind laterality (L or R).\n\nThey relatively equal number of cancer image in each kind of laterality.","metadata":{}},{"cell_type":"code","source":"print(train_df['laterality'].value_counts(normalize=True))\nprint(train_df['laterality'].value_counts())\nsns.countplot(data=train_df, x='laterality')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:55.783654Z","iopub.execute_input":"2023-02-14T15:18:55.784263Z","iopub.status.idle":"2023-02-14T15:18:55.980186Z","shell.execute_reply.started":"2023-02-14T15:18:55.784221Z","shell.execute_reply":"2023-02-14T15:18:55.978887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['laterality','cancer']].groupby(['laterality']).sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:27:11.940745Z","iopub.execute_input":"2023-02-14T15:27:11.941218Z","iopub.status.idle":"2023-02-14T15:27:11.959928Z","shell.execute_reply.started":"2023-02-14T15:27:11.941185Z","shell.execute_reply":"2023-02-14T15:27:11.958804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Site\nThey are 5% more image from site 1 than from site 2.\n\nIt also show more image with cancer from site 1 than from site 2.","metadata":{"execution":{"iopub.status.busy":"2023-02-14T14:12:35.230003Z","iopub.execute_input":"2023-02-14T14:12:35.23037Z","iopub.status.idle":"2023-02-14T14:12:35.234645Z","shell.execute_reply.started":"2023-02-14T14:12:35.23034Z","shell.execute_reply":"2023-02-14T14:12:35.233549Z"}}},{"cell_type":"code","source":"print(train_df['site_id'].value_counts(normalize=True))\nprint(train_df['site_id'].value_counts())\nsns.countplot(data=train_df, x='site_id')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:56.005343Z","iopub.execute_input":"2023-02-14T15:18:56.00617Z","iopub.status.idle":"2023-02-14T15:18:56.179089Z","shell.execute_reply.started":"2023-02-14T15:18:56.006121Z","shell.execute_reply":"2023-02-14T15:18:56.178162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['site_id','cancer']].groupby(['site_id']).sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:19:33.612192Z","iopub.execute_input":"2023-02-14T15:19:33.61258Z","iopub.status.idle":"2023-02-14T15:19:33.628169Z","shell.execute_reply.started":"2023-02-14T15:19:33.61255Z","shell.execute_reply":"2023-02-14T15:19:33.626818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### View\nThey are 6 differents value of view. \n\nMost (99%) images are concentrated in these two views MLO and CC.","metadata":{}},{"cell_type":"code","source":"print(train_df['view'].value_counts(normalize=True))\nprint(train_df['view'].value_counts())\nsns.countplot(data=train_df, x='view')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:56.195167Z","iopub.execute_input":"2023-02-14T15:18:56.195881Z","iopub.status.idle":"2023-02-14T15:18:56.415129Z","shell.execute_reply.started":"2023-02-14T15:18:56.195846Z","shell.execute_reply":"2023-02-14T15:18:56.413759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[['view','cancer']].groupby(['view']).sum()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:19:28.396197Z","iopub.execute_input":"2023-02-14T15:19:28.397223Z","iopub.status.idle":"2023-02-14T15:19:28.418458Z","shell.execute_reply.started":"2023-02-14T15:19:28.397181Z","shell.execute_reply":"2023-02-14T15:19:28.417531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Age\nPatient age range from 26 to 89.\n\nHigher number of cancer image are observed between 40 to 70.\n\n50 is the age with the most cancer image number.","metadata":{}},{"cell_type":"code","source":"print(len(train_df))\nprint(train_df['age'].nunique())\nprint(\"Maximum age: \",train_df['age'].max())\nprint(\"Minimum age: \",train_df['age'].min())\n\ntrain_df['age'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:56.461872Z","iopub.execute_input":"2023-02-14T15:18:56.462361Z","iopub.status.idle":"2023-02-14T15:18:56.478583Z","shell.execute_reply.started":"2023-02-14T15:18:56.4623Z","shell.execute_reply":"2023-02-14T15:18:56.477129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df['age'].value_counts(normalize=True))\nprint(train_df['age'].value_counts())\n\n# setting the dimensions of the plot\nfig, ax = plt.subplots(figsize=(40, 5))\n \n# drawing the plot\nsns.countplot(data=train_df, x='age')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:56.480192Z","iopub.execute_input":"2023-02-14T15:18:56.480534Z","iopub.status.idle":"2023-02-14T15:18:57.459199Z","shell.execute_reply.started":"2023-02-14T15:18:56.480504Z","shell.execute_reply":"2023-02-14T15:18:57.458016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DICOM Image reading","metadata":{}},{"cell_type":"code","source":"!pip install -qU pydicom python-gdcm pylibjpeg","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:18:57.460621Z","iopub.execute_input":"2023-02-14T15:18:57.461613Z","iopub.status.idle":"2023-02-14T15:19:11.275564Z","shell.execute_reply.started":"2023-02-14T15:18:57.461568Z","shell.execute_reply":"2023-02-14T15:19:11.274397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport matplotlib.pyplot as plt\nidx = 21\ntrain_img_dir = '/kaggle/input/rsna-breast-cancer-detection/train_images'\nimg_name = str(train_df['image_id'].iloc[idx])+'.dcm'\npatient_id = str(train_df['patient_id'].iloc[idx])\n\ncancer = train_df['cancer'].iloc[idx]\n\nimg_path = os.path.join(train_img_dir, patient_id, img_name)\n\ndcm_img = pydicom.dcmread(img_path, force=True)\nprint(dcm_img)\n\nimg_array = dcm_img.pixel_array\nplt.imshow(img_array)\nplt.title(f'Patient: {patient_id}; Cancer: {cancer}; Image: {img_name}')\nprint(f\"This image shape is: {img_array.shape}\")","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:19:11.277326Z","iopub.execute_input":"2023-02-14T15:19:11.278107Z","iopub.status.idle":"2023-02-14T15:19:14.445688Z","shell.execute_reply.started":"2023-02-14T15:19:11.278062Z","shell.execute_reply":"2023-02-14T15:19:14.444875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\npatient_folder = os.listdir(train_img_dir)\nprint(\"Number of patient folders: \", len(patient_folder))\n# f_idx = 3\n# patient_id = patient_folder[f_idx]\npatient_id = \"10324\"\nprint(\"Patient ID: \",patient_id)\npatient_dicom = os.listdir(os.path.join(train_img_dir, patient_id))\nprint(len(patient_dicom))\nprint(patient_dicom)\nfig, ax = plt.subplots(math.ceil(len(patient_dicom)/3), 3, figsize=(20, 20))\nfor x, img_name in enumerate(patient_dicom):\n    img_path = os.path.join(train_img_dir, patient_id, img_name)\n    print(img_path)\n    dcm_img = pydicom.dcmread(img_path, force=True)\n    r,c = divmod(x, 3)\n    img_array = dcm_img.pixel_array\n    ax[r][c].imshow(img_array)\n    ax[r][c].set_title(img_name)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T15:19:14.447482Z","iopub.execute_input":"2023-02-14T15:19:14.448286Z","iopub.status.idle":"2023-02-14T15:19:28.392808Z","shell.execute_reply.started":"2023-02-14T15:19:14.448242Z","shell.execute_reply":"2023-02-14T15:19:28.389093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I hope you find this interesting.\nFeel free to add comments on some of your observations.\nShare alsor ideas about the dataset exploration.\n\nThank you !","metadata":{}}]}