{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pylibjpeg\n!pip install python-gdcm\n!pip install plotly==5.11.0","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-04-08T05:29:00.744449Z","iopub.execute_input":"2023-04-08T05:29:00.744864Z","iopub.status.idle":"2023-04-08T05:30:10.320839Z","shell.execute_reply.started":"2023-04-08T05:29:00.74479Z","shell.execute_reply":"2023-04-08T05:30:10.318994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom collections import Counter\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nimport pydicom\nimport pylibjpeg\n\nsns.set_style(\"darkgrid\")","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:10.323338Z","iopub.execute_input":"2023-04-08T05:30:10.323737Z","iopub.status.idle":"2023-04-08T05:30:12.243781Z","shell.execute_reply.started":"2023-04-08T05:30:10.323705Z","shell.execute_reply":"2023-04-08T05:30:12.241902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images = Path(\"/kaggle/input/rsna-breast-cancer-detection/train_images\")\ntest_images = Path(\"/kaggle/input/rsna-breast-cancer-detection/test_images\")","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.245214Z","iopub.execute_input":"2023-04-08T05:30:12.245608Z","iopub.status.idle":"2023-04-08T05:30:12.251548Z","shell.execute_reply.started":"2023-04-08T05:30:12.245573Z","shell.execute_reply":"2023-04-08T05:30:12.250069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\nsample_submission_df = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.254769Z","iopub.execute_input":"2023-04-08T05:30:12.255348Z","iopub.status.idle":"2023-04-08T05:30:12.364925Z","shell.execute_reply.started":"2023-04-08T05:30:12.25531Z","shell.execute_reply":"2023-04-08T05:30:12.363679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.366018Z","iopub.execute_input":"2023-04-08T05:30:12.366327Z","iopub.status.idle":"2023-04-08T05:30:12.401543Z","shell.execute_reply.started":"2023-04-08T05:30:12.366298Z","shell.execute_reply":"2023-04-08T05:30:12.400011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:33:58.605701Z","iopub.execute_input":"2023-04-08T05:33:58.606049Z","iopub.status.idle":"2023-04-08T05:33:58.656272Z","shell.execute_reply.started":"2023-04-08T05:33:58.606023Z","shell.execute_reply":"2023-04-08T05:33:58.65482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:34:11.966642Z","iopub.execute_input":"2023-04-08T05:34:11.967044Z","iopub.status.idle":"2023-04-08T05:34:11.985605Z","shell.execute_reply.started":"2023-04-08T05:34:11.967012Z","shell.execute_reply":"2023-04-08T05:34:11.984275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:34:26.321338Z","iopub.execute_input":"2023-04-08T05:34:26.321743Z","iopub.status.idle":"2023-04-08T05:34:26.328956Z","shell.execute_reply.started":"2023-04-08T05:34:26.321712Z","shell.execute_reply":"2023-04-08T05:34:26.327786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:34:45.897375Z","iopub.execute_input":"2023-04-08T05:34:45.897787Z","iopub.status.idle":"2023-04-08T05:34:45.916182Z","shell.execute_reply.started":"2023-04-08T05:34:45.897754Z","shell.execute_reply":"2023-04-08T05:34:45.914704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.hist","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:34:56.934045Z","iopub.execute_input":"2023-04-08T05:34:56.934474Z","iopub.status.idle":"2023-04-08T05:34:56.950721Z","shell.execute_reply.started":"2023-04-08T05:34:56.934443Z","shell.execute_reply":"2023-04-08T05:34:56.948938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.hist","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:35:07.230204Z","iopub.execute_input":"2023-04-08T05:35:07.230658Z","iopub.status.idle":"2023-04-08T05:35:07.249842Z","shell.execute_reply.started":"2023-04-08T05:35:07.230624Z","shell.execute_reply":"2023-04-08T05:35:07.248682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.40277Z","iopub.execute_input":"2023-04-08T05:30:12.403123Z","iopub.status.idle":"2023-04-08T05:30:12.417231Z","shell.execute_reply.started":"2023-04-08T05:30:12.403094Z","shell.execute_reply":"2023-04-08T05:30:12.415667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:35:51.009749Z","iopub.execute_input":"2023-04-08T05:35:51.01012Z","iopub.status.idle":"2023-04-08T05:35:51.041646Z","shell.execute_reply.started":"2023-04-08T05:35:51.010093Z","shell.execute_reply":"2023-04-08T05:35:51.04065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.tail()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:35:57.105555Z","iopub.execute_input":"2023-04-08T05:35:57.105911Z","iopub.status.idle":"2023-04-08T05:35:57.118651Z","shell.execute_reply.started":"2023-04-08T05:35:57.10588Z","shell.execute_reply":"2023-04-08T05:35:57.117397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:36:03.027218Z","iopub.execute_input":"2023-04-08T05:36:03.027691Z","iopub.status.idle":"2023-04-08T05:36:03.045577Z","shell.execute_reply.started":"2023-04-08T05:36:03.027656Z","shell.execute_reply":"2023-04-08T05:36:03.043557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:36:08.616119Z","iopub.execute_input":"2023-04-08T05:36:08.616557Z","iopub.status.idle":"2023-04-08T05:36:08.624166Z","shell.execute_reply.started":"2023-04-08T05:36:08.616517Z","shell.execute_reply":"2023-04-08T05:36:08.622756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:36:15.202811Z","iopub.execute_input":"2023-04-08T05:36:15.203186Z","iopub.status.idle":"2023-04-08T05:36:15.217874Z","shell.execute_reply.started":"2023-04-08T05:36:15.203155Z","shell.execute_reply":"2023-04-08T05:36:15.216412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.hist","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:36:21.697132Z","iopub.execute_input":"2023-04-08T05:36:21.697514Z","iopub.status.idle":"2023-04-08T05:36:21.707678Z","shell.execute_reply.started":"2023-04-08T05:36:21.697483Z","shell.execute_reply":"2023-04-08T05:36:21.706208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.418557Z","iopub.execute_input":"2023-04-08T05:30:12.418861Z","iopub.status.idle":"2023-04-08T05:30:12.433283Z","shell.execute_reply.started":"2023-04-08T05:30:12.418837Z","shell.execute_reply":"2023-04-08T05:30:12.431892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_counter = train_df[\"patient_id\"].value_counts().sort_index()\n\nfig = px.histogram(images_counter, text_auto=True, title=\"Number of images per patient\")\nfig.update_layout(bargap=0.2)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.434454Z","iopub.execute_input":"2023-04-08T05:30:12.434819Z","iopub.status.idle":"2023-04-08T05:30:12.854747Z","shell.execute_reply.started":"2023-04-08T05:30:12.434785Z","shell.execute_reply":"2023-04-08T05:30:12.853643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_cancer_person = train_df.groupby(\"patient_id\")['cancer'].max().sort_index()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.856462Z","iopub.execute_input":"2023-04-08T05:30:12.857136Z","iopub.status.idle":"2023-04-08T05:30:12.87176Z","shell.execute_reply.started":"2023-04-08T05:30:12.857097Z","shell.execute_reply":"2023-04-08T05:30:12.869854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_counter.index.tolist() == is_cancer_person.index.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.875493Z","iopub.execute_input":"2023-04-08T05:30:12.875813Z","iopub.status.idle":"2023-04-08T05:30:12.885833Z","shell.execute_reply.started":"2023-04-08T05:30:12.875785Z","shell.execute_reply":"2023-04-08T05:30:12.885005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.corrcoef(images_counter.values, is_cancer_person.values)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.887103Z","iopub.execute_input":"2023-04-08T05:30:12.887484Z","iopub.status.idle":"2023-04-08T05:30:12.90078Z","shell.execute_reply.started":"2023-04-08T05:30:12.887451Z","shell.execute_reply":"2023-04-08T05:30:12.898997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"person_age = train_df.groupby(\"patient_id\")['age'].max().sort_index().fillna(0).astype('int64')\n\nfig = px.histogram(person_age, title=\"Distribution of patients' age\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.902795Z","iopub.execute_input":"2023-04-08T05:30:12.903208Z","iopub.status.idle":"2023-04-08T05:30:12.969372Z","shell.execute_reply.started":"2023-04-08T05:30:12.90317Z","shell.execute_reply":"2023-04-08T05:30:12.967572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_correlation_heatmap(df):\n    df = pd.get_dummies(df, columns=['laterality', 'view', 'density'])\n    df['difficult_negative_case'] = df['difficult_negative_case'].astype(int)\n\n    corr = df.corr()\n    mask = np.triu(np.ones_like(corr, dtype=bool))\n    fig, ax = plt.subplots(figsize=(11, 9))\n    cmap = sns.diverging_palette(230, 20, as_cmap=True)\n    sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0, square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.970847Z","iopub.execute_input":"2023-04-08T05:30:12.97121Z","iopub.status.idle":"2023-04-08T05:30:12.977993Z","shell.execute_reply.started":"2023-04-08T05:30:12.971179Z","shell.execute_reply":"2023-04-08T05:30:12.977122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_correlation_heatmap(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:12.978899Z","iopub.execute_input":"2023-04-08T05:30:12.979183Z","iopub.status.idle":"2023-04-08T05:30:13.690703Z","shell.execute_reply.started":"2023-04-08T05:30:12.979156Z","shell.execute_reply":"2023-04-08T05:30:13.689292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_image = \"/kaggle/input/rsna-breast-cancer-detection/train_images/10006/1459541791.dcm\"\npydicom.dcmread(sample_image)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:13.694159Z","iopub.execute_input":"2023-04-08T05:30:13.694833Z","iopub.status.idle":"2023-04-08T05:30:13.803146Z","shell.execute_reply.started":"2023-04-08T05:30:13.69479Z","shell.execute_reply":"2023-04-08T05:30:13.802418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"patient_id\"]","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:13.804242Z","iopub.execute_input":"2023-04-08T05:30:13.804726Z","iopub.status.idle":"2023-04-08T05:30:13.812094Z","shell.execute_reply.started":"2023-04-08T05:30:13.804699Z","shell.execute_reply":"2023-04-08T05:30:13.810179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images_of_patient(patient_id):\n    patient_df = test_df[test_df[\"patient_id\"] == patient_id] if patient_id in test_df[\"patient_id\"].values else train_df[train_df[\"patient_id\"] == patient_id] if patient_id in train_df[\"patient_id\"].values else None\n    if patient_df is None:\n        print(f\"No patient with id {patient_id}\")\n        return\n    \n    patient_folder = test_images / f\"{patient_id}\" if patient_id in test_df[\"patient_id\"].values else train_images / f\"{patient_id}\"\n    num_images = patient_df.shape[0]\n    n_rows, n_cols = (2, 2) if num_images <= 4 else (4, 4) if num_images <= 8 else (8, 8)\n    \n    fig, ax = plt.subplots(figsize=(24., 24.), nrows=n_rows, ncols=n_cols, gridspec_kw={'wspace': 0.6, 'hspace': 0.6})\n    fig.suptitle(f\"{patient_id} - {num_images} images\", fontsize=16)\n    \n    for idx, curr_image in enumerate(patient_folder.iterdir()):\n        curr_image = Path(curr_image)\n        image_data = patient_df[patient_df[\"image_id\"] == int(curr_image.stem)].squeeze()\n        ds = pydicom.dcmread(curr_image)\n        image_as_np = ds.pixel_array.astype(np.float32)\n        curr_ax = ax[idx // n_cols, idx % n_cols]\n        curr_ax.imshow(image_as_np)\n        curr_ax.set_title(\"image_id: {}\\n laterality: {}\\n view {}\\n machine_id {}\".format(image_data[\"image_id\"], image_data[\"laterality\"], image_data[\"view\"], image_data[\"machine_id\"]), fontdict={\"fontsize\": 9})\n","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:13.813997Z","iopub.execute_input":"2023-04-08T05:30:13.814566Z","iopub.status.idle":"2023-04-08T05:30:13.825639Z","shell.execute_reply.started":"2023-04-08T05:30:13.814532Z","shell.execute_reply":"2023-04-08T05:30:13.824627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_images_of_patient(32751)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:13.826942Z","iopub.execute_input":"2023-04-08T05:30:13.827493Z","iopub.status.idle":"2023-04-08T05:30:19.312294Z","shell.execute_reply.started":"2023-04-08T05:30:13.827464Z","shell.execute_reply":"2023-04-08T05:30:19.311419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_images_of_patient(2518)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:19.313848Z","iopub.execute_input":"2023-04-08T05:30:19.314486Z","iopub.status.idle":"2023-04-08T05:30:35.273076Z","shell.execute_reply.started":"2023-04-08T05:30:19.314447Z","shell.execute_reply":"2023-04-08T05:30:35.27217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_images_of_patient(10179)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T05:30:35.274194Z","iopub.execute_input":"2023-04-08T05:30:35.274974Z","iopub.status.idle":"2023-04-08T05:30:39.847178Z","shell.execute_reply.started":"2023-04-08T05:30:35.27494Z","shell.execute_reply":"2023-04-08T05:30:39.846183Z"},"trusted":true},"execution_count":null,"outputs":[]}]}