{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":4874049,"sourceType":"datasetVersion","datasetId":2717932},{"sourceId":104036025,"sourceType":"kernelVersion"}],"dockerImageVersionId":30302,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"⚠️ **I use this notebook to investigate MRI scans and fit a simple model. <br/>Here is a notebook that was submitted to the competition: [Breast Cancer Detection: tf, CNN (test)](https://www.kaggle.com/code/maryiaznak/breast-cancer-detection-tf-cnn-test)**","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-15T05:41:19.903311Z","iopub.execute_input":"2023-12-15T05:41:19.903676Z","iopub.status.idle":"2023-12-15T05:41:19.958289Z","shell.execute_reply.started":"2023-12-15T05:41:19.903595Z","shell.execute_reply":"2023-12-15T05:41:19.957236Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Download data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:41:19.959946Z","iopub.execute_input":"2023-12-15T05:41:19.960242Z","iopub.status.idle":"2023-12-15T05:41:20.110638Z","shell.execute_reply.started":"2023-12-15T05:41:19.960214Z","shell.execute_reply":"2023-12-15T05:41:20.109511Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Data investigation","metadata":{}},{"cell_type":"code","source":"print(f'Length of train dataframe: {len(train_df)}\\n')\nprint(f'Number of NaN values:\\n{train_df.isna().sum()}\\n')","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:41:20.111856Z","iopub.execute_input":"2023-12-15T05:41:20.112178Z","iopub.status.idle":"2023-12-15T05:41:20.137173Z","shell.execute_reply.started":"2023-12-15T05:41:20.112141Z","shell.execute_reply":"2023-12-15T05:41:20.136056Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%capture\n\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n    !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:41:20.139755Z","iopub.execute_input":"2023-12-15T05:41:20.140078Z","iopub.status.idle":"2023-12-15T05:42:28.567423Z","shell.execute_reply.started":"2023-12-15T05:41:20.140047Z","shell.execute_reply":"2023-12-15T05:42:28.566103Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"For simple model fitting and future predictions I'm gonna use only patient scans.","metadata":{}},{"cell_type":"code","source":"#let's consider one particular patient\npatient_id = train_df[train_df.cancer == 1].iloc[0].patient_id\n\none_patient_df = train_df[train_df.patient_id == patient_id]\none_patient_df","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:28.569219Z","iopub.execute_input":"2023-12-15T05:42:28.569659Z","iopub.status.idle":"2023-12-15T05:42:28.606921Z","shell.execute_reply.started":"2023-12-15T05:42:28.569613Z","shell.execute_reply":"2023-12-15T05:42:28.605937Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"images_dir = '/kaggle/input/rsna-breast-cancer-detection/{}_images/{}/{}.dcm'\ntrain = 'train'\ntest = 'test'","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:28.608075Z","iopub.execute_input":"2023-12-15T05:42:28.608391Z","iopub.status.idle":"2023-12-15T05:42:28.613293Z","shell.execute_reply.started":"2023-12-15T05:42:28.608362Z","shell.execute_reply":"2023-12-15T05:42:28.612181Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport dicomsdl\n\nn_rows = len(one_patient_df)\n\nplt.figure(figsize=(5 * n_rows, 5))\nfor i in range(n_rows):\n    row = one_patient_df.iloc[i]\n    \n    plt.subplot(1, n_rows, i + 1)\n    \n    img_arr = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id)).pixelData()\n    plt.imshow(img_arr, cmap = plt.cm.bone)\n    plt.text(200, 300, row['view'], fontsize = 14, bbox={'facecolor': 'white', 'pad' : 5})\n    plt.text(200, 700, row['cancer'], fontsize = 14, bbox={'facecolor': 'white', 'pad' : 5})","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:28.614727Z","iopub.execute_input":"2023-12-15T05:42:28.615122Z","iopub.status.idle":"2023-12-15T05:42:44.697127Z","shell.execute_reply.started":"2023-12-15T05:42:28.615072Z","shell.execute_reply":"2023-12-15T05:42:44.696082Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import seaborn as sns\n\nplt.figure(figsize=(5, 8))\nsns.countplot(data = train_df, x=\"laterality\", hue=\"cancer\", dodge = False)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:44.698571Z","iopub.execute_input":"2023-12-15T05:42:44.698931Z","iopub.status.idle":"2023-12-15T05:42:45.65382Z","shell.execute_reply.started":"2023-12-15T05:42:44.698899Z","shell.execute_reply":"2023-12-15T05:42:45.652539Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nsns.countplot(data = train_df, x=\"view\", hue=\"cancer\", dodge = False)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:45.655623Z","iopub.execute_input":"2023-12-15T05:42:45.656121Z","iopub.status.idle":"2023-12-15T05:42:45.974986Z","shell.execute_reply.started":"2023-12-15T05:42:45.656077Z","shell.execute_reply":"2023-12-15T05:42:45.973977Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.view.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:45.980713Z","iopub.execute_input":"2023-12-15T05:42:45.981047Z","iopub.status.idle":"2023-12-15T05:42:45.999135Z","shell.execute_reply.started":"2023-12-15T05:42:45.981016Z","shell.execute_reply":"2023-12-15T05:42:45.998071Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# here I'm plotting one scan sample of each view (without standardizing)\n\nLIST_OF_VIEWS = sorted(list(train_df.view.unique()))\n\nplt.figure(figsize=(5 * len(LIST_OF_VIEWS), 5))\n\nfor i, v in enumerate(LIST_OF_VIEWS, start = 1):\n    plt.subplot(1, len(LIST_OF_VIEWS), i)\n    row = train_df.loc[train_df.view == v].iloc[0]\n    \n    img_arr = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id)).pixelData()\n    plt.imshow(img_arr, cmap = plt.cm.bone)\n    plt.text(200, 300, v, fontsize = 13, bbox={'facecolor': 'white'})","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:42:46.00074Z","iopub.execute_input":"2023-12-15T05:42:46.001054Z","iopub.status.idle":"2023-12-15T05:43:02.454562Z","shell.execute_reply.started":"2023-12-15T05:42:46.001025Z","shell.execute_reply":"2023-12-15T05:43:02.4535Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# split dataset in two groups depending on view (to investigate them a bit more):\n# 1. MLO and CC - major group\n# 2. all other views - minor group\n\nmajor_group = ['CC', 'MLO']\n\ntrain_df_major_group = train_df[train_df.view.isin(major_group)]\ntrain_df_minor_group = train_df[~train_df.view.isin(major_group)]","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:43:02.456325Z","iopub.execute_input":"2023-12-15T05:43:02.457234Z","iopub.status.idle":"2023-12-15T05:43:02.479655Z","shell.execute_reply.started":"2023-12-15T05:43:02.457189Z","shell.execute_reply":"2023-12-15T05:43:02.478683Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_minor_group.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:43:02.481079Z","iopub.execute_input":"2023-12-15T05:43:02.481434Z","iopub.status.idle":"2023-12-15T05:43:02.489924Z","shell.execute_reply.started":"2023-12-15T05:43:02.481404Z","shell.execute_reply":"2023-12-15T05:43:02.488901Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# there are not so many examples of minor view group\n# let's display all of them\n\nplt.figure(figsize=(4, 4 * len(train_df_minor_group)))\n\nfor i, row in enumerate(train_df_minor_group.itertuples(index = False), start = 1):\n    plt.subplot(len(train_df_minor_group),1,i)\n    \n    img = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id))\n    img_arr = img.pixelData()\n    \n    # standardize all scans\n    img_arr = (img_arr - img_arr.min()) / (img_arr.max() - img_arr.min())\n    if img.PhotometricInterpretation == \"MONOCHROME1\":\n        img_arr = 1 - img_arr\n    \n    plt.imshow(img_arr, cmap = plt.cm.bone)\n    plt.text(200, 300, f'{row.patient_id} {row.image_id}', fontsize = 13, bbox={'facecolor': 'white'})\n    \n# a lot of these scans look bad!!!","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:43:02.491198Z","iopub.execute_input":"2023-12-15T05:43:02.491549Z","iopub.status.idle":"2023-12-15T05:43:52.57235Z","shell.execute_reply.started":"2023-12-15T05:43:02.4915Z","shell.execute_reply":"2023-12-15T05:43:52.571202Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# here i want to investigate patients which scans look weird\n\ndef display_scans_by_patient_id(ids):\n    plt.figure(figsize=(10 * 5, len(ids) * 7)) # let's consider max 10 pic per patient\n    for i, p_id in enumerate(ids):\n        df = train_df[train_df.patient_id == p_id].iloc[:10]\n\n        for j, row in enumerate(df.itertuples(index = False)):\n            plt.subplot(len(ids), 10, i * 10 + j + 1)\n\n            img = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id))\n            img_arr = img.pixelData()\n\n            # standardize all scans\n            img_arr = (img_arr - img_arr.min()) / (img_arr.max() - img_arr.min())\n            if img.PhotometricInterpretation == \"MONOCHROME1\":\n                img_arr = 1 - img_arr\n\n            plt.imshow(img_arr, cmap = plt.cm.bone)\n            plt.text(200, 300, f'{row.patient_id} {row.image_id}', fontsize = 20, bbox={'facecolor': 'white'})\n            plt.text(200, 800, f'{row.cancer}', fontsize = 20, bbox={'facecolor': 'white'})\n            \n\npatient_ids = [1511, 25323, 26530, 38739, 40317, 40832, 43377, 50454]\ndisplay_scans_by_patient_id(patient_ids)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:43:52.573762Z","iopub.execute_input":"2023-12-15T05:43:52.574081Z","iopub.status.idle":"2023-12-15T05:45:40.796213Z","shell.execute_reply.started":"2023-12-15T05:43:52.57405Z","shell.execute_reply":"2023-12-15T05:45:40.794705Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bad_ids = patient_ids.copy()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:45:40.797913Z","iopub.execute_input":"2023-12-15T05:45:40.798392Z","iopub.status.idle":"2023-12-15T05:45:40.803334Z","shell.execute_reply.started":"2023-12-15T05:45:40.798355Z","shell.execute_reply":"2023-12-15T05:45:40.802274Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ok, what's the goal:\n# I'm going to exclude patiens which have bad scans and don't have cancer (because we have a lot non-cancer scans)\n\n# all above patients have no cancer except 25323 patient\n# for this specific one I'm going to remove only 'bad' scans\nspecific_patient = 25_323\nbad_ids.remove(specific_patient)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:45:40.80474Z","iopub.execute_input":"2023-12-15T05:45:40.805424Z","iopub.status.idle":"2023-12-15T05:45:40.815106Z","shell.execute_reply.started":"2023-12-15T05:45:40.805386Z","shell.execute_reply":"2023-12-15T05:45:40.814078Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# thank you\n# https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/373208\nbad_ids.append(27_770) ","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:45:40.816479Z","iopub.execute_input":"2023-12-15T05:45:40.816836Z","iopub.status.idle":"2023-12-15T05:45:40.824936Z","shell.execute_reply.started":"2023-12-15T05:45:40.816796Z","shell.execute_reply":"2023-12-15T05:45:40.823938Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# let's check a few scans (30) which was marked as 'having cancer' from major view group\n# maybe we'll find something weird\n\ndf = train_df_major_group[~(train_df_major_group.patient_id.isin(patient_ids)) & (train_df_major_group.cancer == 1)]\ndf = df.groupby(['patient_id'])['image_id'].apply(lambda x : x.iloc[0]).to_frame().reset_index().sample(n = 30, random_state = 42)\n\nplt.figure(figsize=(6 * 5, 6 * 6)) # 6x5\n\nfor i, row in enumerate(df.itertuples(index = False), start = 1):\n    plt.subplot(6, 5, i)\n\n    img = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id))\n    img_arr = img.pixelData()\n    \n    # standardize all scans\n    img_arr = (img_arr - img_arr.min()) / (img_arr.max() - img_arr.min())\n    if img.PhotometricInterpretation == \"MONOCHROME1\":\n        img_arr = 1 - img_arr\n        \n    plt.imshow(img_arr, cmap = plt.cm.bone)\n    plt.text(200, 300, f'{row.patient_id}', fontsize = 13, bbox={'facecolor': 'white'})","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:45:40.826197Z","iopub.execute_input":"2023-12-15T05:45:40.826611Z","iopub.status.idle":"2023-12-15T05:46:58.823687Z","shell.execute_reply.started":"2023-12-15T05:45:40.826558Z","shell.execute_reply":"2023-12-15T05:46:58.822404Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# and without cancer\n\ndf = train_df_major_group[~(train_df_major_group.patient_id.isin(patient_ids)) & (train_df_major_group.cancer == 0)]\ndf = df.groupby(['patient_id'])['image_id'].apply(lambda x : x.iloc[0]).to_frame().reset_index().sample(n = 30, random_state = 42)\n\nplt.figure(figsize=(6 * 5, 6 * 6)) # 6x5\n\nfor i, row in enumerate(df.itertuples(), start = 1):\n    plt.subplot(6, 5, i)\n\n    img = dicomsdl.open(images_dir.format(train, row.patient_id, row.image_id))\n    img_arr = img.pixelData()\n    \n    # standardize all scans\n    img_arr = (img_arr - img_arr.min()) / (img_arr.max() - img_arr.min())\n    if img.PhotometricInterpretation == \"MONOCHROME1\":\n        img_arr = 1 - img_arr\n        \n    plt.imshow(img_arr, cmap = plt.cm.bone)\n    plt.text(200, 300, f'{row.patient_id}', fontsize = 13, bbox={'facecolor': 'white'})","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:46:58.825142Z","iopub.execute_input":"2023-12-15T05:46:58.825476Z","iopub.status.idle":"2023-12-15T05:48:07.840977Z","shell.execute_reply.started":"2023-12-15T05:46:58.825446Z","shell.execute_reply":"2023-12-15T05:48:07.839936Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# again investigate patients having 'weird' scans\n\npatient_ids = [33588, 12943]\ndisplay_scans_by_patient_id(patient_ids)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:07.842391Z","iopub.execute_input":"2023-12-15T05:48:07.842768Z","iopub.status.idle":"2023-12-15T05:48:27.600702Z","shell.execute_reply.started":"2023-12-15T05:48:07.842732Z","shell.execute_reply":"2023-12-15T05:48:27.59965Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Remove 'bad' scans","metadata":{}},{"cell_type":"code","source":"# patient_ids - list of patients having 'bad' scans\ntrain_df = train_df[~train_df.patient_id.isin(bad_ids)]\n\nindex_to_remove = train_df.loc[(train_df.patient_id == specific_patient) & ( train_df.image_id == 1743461841), :].index\ntrain_df.drop(index = index_to_remove, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:27.602276Z","iopub.execute_input":"2023-12-15T05:48:27.60304Z","iopub.status.idle":"2023-12-15T05:48:27.627236Z","shell.execute_reply.started":"2023-12-15T05:48:27.602994Z","shell.execute_reply":"2023-12-15T05:48:27.626375Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cancer_count = train_df.cancer.value_counts()\ncancer_count","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:27.628529Z","iopub.execute_input":"2023-12-15T05:48:27.628856Z","iopub.status.idle":"2023-12-15T05:48:27.637848Z","shell.execute_reply.started":"2023-12-15T05:48:27.628826Z","shell.execute_reply":"2023-12-15T05:48:27.63676Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"min_cancer_count = cancer_count.min()\n\ntrain_df = pd.concat([train_df[train_df.cancer == i].sample(n = min_cancer_count) \n                                  for i, _ in cancer_count.iteritems()])\n\nlen(train_df)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:27.639216Z","iopub.execute_input":"2023-12-15T05:48:27.639571Z","iopub.status.idle":"2023-12-15T05:48:27.663728Z","shell.execute_reply.started":"2023-12-15T05:48:27.63954Z","shell.execute_reply":"2023-12-15T05:48:27.662668Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Split the data into train and val sets","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val = train_test_split(train_df, test_size = 0.2, random_state = 42)\nlen(X_train), len(X_val)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:27.665062Z","iopub.execute_input":"2023-12-15T05:48:27.665425Z","iopub.status.idle":"2023-12-15T05:48:27.842873Z","shell.execute_reply.started":"2023-12-15T05:48:27.665392Z","shell.execute_reply":"2023-12-15T05:48:27.84184Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. DataGenerator","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:27.844138Z","iopub.execute_input":"2023-12-15T05:48:27.844462Z","iopub.status.idle":"2023-12-15T05:48:37.239815Z","shell.execute_reply.started":"2023-12-15T05:48:27.844432Z","shell.execute_reply":"2023-12-15T05:48:37.238604Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from enum import Enum, auto\n\nclass Mode(Enum):\n    TRAIN = auto()\n    TEST = auto()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:37.241314Z","iopub.execute_input":"2023-12-15T05:48:37.241938Z","iopub.status.idle":"2023-12-15T05:48:37.247679Z","shell.execute_reply.started":"2023-12-15T05:48:37.241904Z","shell.execute_reply":"2023-12-15T05:48:37.246554Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"img_height = 300\nimg_width = 250\nimg_shape = (img_height, img_width, 1)\n\nclass ImageDataGen(tf.keras.utils.Sequence):\n    \n    def __init__(self,\n                 df,\n                 batch_size,\n                 mode = Mode.TRAIN):\n\n        self.df = df\n        self.batch_size = batch_size\n        self.mode = mode\n        self.mode_str = train if mode == Mode.TRAIN else test\n        \n        self.len = len(df)\n        \n    def __getitem__(self, index):\n        \n        start, end = index * self.batch_size, (index + 1) * self.batch_size\n        \n        X = np.zeros((self.batch_size, ) + img_shape)\n        y = np.zeros((self.batch_size, 1))\n        \n        for i , pos in enumerate(range(start, end)):\n            if pos >= self.len: break\n                     \n            row = self.df.iloc[pos]\n            patient_id = row.patient_id\n            img_id = row.image_id\n            \n            file_name = images_dir.format(self.mode_str, patient_id, img_id)\n            \n            img = dicomsdl.open(file_name)\n            img_arr = img.pixelData()\n            \n            # standartize all scans\n            img_arr = (img_arr - img_arr.min()) / (img_arr.max() - img_arr.min())\n            \n            if img.PhotometricInterpretation == \"MONOCHROME1\":\n                img_arr = 1 - img_arr\n\n            img_arr = np.expand_dims(img_arr, axis = -1)\n            img_arr = tf.image.resize(img_arr, img_shape[:-1], method = 'nearest').numpy()\n                 \n            X[i,...] = img_arr\n                \n            \n            if self.mode == Mode.TRAIN:\n                y[i] = row.cancer\n                \n        return (X, y) if self.mode == Mode.TRAIN else X\n                \n    \n    def __len__(self):\n        return self.len // self.batch_size + bool(self.len % self.batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:37.255952Z","iopub.execute_input":"2023-12-15T05:48:37.256322Z","iopub.status.idle":"2023-12-15T05:48:38.590563Z","shell.execute_reply.started":"2023-12-15T05:48:37.256281Z","shell.execute_reply":"2023-12-15T05:48:38.589638Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_gen = ImageDataGen(X_train, 50)\nval_gen = ImageDataGen(X_val, 50)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:38.591742Z","iopub.execute_input":"2023-12-15T05:48:38.592056Z","iopub.status.idle":"2023-12-15T05:48:38.597433Z","shell.execute_reply.started":"2023-12-15T05:48:38.592026Z","shell.execute_reply":"2023-12-15T05:48:38.596201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Model","metadata":{}},{"cell_type":"code","source":"import tensorflow.keras as K\n\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.layers import (Conv2D, \n                                     MaxPooling2D, \n                                     BatchNormalization, \n                                     Dense, \n                                     Dropout,\n                                     GlobalMaxPooling2D)","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:38.599013Z","iopub.execute_input":"2023-12-15T05:48:38.599863Z","iopub.status.idle":"2023-12-15T05:48:38.616079Z","shell.execute_reply.started":"2023-12-15T05:48:38.59981Z","shell.execute_reply":"2023-12-15T05:48:38.615298Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# build a simple model\n\nmodel = Sequential()\n\nmodel.add(Conv2D(32, 5, activation = \"relu\", input_shape = img_shape))\nmodel.add(Conv2D(64, 5, activation = \"relu\"))\nmodel.add(Conv2D(64, 5, activation = \"relu\"))\nmodel.add(MaxPooling2D())\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv2D(64, 5, activation = \"relu\"))\nmodel.add(Conv2D(128, 5, activation = \"relu\"))\nmodel.add(Conv2D(128, 5, activation = \"relu\"))\nmodel.add(MaxPooling2D())\nmodel.add(Dropout(0.3))\n\nmodel.add(Conv2D(128, 5, activation = \"relu\"))\nmodel.add(Conv2D(256, 5, activation = \"relu\"))\nmodel.add(Conv2D(256, 5, activation = \"relu\"))\nmodel.add(GlobalMaxPooling2D())\nmodel.add(Dropout(0.3))\n\nmodel.add(Dense(64, activation = 'relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation = 'sigmoid'))\n\nrecall_thresholds = [0.4, 0.5, 0.6, 0.8]\nmodel.compile(optimizer = Adam(learning_rate = 5e-5), \n              loss = 'binary_crossentropy', \n              metrics = [tf.keras.metrics.BinaryAccuracy(threshold = 0.5), \n                         tf.keras.metrics.Recall(thresholds = recall_thresholds),\n                         tf.keras.metrics.Precision(thresholds = recall_thresholds)])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:38.617373Z","iopub.execute_input":"2023-12-15T05:48:38.617695Z","iopub.status.idle":"2023-12-15T05:48:43.904855Z","shell.execute_reply.started":"2023-12-15T05:48:38.617666Z","shell.execute_reply":"2023-12-15T05:48:43.903839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# callbacks\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nearly_stop = EarlyStopping(patience = 5, restore_best_weights = True, verbose = 1) # val_loss\nreduce_lr = ReduceLROnPlateau(factor = 0.1, patience = 2, mode = 'min', verbose = 1) # val_loss ","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:43.906227Z","iopub.execute_input":"2023-12-15T05:48:43.90656Z","iopub.status.idle":"2023-12-15T05:48:43.913471Z","shell.execute_reply.started":"2023-12-15T05:48:43.906529Z","shell.execute_reply":"2023-12-15T05:48:43.912307Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history = model.fit(train_gen,\n                    validation_data = val_gen,\n                    epochs = 15,\n                    verbose = 1,\n                    workers = 8,\n                    callbacks = [reduce_lr])","metadata":{"execution":{"iopub.status.busy":"2023-12-15T05:48:43.914837Z","iopub.execute_input":"2023-12-15T05:48:43.915151Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.save('model.h5')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"loss = history.history['loss']\nval_loss = history.history['val_loss']\n\nacc = history.history['binary_accuracy']\nval_acc = history.history['val_binary_accuracy']\n\n\nepochs = range(1, len(loss) + 1)\n\nplt.figure(figsize=(16, 5))\n#accuracy\nplt.subplot(1,2,1)\nplt.plot(epochs, acc, 'bo', label = 'Training accuracy')\nplt.plot(epochs, val_acc, 'r', label = 'Validation accuracy')\nplt.legend()\n\n#loss\nplt.subplot(1,2,2)\nplt.plot(epochs, loss, 'bo', label = 'Trainig loss')\nplt.plot(epochs, val_loss, 'r', label = 'Validation loss')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"recall = history.history['recall']\nval_recall = history.history['val_recall']\n\nrecall_len = len(recall_thresholds)\nplt.figure(figsize=(10, 5 * recall_len))\n\nfor i, (r, vr) in enumerate(zip(zip(*recall), zip(*val_recall)), start = 1):\n    plt.subplot(recall_len, 1, i)\n    plt.plot(epochs, r, 'bo', label = 'Training recall')\n    plt.plot(epochs, vr, 'r', label = 'Validation recall')\n    plt.title(f'Recall threshold = {recall_thresholds[i - 1]}')\n    plt.legend()\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"precision = history.history['precision']\nval_precision = history.history['val_precision']\n\nrecall_len = len(recall_thresholds)\nplt.figure(figsize=(10, 5 * recall_len))\n\nfor i, (p, vp) in enumerate(zip(zip(*precision), zip(*val_precision)), start = 1):\n    plt.subplot(recall_len, 1, i)\n    plt.plot(epochs, p, 'bo', label = 'Training precision')\n    plt.plot(epochs, vp, 'r', label = 'Validation precision')\n    plt.title(f'Precision threshold = {recall_thresholds[i - 1]}')\n    plt.legend()\n\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Test dataset","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_gen = ImageDataGen(test_df, 16, mode = Mode.TEST)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred = model.predict(test_gen)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df['cancer'] = pred[:len(test_df)]\ntest_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = test_df.groupby('prediction_id')['cancer'].max().to_frame().reset_index()\nsub.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index = False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}