{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.017922Z","iopub.execute_input":"2023-01-11T04:10:24.018203Z","iopub.status.idle":"2023-01-11T04:10:24.023027Z","shell.execute_reply.started":"2023-01-11T04:10:24.018174Z","shell.execute_reply":"2023-01-11T04:10:24.02215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data\n\n# Get current working directory\ncurrent_dir = os.getcwd() \n\n# Append data/mnist.npz to the previous path to get the full path\ndata_path = \"/kaggle/input/rsna-breast-cancer-detection/train_images\"","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.035506Z","iopub.execute_input":"2023-01-11T04:10:24.036466Z","iopub.status.idle":"2023-01-11T04:10:24.041329Z","shell.execute_reply.started":"2023-01-11T04:10:24.036423Z","shell.execute_reply":"2023-01-11T04:10:24.040548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_and_normalize(images):\n    \n    # Reshape the images to add an extra dimension\n    # images = images[..., np.newaxis]\n    \n    # Normalize pixel values\n    images = images / 255.0\n    \n    ### END CODE HERE\n    return images","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.043061Z","iopub.execute_input":"2023-01-11T04:10:24.043586Z","iopub.status.idle":"2023-01-11T04:10:24.050275Z","shell.execute_reply.started":"2023-01-11T04:10:24.043549Z","shell.execute_reply":"2023-01-11T04:10:24.049463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntest_csv = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.05191Z","iopub.execute_input":"2023-01-11T04:10:24.052269Z","iopub.status.idle":"2023-01-11T04:10:24.136293Z","shell.execute_reply.started":"2023-01-11T04:10:24.052235Z","shell.execute_reply":"2023-01-11T04:10:24.135281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.137852Z","iopub.execute_input":"2023-01-11T04:10:24.138256Z","iopub.status.idle":"2023-01-11T04:10:24.171539Z","shell.execute_reply.started":"2023-01-11T04:10:24.138219Z","shell.execute_reply":"2023-01-11T04:10:24.170572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.173109Z","iopub.execute_input":"2023-01-11T04:10:24.174138Z","iopub.status.idle":"2023-01-11T04:10:24.187949Z","shell.execute_reply.started":"2023-01-11T04:10:24.17407Z","shell.execute_reply":"2023-01-11T04:10:24.187046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.190658Z","iopub.execute_input":"2023-01-11T04:10:24.190983Z","iopub.status.idle":"2023-01-11T04:10:24.199534Z","shell.execute_reply.started":"2023-01-11T04:10:24.190943Z","shell.execute_reply":"2023-01-11T04:10:24.198661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.201234Z","iopub.execute_input":"2023-01-11T04:10:24.20171Z","iopub.status.idle":"2023-01-11T04:10:24.211996Z","shell.execute_reply.started":"2023-01-11T04:10:24.201677Z","shell.execute_reply":"2023-01-11T04:10:24.211146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(train_csv.patient_id))","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.213589Z","iopub.execute_input":"2023-01-11T04:10:24.214252Z","iopub.status.idle":"2023-01-11T04:10:24.227153Z","shell.execute_reply.started":"2023-01-11T04:10:24.214219Z","shell.execute_reply":"2023-01-11T04:10:24.226423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.age.hist()","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.228596Z","iopub.execute_input":"2023-01-11T04:10:24.228968Z","iopub.status.idle":"2023-01-11T04:10:24.48424Z","shell.execute_reply.started":"2023-01-11T04:10:24.228934Z","shell.execute_reply":"2023-01-11T04:10:24.483418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.laterality.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.485716Z","iopub.execute_input":"2023-01-11T04:10:24.485987Z","iopub.status.idle":"2023-01-11T04:10:24.49732Z","shell.execute_reply.started":"2023-01-11T04:10:24.485952Z","shell.execute_reply":"2023-01-11T04:10:24.496459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_0 = train_csv[train_csv.cancer == 0]\ntrain_subset_1 = train_csv[train_csv.cancer == 1]\nprint(train_subset_0.shape, train_subset_1.shape)\nprint(train_subset_0.laterality.value_counts())\nprint(train_subset_1.laterality.value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.498955Z","iopub.execute_input":"2023-01-11T04:10:24.49931Z","iopub.status.idle":"2023-01-11T04:10:24.521566Z","shell.execute_reply.started":"2023-01-11T04:10:24.499272Z","shell.execute_reply":"2023-01-11T04:10:24.520709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_0_L = train_subset_0[train_subset_0.laterality == \"L\"].iloc[:588,]\ntrain_subset_0_R = train_subset_0[train_subset_0.laterality == \"R\"].iloc[:570,]\ntrain_subset_main = pd.concat([train_subset_0_L, train_subset_0_R, train_subset_1])","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.524437Z","iopub.execute_input":"2023-01-11T04:10:24.524702Z","iopub.status.idle":"2023-01-11T04:10:24.546536Z","shell.execute_reply.started":"2023-01-11T04:10:24.524672Z","shell.execute_reply":"2023-01-11T04:10:24.545828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_main","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.54788Z","iopub.execute_input":"2023-01-11T04:10:24.548182Z","iopub.status.idle":"2023-01-11T04:10:24.580314Z","shell.execute_reply.started":"2023-01-11T04:10:24.548148Z","shell.execute_reply":"2023-01-11T04:10:24.579278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_main.laterality.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.58194Z","iopub.execute_input":"2023-01-11T04:10:24.582244Z","iopub.status.idle":"2023-01-11T04:10:24.591498Z","shell.execute_reply.started":"2023-01-11T04:10:24.582207Z","shell.execute_reply":"2023-01-11T04:10:24.590445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_main.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.593432Z","iopub.execute_input":"2023-01-11T04:10:24.593751Z","iopub.status.idle":"2023-01-11T04:10:24.600754Z","shell.execute_reply.started":"2023-01-11T04:10:24.593714Z","shell.execute_reply":"2023-01-11T04:10:24.599636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/input_transformed/')","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.602399Z","iopub.execute_input":"2023-01-11T04:10:24.602707Z","iopub.status.idle":"2023-01-11T04:10:24.610896Z","shell.execute_reply.started":"2023-01-11T04:10:24.602671Z","shell.execute_reply":"2023-01-11T04:10:24.60969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/input_transformed/0/')\nos.mkdir('/kaggle/working/input_transformed/1/')","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.612539Z","iopub.execute_input":"2023-01-11T04:10:24.612909Z","iopub.status.idle":"2023-01-11T04:10:24.621379Z","shell.execute_reply.started":"2023-01-11T04:10:24.612874Z","shell.execute_reply":"2023-01-11T04:10:24.620457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nfrom tqdm import tqdm\n# shutil.copyfile(src, dst)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.622837Z","iopub.execute_input":"2023-01-11T04:10:24.623093Z","iopub.status.idle":"2023-01-11T04:10:24.630529Z","shell.execute_reply.started":"2023-01-11T04:10:24.623062Z","shell.execute_reply":"2023-01-11T04:10:24.629705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_id = train_subset_main.patient_id\ni_id = train_subset_main.image_id\ncncr = train_subset_main.cancer\nfor pp, ii, cc in tqdm(zip(p_id, i_id, cncr)):\n    tmpFile = str(pp) + \"_\" + str(ii) + \".png\"\n    tmpSrc = \"/kaggle/input/rsna-breast-cancer-512-pngs/\" + tmpFile\n    tmpDst = \"/kaggle/working/input_transformed/\" + str(cc) + \"/\" + tmpFile\n    shutil.copyfile(tmpSrc, tmpDst)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:24.631994Z","iopub.execute_input":"2023-01-11T04:10:24.632329Z","iopub.status.idle":"2023-01-11T04:10:36.984321Z","shell.execute_reply.started":"2023-01-11T04:10:24.632296Z","shell.execute_reply":"2023-01-11T04:10:36.983577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    \"/kaggle/working/input_transformed/\",\n    color_mode='rgb',\n    image_size=(512, 512),\n    shuffle=True,\n    validation_split=0.2,\n    subset=\"training\",\n    seed=2023)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:36.985823Z","iopub.execute_input":"2023-01-11T04:10:36.986339Z","iopub.status.idle":"2023-01-11T04:10:40.257519Z","shell.execute_reply.started":"2023-01-11T04:10:36.986299Z","shell.execute_reply":"2023-01-11T04:10:40.256691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    \"/kaggle/working/input_transformed/\",\n    color_mode='rgb',\n    image_size=(512, 512),\n    shuffle=True,\n    validation_split=0.2,\n    subset=\"validation\",\n    seed=2023)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:40.258844Z","iopub.execute_input":"2023-01-11T04:10:40.259093Z","iopub.status.idle":"2023-01-11T04:10:40.381207Z","shell.execute_reply.started":"2023-01-11T04:10:40.25906Z","shell.execute_reply":"2023-01-11T04:10:40.379769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 2\n\nmodel = Sequential([\n  layers.experimental.preprocessing.Rescaling(1./255, input_shape=(512, 512, 3)),\n  layers.Conv2D(16, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(32, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Conv2D(64, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Flatten(),\n  layers.Dense(128, activation='relu'),\n  layers.Dense(num_classes)\n])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:40.382665Z","iopub.execute_input":"2023-01-11T04:10:40.382941Z","iopub.status.idle":"2023-01-11T04:10:40.478532Z","shell.execute_reply.started":"2023-01-11T04:10:40.382906Z","shell.execute_reply":"2023-01-11T04:10:40.477815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:40.479607Z","iopub.execute_input":"2023-01-11T04:10:40.479888Z","iopub.status.idle":"2023-01-11T04:10:40.497104Z","shell.execute_reply.started":"2023-01-11T04:10:40.479854Z","shell.execute_reply":"2023-01-11T04:10:40.496153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class myCallback(tf.keras.callbacks.Callback):\n    # Define the method that checks the accuracy at the end of each epoch\n    def on_epoch_end(self, epoch, logs={}):\n        if logs.get('accuracy') is not None and logs.get('accuracy') >= 0.95:\n            print(\"\\nReached 99.5% accuracy so cancelling training!\") \n            # Stop training once the above condition is met\n            self.model.stop_training = True","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:40.498542Z","iopub.execute_input":"2023-01-11T04:10:40.498818Z","iopub.status.idle":"2023-01-11T04:10:40.504457Z","shell.execute_reply.started":"2023-01-11T04:10:40.498785Z","shell.execute_reply":"2023-01-11T04:10:40.503596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=2\nhistory = model.fit(\n  train_ds,\n  validation_data=valid_ds,\n  epochs=epochs\n)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:10:40.505914Z","iopub.execute_input":"2023-01-11T04:10:40.506345Z","iopub.status.idle":"2023-01-11T04:11:39.069948Z","shell.execute_reply.started":"2023-01-11T04:10:40.506309Z","shell.execute_reply":"2023-01-11T04:11:39.068988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(epochs)\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:11:39.071862Z","iopub.execute_input":"2023-01-11T04:11:39.072112Z","iopub.status.idle":"2023-01-11T04:11:39.456631Z","shell.execute_reply.started":"2023-01-11T04:11:39.072082Z","shell.execute_reply":"2023-01-11T04:11:39.455828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nfrom skimage import transform\ndef load(filename):\n   np_image = Image.open(filename)\n   np_image = np.array(np_image).astype('float32')/255\n   np_image = transform.resize(np_image, (512, 512, 3))\n   np_image = np.expand_dims(np_image, axis=0)\n   return np_image","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:28:20.414725Z","iopub.execute_input":"2023-01-11T04:28:20.415005Z","iopub.status.idle":"2023-01-11T04:28:20.420977Z","shell.execute_reply.started":"2023-01-11T04:28:20.414976Z","shell.execute_reply":"2023-01-11T04:28:20.420226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata_path = \"/kaggle/input/rsna-screen-breast-cancer-detect-testdata-512x512\"\npred_dict = dict()\nfor ii in os.listdir(testdata_path):\n    tmpPath = testdata_path + \"/\" + ii\n    image = load(tmpPath)\n    predictions = model.predict(image)\n    score = tf.nn.softmax(predictions[0])\n    pred_dict[ii] = float(max(score))","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:42:19.555313Z","iopub.execute_input":"2023-01-11T04:42:19.555629Z","iopub.status.idle":"2023-01-11T04:42:20.176889Z","shell.execute_reply.started":"2023-01-11T04:42:19.555597Z","shell.execute_reply":"2023-01-11T04:42:20.175964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pprint import pprint\ntestD = {\"10008_L\":{\"736471439.png\":0, \"1591370361.png\":0}, \n         \"10008_R\":{\"68070693.png\":0,\"361203119.png\":0}}\n\nfor k1 in testD:\n    for k2 in testD[k1]:\n        testD[k1][k2] = pred_dict[k2]\n\npprint(testD)","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:52:09.855665Z","iopub.execute_input":"2023-01-11T04:52:09.85596Z","iopub.status.idle":"2023-01-11T04:52:09.863427Z","shell.execute_reply.started":"2023-01-11T04:52:09.85593Z","shell.execute_reply":"2023-01-11T04:52:09.862573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-11T04:51:07.265547Z","iopub.execute_input":"2023-01-11T04:51:07.265841Z","iopub.status.idle":"2023-01-11T04:51:07.287328Z","shell.execute_reply.started":"2023-01-11T04:51:07.265813Z","shell.execute_reply":"2023-01-11T04:51:07.286408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for jj in range(submission.shape[0]):\n    tmpKey = submission.prediction_id.iloc[jj]\n    submission.cancer.iloc[jj] = np.mean(list(testD[tmpKey].values()))","metadata":{"execution":{"iopub.status.busy":"2023-01-11T05:02:19.857967Z","iopub.execute_input":"2023-01-11T05:02:19.858492Z","iopub.status.idle":"2023-01-11T05:02:19.87153Z","shell.execute_reply.started":"2023-01-11T05:02:19.858424Z","shell.execute_reply":"2023-01-11T05:02:19.870641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-11T05:03:00.426558Z","iopub.execute_input":"2023-01-11T05:03:00.426836Z","iopub.status.idle":"2023-01-11T05:03:00.442148Z","shell.execute_reply.started":"2023-01-11T05:03:00.426807Z","shell.execute_reply":"2023-01-11T05:03:00.441299Z"},"trusted":true},"execution_count":null,"outputs":[]}]}