{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":39272,"databundleVersionId":4629629,"sourceType":"competition"},{"sourceId":4619805,"sourceType":"datasetVersion","datasetId":2688675},{"sourceId":4836351,"sourceType":"datasetVersion","datasetId":2802588}],"dockerImageVersionId":30132,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:02:56.811985Z","iopub.execute_input":"2024-02-02T18:02:56.812868Z","iopub.status.idle":"2024-02-02T18:02:56.81773Z","shell.execute_reply.started":"2024-02-02T18:02:56.812827Z","shell.execute_reply":"2024-02-02T18:02:56.81688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:01.839804Z","iopub.execute_input":"2024-02-02T18:03:01.840498Z","iopub.status.idle":"2024-02-02T18:03:02.416211Z","shell.execute_reply.started":"2024-02-02T18:03:01.840447Z","shell.execute_reply":"2024-02-02T18:03:02.415387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the data\n\n# Get current working directory\ncurrent_dir = os.getcwd() \n\n# Append data/mnist.npz to the previous path to get the full path\ndata_path = \"/kaggle/input/rsna-breast-cancer-detection/train_images\"","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:09.312478Z","iopub.execute_input":"2024-02-02T18:03:09.313207Z","iopub.status.idle":"2024-02-02T18:03:09.317145Z","shell.execute_reply.started":"2024-02-02T18:03:09.313171Z","shell.execute_reply":"2024-02-02T18:03:09.316381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_and_normalize(images):\n    \n    # Reshape the images to add an extra dimension\n    # images = images[..., np.newaxis]\n    \n    # Normalize pixel values\n    images = images / 255.0\n    \n    ### END CODE HERE\n    return images","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:15.530614Z","iopub.execute_input":"2024-02-02T18:03:15.531285Z","iopub.status.idle":"2024-02-02T18:03:15.535626Z","shell.execute_reply.started":"2024-02-02T18:03:15.531252Z","shell.execute_reply":"2024-02-02T18:03:15.534758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ntest_csv = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:21.492533Z","iopub.execute_input":"2024-02-02T18:03:21.492824Z","iopub.status.idle":"2024-02-02T18:03:21.606837Z","shell.execute_reply.started":"2024-02-02T18:03:21.492793Z","shell.execute_reply":"2024-02-02T18:03:21.606016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:24.105787Z","iopub.execute_input":"2024-02-02T18:03:24.106434Z","iopub.status.idle":"2024-02-02T18:03:24.152331Z","shell.execute_reply.started":"2024-02-02T18:03:24.106394Z","shell.execute_reply":"2024-02-02T18:03:24.151488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:03:34.687419Z","iopub.execute_input":"2024-02-02T18:03:34.6877Z","iopub.status.idle":"2024-02-02T18:03:34.700767Z","shell.execute_reply.started":"2024-02-02T18:03:34.687671Z","shell.execute_reply":"2024-02-02T18:03:34.70001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(set(train_csv.patient_id))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:04:12.029122Z","iopub.execute_input":"2024-02-02T18:04:12.029835Z","iopub.status.idle":"2024-02-02T18:04:12.045597Z","shell.execute_reply.started":"2024-02-02T18:04:12.029796Z","shell.execute_reply":"2024-02-02T18:04:12.044805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv.laterality.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:06:50.389971Z","iopub.execute_input":"2024-02-02T18:06:50.390312Z","iopub.status.idle":"2024-02-02T18:06:50.413146Z","shell.execute_reply.started":"2024-02-02T18:06:50.390263Z","shell.execute_reply":"2024-02-02T18:06:50.41238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_0 = train_csv[train_csv.cancer == 0]\ntrain_subset_1 = train_csv[train_csv.cancer == 1]\nprint(train_subset_0.shape, train_subset_1.shape)\nprint(train_subset_0.laterality.value_counts())\nprint(train_subset_1.laterality.value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:06:54.516878Z","iopub.execute_input":"2024-02-02T18:06:54.517687Z","iopub.status.idle":"2024-02-02T18:06:54.549045Z","shell.execute_reply.started":"2024-02-02T18:06:54.517644Z","shell.execute_reply":"2024-02-02T18:06:54.54816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"take data subset","metadata":{}},{"cell_type":"code","source":"train_subset_0_L = train_subset_0[train_subset_0.laterality == \"L\"].iloc[:588,]\ntrain_subset_0_R = train_subset_0[train_subset_0.laterality == \"R\"].iloc[:570,]\ntrain_subset_main = pd.concat([train_subset_0_L, train_subset_0_R, train_subset_1])","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:06:59.196037Z","iopub.execute_input":"2024-02-02T18:06:59.196637Z","iopub.status.idle":"2024-02-02T18:06:59.233201Z","shell.execute_reply.started":"2024-02-02T18:06:59.196599Z","shell.execute_reply":"2024-02-02T18:06:59.232476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_main","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:03.747882Z","iopub.execute_input":"2024-02-02T18:07:03.748185Z","iopub.status.idle":"2024-02-02T18:07:03.784088Z","shell.execute_reply.started":"2024-02-02T18:07:03.748151Z","shell.execute_reply":"2024-02-02T18:07:03.783234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_subset_main.shape","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:20.537288Z","iopub.execute_input":"2024-02-02T18:07:20.537659Z","iopub.status.idle":"2024-02-02T18:07:20.544262Z","shell.execute_reply.started":"2024-02-02T18:07:20.537607Z","shell.execute_reply":"2024-02-02T18:07:20.543503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/input_transformed/')","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:24.90853Z","iopub.execute_input":"2024-02-02T18:07:24.909338Z","iopub.status.idle":"2024-02-02T18:07:24.913335Z","shell.execute_reply.started":"2024-02-02T18:07:24.90928Z","shell.execute_reply":"2024-02-02T18:07:24.912444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir('/kaggle/working/input_transformed/0/')\nos.mkdir('/kaggle/working/input_transformed/1/')","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:27.827543Z","iopub.execute_input":"2024-02-02T18:07:27.828347Z","iopub.status.idle":"2024-02-02T18:07:27.832326Z","shell.execute_reply.started":"2024-02-02T18:07:27.828297Z","shell.execute_reply":"2024-02-02T18:07:27.831523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\nfrom tqdm import tqdm\n# shutil.copyfile(src, dst)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:30.80213Z","iopub.execute_input":"2024-02-02T18:07:30.80245Z","iopub.status.idle":"2024-02-02T18:07:30.806694Z","shell.execute_reply.started":"2024-02-02T18:07:30.802416Z","shell.execute_reply":"2024-02-02T18:07:30.80579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_id = train_subset_main.patient_id\ni_id = train_subset_main.image_id\ncncr = train_subset_main.cancer\nfor pp, ii, cc in tqdm(zip(p_id, i_id, cncr)):\n    tmpFile = str(pp) + \"_\" + str(ii) + \".png\"\n    tmpSrc = \"/kaggle/input/rsna-breast-cancer-512-pngs/\" + tmpFile\n    tmpDst = \"/kaggle/working/input_transformed/\" + str(cc) + \"/\" + tmpFile\n    shutil.copyfile(tmpSrc, tmpDst)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:07:34.357144Z","iopub.execute_input":"2024-02-02T18:07:34.35746Z","iopub.status.idle":"2024-02-02T18:08:02.198115Z","shell.execute_reply.started":"2024-02-02T18:07:34.357427Z","shell.execute_reply":"2024-02-02T18:08:02.197269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    \"/kaggle/working/input_transformed/\",\n    color_mode='rgb',\n    image_size=(512, 512),\n    shuffle=True,\n    validation_split=0.2,\n    subset=\"training\",\n    seed=2023)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:08:23.284998Z","iopub.execute_input":"2024-02-02T18:08:23.285838Z","iopub.status.idle":"2024-02-02T18:08:25.551277Z","shell.execute_reply.started":"2024-02-02T18:08:23.285791Z","shell.execute_reply":"2024-02-02T18:08:25.550534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_ds = tf.keras.preprocessing.image_dataset_from_directory(\n    \"/kaggle/working/input_transformed/\",\n    color_mode='rgb',\n    image_size=(512, 512),\n    shuffle=True,\n    validation_split=0.2,\n    subset=\"validation\",\n    seed=2023)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:08:38.820199Z","iopub.execute_input":"2024-02-02T18:08:38.821056Z","iopub.status.idle":"2024-02-02T18:08:38.947011Z","shell.execute_reply.started":"2024-02-02T18:08:38.821011Z","shell.execute_reply":"2024-02-02T18:08:38.94617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Dropout","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:08:42.795936Z","iopub.execute_input":"2024-02-02T18:08:42.796251Z","iopub.status.idle":"2024-02-02T18:08:42.801428Z","shell.execute_reply.started":"2024-02-02T18:08:42.796217Z","shell.execute_reply":"2024-02-02T18:08:42.800506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> **#CNN Model**","metadata":{}},{"cell_type":"code","source":"num_classes = 2\n\nmodel = Sequential([\n  layers.experimental.preprocessing.Rescaling(1./255, input_shape=(512, 512, 3)),\n  layers.Conv2D(16, 3, padding='same', activation='relu'),  \n  layers.MaxPooling2D(),   \n  layers.Conv2D(32, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Dropout(0.2),\n  layers.Conv2D(64, 3, padding='same', activation='relu'),\n  layers.MaxPooling2D(),\n  layers.Flatten(),\n  layers.Dense(128, activation='relu'),\n  layers.Dense(num_classes)\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:09:40.437403Z","iopub.execute_input":"2024-02-02T18:09:40.437724Z","iopub.status.idle":"2024-02-02T18:09:40.511843Z","shell.execute_reply.started":"2024-02-02T18:09:40.437692Z","shell.execute_reply":"2024-02-02T18:09:40.511028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:09:44.713495Z","iopub.execute_input":"2024-02-02T18:09:44.714349Z","iopub.status.idle":"2024-02-02T18:09:44.724024Z","shell.execute_reply.started":"2024-02-02T18:09:44.714297Z","shell.execute_reply":"2024-02-02T18:09:44.722034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n              metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:09:48.306905Z","iopub.execute_input":"2024-02-02T18:09:48.307177Z","iopub.status.idle":"2024-02-02T18:09:48.326517Z","shell.execute_reply.started":"2024-02-02T18:09:48.307149Z","shell.execute_reply":"2024-02-02T18:09:48.325549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class myCallback(tf.keras.callbacks.Callback):\n    # Define the method that checks the accuracy at the end of each epoch\n    def on_epoch_end(self, epoch, logs={}):\n        if logs.get('accuracy') is not None and logs.get('accuracy') >= 0.95:\n            print(\"\\nReached 99.5% accuracy so cancelling training!\") \n            # Stop training once the above condition is met\n            self.model.stop_training = True","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:09:51.375692Z","iopub.execute_input":"2024-02-02T18:09:51.376018Z","iopub.status.idle":"2024-02-02T18:09:51.381932Z","shell.execute_reply.started":"2024-02-02T18:09:51.375986Z","shell.execute_reply":"2024-02-02T18:09:51.381196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs=5\nhistory = model.fit(\n  train_ds,\n  validation_data=valid_ds,\n  epochs=epochs\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:09:54.527089Z","iopub.execute_input":"2024-02-02T18:09:54.527918Z","iopub.status.idle":"2024-02-02T18:11:42.971753Z","shell.execute_reply.started":"2024-02-02T18:09:54.527875Z","shell.execute_reply":"2024-02-02T18:11:42.970899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs_range = range(epochs)\n\nplt.figure(figsize=(8, 8))\nplt.subplot(1, 2, 1)\nplt.plot(epochs_range, acc, label='Training Accuracy')\nplt.plot(epochs_range, val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(epochs_range, loss, label='Training Loss')\nplt.plot(epochs_range, val_loss, label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:15.078792Z","iopub.execute_input":"2024-02-02T18:13:15.079588Z","iopub.status.idle":"2024-02-02T18:13:15.519239Z","shell.execute_reply.started":"2024-02-02T18:13:15.079546Z","shell.execute_reply":"2024-02-02T18:13:15.518431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport numpy as np\nfrom skimage import transform\ndef load(filename):\n   np_image = Image.open(filename)\n   np_image = np.array(np_image).astype('float32')/255\n   np_image = transform.resize(np_image, (512, 512, 3))\n   np_image = np.expand_dims(np_image, axis=0)\n   return np_image","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:36.913083Z","iopub.execute_input":"2024-02-02T18:13:36.913397Z","iopub.status.idle":"2024-02-02T18:13:37.186463Z","shell.execute_reply.started":"2024-02-02T18:13:36.913363Z","shell.execute_reply":"2024-02-02T18:13:37.185562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testdata_path = \"/kaggle/input/rsna-screen-breast-cancer-detect-testdata-512x512\"\npred_dict = dict()\nfor ii in os.listdir(testdata_path):\n    tmpPath = testdata_path + \"/\" + ii\n    image = load(tmpPath)\n    predictions = model.predict(image)\n    score = tf.nn.softmax(predictions[0])\n    pred_dict[ii] = float(max(score))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:39.136104Z","iopub.execute_input":"2024-02-02T18:13:39.136888Z","iopub.status.idle":"2024-02-02T18:13:39.991645Z","shell.execute_reply.started":"2024-02-02T18:13:39.136849Z","shell.execute_reply":"2024-02-02T18:13:39.990724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pprint import pprint\ntestD = {\"10008_L\":{\"736471439.png\":0, \"1591370361.png\":0}, \n         \"10008_R\":{\"68070693.png\":0,\"361203119.png\":0}}\n\nfor k1 in testD:\n    for k2 in testD[k1]:\n        testD[k1][k2] = pred_dict[k2]\n\npprint(testD)","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:43.048373Z","iopub.execute_input":"2024-02-02T18:13:43.048687Z","iopub.status.idle":"2024-02-02T18:13:43.057486Z","shell.execute_reply.started":"2024-02-02T18:13:43.048653Z","shell.execute_reply":"2024-02-02T18:13:43.056509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/sample_submission.csv\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:47.823176Z","iopub.execute_input":"2024-02-02T18:13:47.823915Z","iopub.status.idle":"2024-02-02T18:13:47.843105Z","shell.execute_reply.started":"2024-02-02T18:13:47.823877Z","shell.execute_reply":"2024-02-02T18:13:47.842333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for jj in range(submission.shape[0]):\n    tmpKey = submission.prediction_id.iloc[jj]\n    submission.cancer.iloc[jj] = np.mean(list(testD[tmpKey].values()))","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:13:51.570026Z","iopub.execute_input":"2024-02-02T18:13:51.570348Z","iopub.status.idle":"2024-02-02T18:13:51.58135Z","shell.execute_reply.started":"2024-02-02T18:13:51.570314Z","shell.execute_reply":"2024-02-02T18:13:51.580494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\")\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-02-02T18:15:35.791994Z","iopub.execute_input":"2024-02-02T18:15:35.792802Z","iopub.status.idle":"2024-02-02T18:15:35.806454Z","shell.execute_reply.started":"2024-02-02T18:15:35.792766Z","shell.execute_reply":"2024-02-02T18:15:35.805588Z"},"trusted":true},"execution_count":null,"outputs":[]}]}