{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":13451,"databundleVersionId":1188070,"sourceType":"competition"},{"sourceId":762454,"sourceType":"datasetVersion","datasetId":396552},{"sourceId":20989393,"sourceType":"kernelVersion"},{"sourceId":22701660,"sourceType":"kernelVersion"},{"sourceId":23446541,"sourceType":"kernelVersion"},{"sourceId":55994290,"sourceType":"kernelVersion"}],"dockerImageVersionId":30579,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import glob, pylab, pandas as pd\nimport pydicom, numpy as np\nfrom os import listdir\nfrom os.path import isfile, join\nimport matplotlib.pylab as plt\nimport os\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-22T10:56:55.232319Z","iopub.execute_input":"2023-11-22T10:56:55.232703Z","iopub.status.idle":"2023-11-22T10:56:56.504197Z","shell.execute_reply.started":"2023-11-22T10:56:55.232669Z","shell.execute_reply":"2023-11-22T10:56:56.503335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras import layers\nfrom keras.applications import DenseNet121\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import Callback, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:56:56.506435Z","iopub.execute_input":"2023-11-22T10:56:56.507165Z","iopub.status.idle":"2023-11-22T10:57:09.758281Z","shell.execute_reply.started":"2023-11-22T10:56:56.507121Z","shell.execute_reply":"2023-11-22T10:57:09.757376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH=\"../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection\"\n!ls ../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:57:09.759631Z","iopub.execute_input":"2023-11-22T10:57:09.760218Z","iopub.status.idle":"2023-11-22T10:57:10.858055Z","shell.execute_reply.started":"2023-11-22T10:57:09.760189Z","shell.execute_reply":"2023-11-22T10:57:10.85688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(join(PATH,'stage_2_train.csv'))\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:57:10.861263Z","iopub.execute_input":"2023-11-22T10:57:10.861603Z","iopub.status.idle":"2023-11-22T10:57:15.812293Z","shell.execute_reply.started":"2023-11-22T10:57:10.861572Z","shell.execute_reply":"2023-11-22T10:57:15.811312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['Sub_type'] = train['ID'].str.split(\"_\", n = 2, expand = True)[2]\ntrain['PatientID'] = train['ID'].str.split(\"_\", n = 2, expand = True)[1]\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:57:15.813529Z","iopub.execute_input":"2023-11-22T10:57:15.813806Z","iopub.status.idle":"2023-11-22T10:57:47.408374Z","shell.execute_reply.started":"2023-11-22T10:57:15.813781Z","shell.execute_reply":"2023-11-22T10:57:47.40723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_of_training_patients=len(os.listdir(\"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train\"))\nnum_of_testing_patients=len(os.listdir(\"/kaggle/input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_test\"))\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:57:47.409857Z","iopub.execute_input":"2023-11-22T10:57:47.410777Z","iopub.status.idle":"2023-11-22T10:58:09.173312Z","shell.execute_reply.started":"2023-11-22T10:57:47.410724Z","shell.execute_reply":"2023-11-22T10:58:09.172388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ploting amount of training and testing data\nlabels = 'Training', 'Testing'\nsizes = [num_of_training_patients, num_of_testing_patients]\nexplode = (0, 0.1)\n\nfig, ax = plt.subplots(figsize=(6, 6))\nax.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%', shadow=True, startangle=90)\nax.axis('equal')\nax.set_title('Training and Testing Data')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:09.176212Z","iopub.execute_input":"2023-11-22T10:58:09.176565Z","iopub.status.idle":"2023-11-22T10:58:09.44705Z","shell.execute_reply.started":"2023-11-22T10:58:09.176536Z","shell.execute_reply":"2023-11-22T10:58:09.44557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Comparing 0 labels to 1 labels\nprint(train.Label.value_counts())\nsns.countplot(x='Label', data=train)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:09.449356Z","iopub.execute_input":"2023-11-22T10:58:09.4502Z","iopub.status.idle":"2023-11-22T10:58:10.17247Z","shell.execute_reply.started":"2023-11-22T10:58:09.450128Z","shell.execute_reply":"2023-11-22T10:58:10.171665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Number of each subtype labeled as 1\nsubtype_counts = train.groupby(\"Sub_type\").Label.value_counts().unstack()\nsubtype_counts = subtype_counts.loc[:, 1]\nsubtype_counts\n","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:10.173647Z","iopub.execute_input":"2023-11-22T10:58:10.174185Z","iopub.status.idle":"2023-11-22T10:58:11.180087Z","shell.execute_reply.started":"2023-11-22T10:58:10.174145Z","shell.execute_reply":"2023-11-22T10:58:11.178979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig=plt.figure(figsize=(20, 8))\n\nsns.countplot(x=\"Sub_type\", hue=\"Label\", data=train)\n\nplt.title(\"Total Images by Subtype\")","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:11.184287Z","iopub.execute_input":"2023-11-22T10:58:11.18465Z","iopub.status.idle":"2023-11-22T10:58:19.632842Z","shell.execute_reply.started":"2023-11-22T10:58:11.184618Z","shell.execute_reply":"2023-11-22T10:58:19.63173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:**\n* The samples labeled with 0s are too much when compared to the samples labeled with 1s for each subtype.","metadata":{}},{"cell_type":"code","source":"labels =  'epidural','intraparenchymal','intraventricular','subarachnoid','subdural'\nsizes = [subtype_counts[1],subtype_counts[2],subtype_counts[3],subtype_counts[4],subtype_counts[5]]\nexplode = (0, 0.1)\n\nfig, ax = plt.subplots(figsize=(6, 6))\nax.pie(sizes, labels=labels, autopct='%1.1f%%', shadow=False, startangle=90)\nax.axis('equal')\nax.set_title('Subtypes')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:19.634045Z","iopub.execute_input":"2023-11-22T10:58:19.634353Z","iopub.status.idle":"2023-11-22T10:58:19.819481Z","shell.execute_reply.started":"2023-11-22T10:58:19.634326Z","shell.execute_reply":"2023-11-22T10:58:19.817966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Notes:**\n*  We can see here that the data is not balanced, some subtypes have few examples, and that will make it hard to train the model to detectthose subtypes(epidural for example).\n*  Data augmentation techniques will be required to perform the IH detection. Or random sampling can be used too, in such a way that the number of positive patients are equal to the number of negative patients. And as the data is so big I suggest to use subset of it, and the choosen subset have to balanced. ","metadata":{}},{"cell_type":"code","source":"traindf=train.copy()\ntraindf[['ID', 'Image', 'Diagnosis']] = traindf['ID'].str.split('_', expand=True)\ntraindf = traindf[['Image', 'Diagnosis', 'Label']]\ntraindf.drop_duplicates(inplace=True)\ntraindf = traindf.pivot(index='Image', columns='Diagnosis', values='Label').reset_index()\ntraindf['Image'] = 'ID_' + traindf['Image']\ntraindf.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:19.821611Z","iopub.execute_input":"2023-11-22T10:58:19.823049Z","iopub.status.idle":"2023-11-22T10:58:51.74354Z","shell.execute_reply.started":"2023-11-22T10:58:19.82299Z","shell.execute_reply":"2023-11-22T10:58:51.742456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cases with more than one IH subtype detected in the training dataset\nx=[]\nfor n in range(6):\n    many = traindf[traindf[['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural']].sum(1) == n].copy()\n    x.append(len(many))\n    print('Number of hemorrhages: {}, amount of such images: {}, fraction: {:.3f}%'.format(n, len(many), 100 * len(many) / len(traindf)))\n    print(x[n])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:51.744751Z","iopub.execute_input":"2023-11-22T10:58:51.74562Z","iopub.status.idle":"2023-11-22T10:58:52.87607Z","shell.execute_reply.started":"2023-11-22T10:58:51.745592Z","shell.execute_reply":"2023-11-22T10:58:52.875356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=['0','1','2','3','4','5']\nfig, ax = plt.subplots()\nsns.barplot(x=list(y[1:]), y=list(x[1:]), ax=ax)\nax.set_title(\"the number of images for each class\")\nax.set_xlabel(\"class\")","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:52.877241Z","iopub.execute_input":"2023-11-22T10:58:52.877516Z","iopub.status.idle":"2023-11-22T10:58:53.195177Z","shell.execute_reply.started":"2023-11-22T10:58:52.877491Z","shell.execute_reply":"2023-11-22T10:58:53.194063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom skimage.io import imread_collection\nimport skimage.io\nimport skimage.color\nimport skimage.transform\nfrom platform import python_version","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:53.196569Z","iopub.execute_input":"2023-11-22T10:58:53.19693Z","iopub.status.idle":"2023-11-22T10:58:53.721915Z","shell.execute_reply.started":"2023-11-22T10:58:53.1969Z","shell.execute_reply":"2023-11-22T10:58:53.720291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note:**\nIn most of the cases where IH is detected we have one subtype hemorhage detected. ","metadata":{}},{"cell_type":"code","source":"# extract filenames from the folder of images\nfilenames = []\nfor root, dirs, files in os.walk('../input/rsna-hemorrhage-jpg/train_jpg/train_jpg'):\n    for file in files:\n        if file.endswith('.jpg'):\n            filenames.append(file)\n            \n# should be the same as the images imported\nlen(filenames)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T10:58:53.724361Z","iopub.execute_input":"2023-11-22T10:58:53.725365Z","iopub.status.idle":"2023-11-22T11:01:01.98001Z","shell.execute_reply.started":"2023-11-22T10:58:53.725334Z","shell.execute_reply":"2023-11-22T11:01:01.978922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_dir = '../input/rsna-hemorrhage-jpg/train_jpg/train_jpg/*.jpg'\n\n# Create a collection with the available images\nimages = imread_collection(col_dir)\n\nlen(images)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:01.981811Z","iopub.execute_input":"2023-11-22T11:01:01.982243Z","iopub.status.idle":"2023-11-22T11:01:05.233233Z","shell.execute_reply.started":"2023-11-22T11:01:01.982203Z","shell.execute_reply":"2023-11-22T11:01:05.232112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the first image\nplt.figure()\nplt.imshow(images[0])\nplt.colorbar()\nplt.grid(False)\nplt.show()\n\nprint(images[0])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:05.234722Z","iopub.execute_input":"2023-11-22T11:01:05.235081Z","iopub.status.idle":"2023-11-22T11:01:05.676824Z","shell.execute_reply.started":"2023-11-22T11:01:05.235053Z","shell.execute_reply":"2023-11-22T11:01:05.675709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(images[0].shape)\nprint(images[1].shape)\nprint(images[2].shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:05.678123Z","iopub.execute_input":"2023-11-22T11:01:05.678532Z","iopub.status.idle":"2023-11-22T11:01:05.691659Z","shell.execute_reply.started":"2023-11-22T11:01:05.678497Z","shell.execute_reply":"2023-11-22T11:01:05.690588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Select only the first 5000 images\nimages_trn = images[:2000]\nprint(len(images_trn))\nimages_val = images[20000:22000]\nprint(len(images_val))\nimages_tst = images[25000:30000]\nprint(len(images_tst))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:05.692653Z","iopub.execute_input":"2023-11-22T11:01:05.693482Z","iopub.status.idle":"2023-11-22T11:01:05.701976Z","shell.execute_reply.started":"2023-11-22T11:01:05.69345Z","shell.execute_reply":"2023-11-22T11:01:05.701283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_arr_trn = skimage.io.collection.concatenate_images(images_trn)\nimages_arr_val = skimage.io.collection.concatenate_images(images_val)\nimages_arr_tst = skimage.io.collection.concatenate_images(images_tst)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:05.703302Z","iopub.execute_input":"2023-11-22T11:01:05.703621Z","iopub.status.idle":"2023-11-22T11:01:44.708612Z","shell.execute_reply.started":"2023-11-22T11:01:05.703592Z","shell.execute_reply":"2023-11-22T11:01:44.707562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_feather('../input/rsna-hemorrhage-jpg/meta/meta/labels.fth')\n\n#manipulate the filenames list, stripping the .jpg at the end\nidstosearch = [item.rstrip(\".jpg\") for item in filenames]\n\n#now search the \"ID\" column for ids that correspond to our filenames\n#made the reduced dataframe \"labels2\" for now\nlabels2 = labels[labels['ID'].isin(idstosearch)]\nlabels2.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:44.710058Z","iopub.execute_input":"2023-11-22T11:01:44.710438Z","iopub.status.idle":"2023-11-22T11:01:45.816369Z","shell.execute_reply.started":"2023-11-22T11:01:44.710404Z","shell.execute_reply":"2023-11-22T11:01:45.81523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = labels2.iloc[:, 1]\nprint(labels)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:45.818105Z","iopub.execute_input":"2023-11-22T11:01:45.818931Z","iopub.status.idle":"2023-11-22T11:01:45.858715Z","shell.execute_reply.started":"2023-11-22T11:01:45.818887Z","shell.execute_reply":"2023-11-22T11:01:45.857456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_trn = labels[:2000]\nprint(len(labels_trn))\nlabels_val = labels[20000:22000]\nprint(len(labels_val))\nlabels_tst = labels[25000:30000]\nprint(len(labels_tst))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:45.860392Z","iopub.execute_input":"2023-11-22T11:01:45.861598Z","iopub.status.idle":"2023-11-22T11:01:45.872864Z","shell.execute_reply.started":"2023-11-22T11:01:45.861563Z","shell.execute_reply":"2023-11-22T11:01:45.871644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(labels_trn))\nprint(labels_trn.sum())","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:45.876548Z","iopub.execute_input":"2023-11-22T11:01:45.87746Z","iopub.status.idle":"2023-11-22T11:01:45.885875Z","shell.execute_reply.started":"2023-11-22T11:01:45.877416Z","shell.execute_reply":"2023-11-22T11:01:45.884674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform labels into array\nlabels_trn = pd.Series.to_numpy(labels_trn)\nprint(len(labels_trn))\nlabels_val = pd.Series.to_numpy(labels_val)\nprint(len(labels_val))\nlabels_tst = pd.Series.to_numpy(labels_tst)\nprint(len(labels_tst))","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:45.887314Z","iopub.execute_input":"2023-11-22T11:01:45.887687Z","iopub.status.idle":"2023-11-22T11:01:45.899688Z","shell.execute_reply.started":"2023-11-22T11:01:45.887607Z","shell.execute_reply":"2023-11-22T11:01:45.898449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.applications import resnet50\n\nmodel = resnet50.ResNet50(weights=\"imagenet\")","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:45.90084Z","iopub.execute_input":"2023-11-22T11:01:45.901166Z","iopub.status.idle":"2023-11-22T11:01:51.264774Z","shell.execute_reply.started":"2023-11-22T11:01:45.90112Z","shell.execute_reply":"2023-11-22T11:01:51.263849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Resize all images \n\nimages_final = []\n\nfor i in range(len(images_arr_trn)):\n  image_rescaled = skimage.transform.resize(images_arr_trn[i], (224, 224, 3))\n  images_final.append(image_rescaled)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:01:51.272311Z","iopub.execute_input":"2023-11-22T11:01:51.273147Z","iopub.status.idle":"2023-11-22T11:02:20.153859Z","shell.execute_reply.started":"2023-11-22T11:01:51.2731Z","shell.execute_reply":"2023-11-22T11:02:20.152654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(images_final)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:02:20.155299Z","iopub.execute_input":"2023-11-22T11:02:20.155662Z","iopub.status.idle":"2023-11-22T11:02:20.162293Z","shell.execute_reply.started":"2023-11-22T11:02:20.155629Z","shell.execute_reply":"2023-11-22T11:02:20.16133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_final = skimage.io.collection.concatenate_images(images_final)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:02:20.163745Z","iopub.execute_input":"2023-11-22T11:02:20.164435Z","iopub.status.idle":"2023-11-22T11:02:22.01582Z","shell.execute_reply.started":"2023-11-22T11:02:20.164403Z","shell.execute_reply":"2023-11-22T11:02:22.014855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:02:22.019244Z","iopub.execute_input":"2023-11-22T11:02:22.019612Z","iopub.status.idle":"2023-11-22T11:02:22.044793Z","shell.execute_reply.started":"2023-11-22T11:02:22.019579Z","shell.execute_reply":"2023-11-22T11:02:22.043791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train model\nmodel.fit(images_final, labels_trn, epochs=8)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T11:02:22.046217Z","iopub.execute_input":"2023-11-22T11:02:22.046528Z","iopub.status.idle":"2023-11-22T12:37:22.005006Z","shell.execute_reply.started":"2023-11-22T11:02:22.046499Z","shell.execute_reply":"2023-11-22T12:37:22.00163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_val = []\n\nfor i in range(len(images_arr_val)):\n  image_rescaled = skimage.transform.resize(images_arr_val[i], (224, 224, 3))\n  images_val.append(image_rescaled)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T12:37:22.011002Z","iopub.execute_input":"2023-11-22T12:37:22.013401Z","iopub.status.idle":"2023-11-22T12:37:52.261195Z","shell.execute_reply.started":"2023-11-22T12:37:22.013279Z","shell.execute_reply":"2023-11-22T12:37:52.260001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images_val = skimage.io.collection.concatenate_images(images_val)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T12:37:52.262982Z","iopub.execute_input":"2023-11-22T12:37:52.26344Z","iopub.status.idle":"2023-11-22T12:37:55.226562Z","shell.execute_reply.started":"2023-11-22T12:37:52.263389Z","shell.execute_reply":"2023-11-22T12:37:55.22516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Validate model\ntest_loss, test_acc = model.evaluate(images_val, labels_val, verbose=2)\n\nprint('\\nTest accuracy:', test_acc)","metadata":{"execution":{"iopub.status.busy":"2023-11-22T12:37:55.228868Z","iopub.execute_input":"2023-11-22T12:37:55.229274Z","iopub.status.idle":"2023-11-22T12:41:22.136665Z","shell.execute_reply.started":"2023-11-22T12:37:55.229241Z","shell.execute_reply":"2023-11-22T12:41:22.132346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}