{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6968978,"sourceType":"datasetVersion","datasetId":3971809},{"sourceId":193,"sourceType":"modelInstanceVersion","modelInstanceId":137},{"sourceId":3107,"sourceType":"modelInstanceVersion","modelInstanceId":2320},{"sourceId":3213,"sourceType":"modelInstanceVersion","modelInstanceId":2390}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CNN Inception V3 Fine Adjustment\n\n#### Hi friends, this is my model for challenge detect cancer ovarian. Share my notebook with the intention of helping give more ideas to the community and maybe they will help me too. 🙏 😃","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nimport keras\n\nfrom keras.models import Sequential\nfrom keras.metrics import Recall\nfrom keras.layers import Dense, Dropout, BatchNormalization, Activation\nfrom keras.optimizers import Adam\nfrom keras.utils import to_categorical\nfrom keras.applications.resnet_v2 import preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\n\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\n\nfrom sklearn.metrics import confusion_matrix, classification_report\n\nimport os\nimport math\nimport seaborn as sns\nimport PIL\nimport cv2 as cv\nimport numpy as np\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt\n\nROOT_PATH = '/kaggle/input/UBC-OCEAN/'\nIMAGES_PATH = '/kaggle/input/UBC-OCEAN/train_images/'\nTHUMBNAILS_PATH = '/kaggle/input/UBC-OCEAN/train_thumbnails/'\nTILES_PATH = '/kaggle/input/ubc-ocean-thumbnails-crop/tile_images/'\n\nTEST_IMAGES_PATH = '/kaggle/input/UBC-OCEAN/test_images/'\nTEST_THUMBNAILS_PATH = '/kaggle/input/UBC-OCEAN/test_thumbnails/'\n\nWORKING_PATH = '/kaggle/working/'\n\nMODEL = '/kaggle/input/efficientnet-v2/tensorflow2/imagenet1k-b0-classification/2'\nWEIGHTS = '/kaggle/working/efficientnet_x20.h5'\n\nSAMPLE = 20\nTHRESHOLD = 2\nZOOM = 20\nKILOBYTES = 1024\nSIZE = (224, 224)\nLABELS = ['CC', 'EC', 'HGSC', 'LGSC', 'MC']\n\nPIL.Image.MAX_IMAGE_PIXELS = 933120000000","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:20:21.227858Z","iopub.execute_input":"2023-11-14T21:20:21.228189Z","iopub.status.idle":"2023-11-14T21:20:21.238031Z","shell.execute_reply.started":"2023-11-14T21:20:21.228164Z","shell.execute_reply":"2023-11-14T21:20:21.237117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Segment image ✂️ 🔲\n#### This images are cropped in 50 tiles same bounds squares. The choice top 3 tiles images by entropy function.","metadata":{}},{"cell_type":"code","source":"def get_entropy(img):\n    r = cv.split(img)[0]\n\n    hist = cv.calcHist([r], [0], None, [256], [0, 256])\n    hist /= hist.sum()\n    \n    return -np.sum(hist * np.log2(hist + np.finfo(float).eps))\n\ndef is_within(coord, size_square, size):\n    (y, x) = coord\n    (h, w) = size\n    (square_h, square_w) = size_square\n    return y >= 0 and x >= 0 and y + square_h <= h and x + square_w <= w\n\ndef point_adjustment(points, size_square, size):\n    (h, w) = size\n    (square_h, square_w) = size_square\n\n    list_is_within = np.array([is_within(point, size_square, size) for point in points])\n    index_without = np.argwhere(list_is_within == False)\n\n    for [index] in index_without:\n        (y, x) = points[index]\n\n        left_outside = x * -1\n        right_outside = (x + square_w) - w\n        top_outside = y * -1\n        buttom_outside = (y + square_h) - h\n\n        if left_outside > 0:\n            x = 0\n        \n        if right_outside > 0:\n            x -= right_outside\n\n        if top_outside > 0:\n            y = 0\n        \n        if buttom_outside > 0:\n            y -= buttom_outside\n\n        points[index] = (y, x)\n\n    return points","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:18:10.199494Z","iopub.execute_input":"2023-11-14T21:18:10.200019Z","iopub.status.idle":"2023-11-14T21:18:10.211365Z","shell.execute_reply.started":"2023-11-14T21:18:10.199991Z","shell.execute_reply":"2023-11-14T21:18:10.210365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"KILOBYTES = 1024\n\ndef get_is_tma(dim, image_path=''):\n    (h, w) = dim\n    area = h * w\n    return area / (KILOBYTES ** 3) < 0.1 and not image_path.__contains__('thumbnail')\n\ndef get_zoom(dim, zoom, image_path=''):\n    (h, w) = dim\n\n    is_tma = get_is_tma(dim, image_path)\n\n    min_side = min(h, w) // (1 if is_tma else zoom)\n\n    amount_h = h // min_side\n    amount_w = w // min_side\n\n    num_squares = amount_h * amount_w\n\n    return num_squares, min_side, is_tma","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:18:10.212384Z","iopub.execute_input":"2023-11-14T21:18:10.212746Z","iopub.status.idle":"2023-11-14T21:18:10.226518Z","shell.execute_reply.started":"2023-11-14T21:18:10.21271Z","shell.execute_reply":"2023-11-14T21:18:10.225624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_hotmap(img, dim, num_tiles):\n    (h, w) = img.shape[:2]\n    area = h * w\n    \n    (square_h, square_w) = dim\n    ratio_h, ratio_w = square_h // 2, square_w // 2\n\n    img_hsv = cv.cvtColor(img, cv.COLOR_RGB2HSV)\n    lower = np.array([140,50,110])\n    upper = np.array([160,100,140])\n    mask = cv.inRange(img_hsv, lower, upper)\n\n    if len(mask[mask==255]) < area * 0.01:\n        lower = np.array([100,0,140])\n        upper = np.array([175,250,215])\n        mask = cv.inRange(img_hsv, lower, upper)\n    \n    kernel = np.ones((square_h, square_w), np.float32) / (square_h * square_w)\n    mask = cv.filter2D(mask.astype(np.uint8), -1, kernel, borderType=cv.BORDER_CONSTANT)\n\n    hotmap = []\n\n    for _ in range(num_tiles):\n        coord = np.unravel_index(np.argmax(mask), mask.shape)\n        hotmap.append(list(coord))\n        cv.rectangle(mask, (coord[1] - ratio_h, coord[0] - ratio_w), (coord[1] + ratio_h, coord[0] + ratio_w), 0, -1)\n\n    return hotmap, mask","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:19:50.05081Z","iopub.execute_input":"2023-11-14T21:19:50.051174Z","iopub.status.idle":"2023-11-14T21:19:50.061063Z","shell.execute_reply.started":"2023-11-14T21:19:50.051145Z","shell.execute_reply":"2023-11-14T21:19:50.060042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_segment(image_path, num_tiles=0, size=(300, 300), zoom=1, threshold=0, is_show=False):\n    image = load_img(image_path, color_mode='rgb')\n    image = np.array(img_to_array(image), dtype='uint8')\n\n    image_processed = []\n    entropies = []\n\n    (h, w) = image.shape[:2]\n\n    num_squares, min_side, _ = get_zoom((h, w), zoom, image_path)\n\n    if num_tiles == 0 or num_tiles > num_squares:\n        num_tiles = num_squares\n\n    (square_h, square_w) = (min_side, min_side)\n    ratio_h, ratio_w = square_h // 2, square_w // 2\n\n    hotmap, mask = get_hotmap(image, (square_h, square_w), num_tiles)\n\n    if is_show:\n        plt.figure()\n        plt.imshow(mask)\n    \n    hotmap = np.array(hotmap)\n    hotmap[:, 0] -= ratio_h\n    hotmap[:, 1] -= ratio_w\n\n    hotmap = point_adjustment(hotmap, (square_h, square_w), (h, w))\n\n    for coord in hotmap:\n        [y, x] = coord\n        cx = x + ratio_w\n        cy = y + ratio_h\n\n        if is_within((y, x), (square_h, square_w), (h, w)):\n            img_tile = image[y:y + square_h, x:x + square_w]\n            entropy = get_entropy(img_tile)\n            \n            if entropy >= threshold:\n                entropies.append(entropy)\n                image_processed.append(img_tile)\n                if is_show:\n                    cv.circle(image, (cx, cy), 20, (0, 255, 255), -1)\n                    cv.rectangle(image, (x, y), (x + square_w, y + square_h), (0, 255, 255), 5)\n            else:\n                cv.circle(image, (cx, cy), 20, (255, 255, 0), -1)\n                cv.rectangle(image, (x, y), (x + square_w, y + square_h), (255, 255, 0), 5, 2)\n                \n            if len(entropies) >= num_tiles: break\n                \n        else:\n            if is_show:\n                cv.circle(image, (cx, cy), 20, (255, 0, 0), -1)\n\n    if is_show:\n        plt.figure()\n        plt.imshow(image)\n        plt.show()\n\n    entropies = np.array(entropies)\n    bests = np.argsort(-entropies)\n    entropies = entropies[bests]\n\n    bests_ = []\n   \n    for best in bests:\n        img = image_processed[best]\n        img = cv.resize(img, size)\n        bests_.append(img)\n    \n    return bests_","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:19:54.046232Z","iopub.execute_input":"2023-11-14T21:19:54.04661Z","iopub.status.idle":"2023-11-14T21:19:54.061818Z","shell.execute_reply.started":"2023-11-14T21:19:54.046569Z","shell.execute_reply":"2023-11-14T21:19:54.060943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Top 3 images, pos in 0 is image with more entropy. In the view applied filter of Inception V3.","metadata":{}},{"cell_type":"code","source":"pos = 47\n\ndf_sample = pd.read_csv(f'{ROOT_PATH}train.csv')\nresult = get_segment(f\"{THUMBNAILS_PATH}{df_sample['image_id'][pos]}_thumbnail.png\", num_tiles=SAMPLE, zoom=ZOOM, threshold=THRESHOLD, is_show=True)\n\n_, ax = plt.subplots(3, 3, figsize=(3, 3))\nplt.subplots_adjust(wspace=0, hspace=0)\n\nfor i, img in enumerate(result[:9]):\n    ax[i // 3, i % 3].imshow(preprocess_input(img))\n    ax[i // 3, i % 3].axis('off')\n    \n_.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:20:30.285244Z","iopub.execute_input":"2023-11-14T21:20:30.285609Z","iopub.status.idle":"2023-11-14T21:20:32.101146Z","shell.execute_reply.started":"2023-11-14T21:20:30.285567Z","shell.execute_reply":"2023-11-14T21:20:32.099406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define model CNN 📠 \n#### We take model pretrained Inception V3. Fine adjustment is made adding layer Flatten, Dense x512 and layer out x5 (classes).","metadata":{}},{"cell_type":"code","source":"'''\nmodel_ = tf.keras.applications.DenseNet121(weights='imagenet', include_top=False, input_shape=(224, 224, 3))\n\nfor layer in model_.layers:\n    layer.trainable = False\n    \nmodel = Sequential()\nmodel.add(model_)\nmodel.add(GlobalAveragePooling2D())\nmodel.add(Dense(5, activation='softmax'))\n'''","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:18:11.687547Z","iopub.status.idle":"2023-11-14T21:18:11.687911Z","shell.execute_reply.started":"2023-11-14T21:18:11.687752Z","shell.execute_reply":"2023-11-14T21:18:11.687768Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_ = hub.KerasLayer(MODEL, input_shape=(*SIZE, 3))\nmodel_.trainable = False","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:20:40.620814Z","iopub.execute_input":"2023-11-14T21:20:40.621619Z","iopub.status.idle":"2023-11-14T21:20:49.652521Z","shell.execute_reply.started":"2023-11-14T21:20:40.621566Z","shell.execute_reply":"2023-11-14T21:20:49.651505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    model_,\n    Dense(5, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:20:50.874959Z","iopub.execute_input":"2023-11-14T21:20:50.875352Z","iopub.status.idle":"2023-11-14T21:20:51.794756Z","shell.execute_reply.started":"2023-11-14T21:20:50.87532Z","shell.execute_reply":"2023-11-14T21:20:51.793957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:18:11.692098Z","iopub.status.idle":"2023-11-14T21:18:11.692416Z","shell.execute_reply.started":"2023-11-14T21:18:11.692259Z","shell.execute_reply":"2023-11-14T21:18:11.692275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load dataset\n#### We loaded all images from folder images thumbnails and complements folder images tma.","metadata":{}},{"cell_type":"code","source":"%%time\n\ndf_sample = pd.read_csv(f'{ROOT_PATH}train.csv')\n\nlist_dir = os.listdir(TILES_PATH)\n\nX_train = []\ny_train = []\n\nfor dir_ in list_dir:\n    result = df_sample.loc[df_sample['image_id'] == int(dir_)]\n    label = result['label'].values[0]\n    for file in os.listdir(f'{TILES_PATH}{dir_}'):\n        image = load_img(f'{TILES_PATH}{dir_}/{file}', color_mode='rgb')\n        image = cv.resize(np.array(img_to_array(image), dtype='uint8'), SIZE)\n        X_train.append(preprocess_input(image))\n        y_train.append(LABELS.index(label))\n\ny_train = keras.utils.to_categorical(y_train)\n    \nX_train = np.array(X_train)\ny_train = np.array(y_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:20:58.870044Z","iopub.execute_input":"2023-11-14T21:20:58.870865Z","iopub.status.idle":"2023-11-14T21:22:23.598612Z","shell.execute_reply.started":"2023-11-14T21:20:58.870827Z","shell.execute_reply":"2023-11-14T21:22:23.597626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = np.arange(len(X_train))\nnp.random.shuffle(indices)\n\nX_train = X_train[indices]\ny_train = y_train[indices]","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:23:49.270174Z","iopub.execute_input":"2023-11-14T21:23:49.27054Z","iopub.status.idle":"2023-11-14T21:23:51.263733Z","shell.execute_reply.started":"2023-11-14T21:23:49.270511Z","shell.execute_reply":"2023-11-14T21:23:51.262937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nplt.imshow(X_train[85])\nplt.show()\n\nprint(X_train.shape)\nprint(y_train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:23:54.24477Z","iopub.execute_input":"2023-11-14T21:23:54.245137Z","iopub.status.idle":"2023-11-14T21:23:54.613884Z","shell.execute_reply.started":"2023-11-14T21:23:54.245109Z","shell.execute_reply":"2023-11-14T21:23:54.612947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Applied technique augmentation images data with rotation, horizontal flip and others.","metadata":{}},{"cell_type":"code","source":"def augment_dataset(X_train, X_test, y_train, y_test, shuffle=False, channels=1):\n    if shuffle:\n        indices = np.arange(len(X_train))\n        np.random.shuffle(indices)\n\n        X_train = X_train[indices]\n        y_train = y_train[indices]\n\n    train_datagen = ImageDataGenerator(\n        width_shift_range=0.2,\n        height_shift_range=0.2,\n        horizontal_flip=True\n    )\n\n    test_datagen = ImageDataGenerator()\n\n    train_generator = train_datagen.flow(X_train, y_train, batch_size=64)\n    test_generator = test_datagen.flow(X_test, y_test, batch_size=64)\n    \n    return train_generator, test_generator","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:24:04.163007Z","iopub.execute_input":"2023-11-14T21:24:04.163722Z","iopub.status.idle":"2023-11-14T21:24:04.170016Z","shell.execute_reply.started":"2023-11-14T21:24:04.163691Z","shell.execute_reply":"2023-11-14T21:24:04.16904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"split_valid = int(len(X_train) * 0.65)\nsplit_train = int(split_valid * 0.8)\n\nX_train, X_test, X_valid = X_train[:split_train], X_train[split_train:split_valid], X_train[split_valid:]\ny_train, y_test, y_valid = y_train[:split_train], y_train[split_train:split_valid], y_train[split_valid:]\n\ntrain_generator, test_generator = augment_dataset(X_train, X_test, y_train, y_test, shuffle=True, channels=3)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:26:45.925999Z","iopub.execute_input":"2023-11-14T21:26:45.926355Z","iopub.status.idle":"2023-11-14T21:26:47.18939Z","shell.execute_reply.started":"2023-11-14T21:26:45.926326Z","shell.execute_reply":"2023-11-14T21:26:47.188604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model 🏃\n#### Added checkpoints for prevent overfitting.","metadata":{}},{"cell_type":"code","source":"early_stopping = EarlyStopping(monitor='val_recall', patience=5, verbose=1)\nmodel_checkpoint = ModelCheckpoint(WEIGHTS, save_best_only=True, save_weights_only=True, monitor='val_recall', mode='max', verbose=1)\n\nmodel.compile(optimizer=Adam(learning_rate=0.0005), loss='categorical_crossentropy', metrics=[Recall(name='recall'), 'accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:26:52.219958Z","iopub.execute_input":"2023-11-14T21:26:52.220769Z","iopub.status.idle":"2023-11-14T21:26:52.246124Z","shell.execute_reply.started":"2023-11-14T21:26:52.220717Z","shell.execute_reply":"2023-11-14T21:26:52.245271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_generator, epochs=30, batch_size=128, validation_data=train_generator, callbacks=[early_stopping, model_checkpoint])","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:26:56.594017Z","iopub.execute_input":"2023-11-14T21:26:56.594487Z","iopub.status.idle":"2023-11-14T22:42:46.012676Z","shell.execute_reply.started":"2023-11-14T21:26:56.59445Z","shell.execute_reply":"2023-11-14T22:42:46.011829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights(WEIGHTS)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:46:09.270425Z","iopub.execute_input":"2023-11-14T22:46:09.271297Z","iopub.status.idle":"2023-11-14T22:46:09.606317Z","shell.execute_reply.started":"2023-11-14T22:46:09.271264Z","shell.execute_reply":"2023-11-14T22:46:09.605289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loss, test_recall, test_accuracy = model.evaluate(X_valid, y_valid)\nprint(f\"Pérdida en el conjunto de prueba: { test_loss }\")\nprint(f\"Recall en el conjunto de prueba: { test_recall }\")\nprint(f\"Precisión en el conjunto de prueba: { test_accuracy }\")","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:46:55.528349Z","iopub.execute_input":"2023-11-14T22:46:55.529211Z","iopub.status.idle":"2023-11-14T22:47:00.996194Z","shell.execute_reply.started":"2023-11-14T22:46:55.529177Z","shell.execute_reply":"2023-11-14T22:47:00.995367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### This model prioritize metric recall.","metadata":{}},{"cell_type":"code","source":"training_accuracy = history.history['recall']\nvalidation_accuracy = history.history['val_recall']\n\ntraining_loss = history.history['loss']\nvalidation_loss = history.history['val_loss']\n\nplt.figure(figsize=(12, 4))\n\nplt.subplot(1, 2, 1)\nplt.plot(range(1, len(training_accuracy) + 1), training_accuracy, label='Training Recall')\nplt.plot(range(1, len(validation_accuracy) + 1), validation_accuracy, label='Validation Recall')\nplt.xlabel('Epoch')\nplt.ylabel('Recall')\nplt.title('Recall vs. Epoch')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.plot(range(1, len(training_loss) + 1), training_loss, label='Training Loss')\nplt.plot(range(1, len(validation_loss) + 1), validation_loss, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Loss vs. Epoch')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:47:07.685129Z","iopub.execute_input":"2023-11-14T22:47:07.685478Z","iopub.status.idle":"2023-11-14T22:47:08.384734Z","shell.execute_reply.started":"2023-11-14T22:47:07.685448Z","shell.execute_reply":"2023-11-14T22:47:08.383719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:47:25.737734Z","iopub.execute_input":"2023-11-14T22:47:25.738441Z","iopub.status.idle":"2023-11-14T22:47:31.326436Z","shell.execute_reply.started":"2023-11-14T22:47:25.738408Z","shell.execute_reply":"2023-11-14T22:47:31.325707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### It's noted similares rate prediction for all classes, despite trained with dataset unbalanced.","metadata":{}},{"cell_type":"code","source":"y_pred_argmax = [np.argmax(prob) for prob in y_pred]\ny_pred_binary = [[1 if i == n else 0 for i in range(len(LABELS))] for n in y_pred_argmax]\n\nreport = classification_report(y_valid, y_pred_binary, target_names=LABELS, zero_division=0)\n\nprint(report)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:47:37.855883Z","iopub.execute_input":"2023-11-14T22:47:37.856369Z","iopub.status.idle":"2023-11-14T22:47:37.901079Z","shell.execute_reply.started":"2023-11-14T22:47:37.856305Z","shell.execute_reply":"2023-11-14T22:47:37.900145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_matrix = confusion_matrix([np.argmax(y) for y in y_valid], [np.argmax(y) for y in y_pred])\n\nplt.figure(figsize=(8, 6))\n\nsns.heatmap(conf_matrix, annot=True, fmt='d', cmap='Blues', cbar=True, xticklabels=LABELS, yticklabels=LABELS)\n\nplt.xlabel('Predicciones')\nplt.ylabel('Etiquetas Verdaderas')\nplt.title('Matriz de Confusión')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:47:58.250936Z","iopub.execute_input":"2023-11-14T22:47:58.251815Z","iopub.status.idle":"2023-11-14T22:47:58.635152Z","shell.execute_reply.started":"2023-11-14T22:47:58.251783Z","shell.execute_reply":"2023-11-14T22:47:58.634193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate file submission","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(f'{ROOT_PATH}test.csv')\ndf['label'] = ['HGSC'] * len(df)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:48:12.551176Z","iopub.execute_input":"2023-11-14T22:48:12.551547Z","iopub.status.idle":"2023-11-14T22:48:12.562462Z","shell.execute_reply.started":"2023-11-14T22:48:12.551517Z","shell.execute_reply":"2023-11-14T22:48:12.561313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport gc\n\ndefault_dir = os.listdir(TEST_THUMBNAILS_PATH)\n\npredict = []\nfor _, row in df.iterrows():\n    row = dict(row)\n    \n    image_name = f'{row[\"image_id\"]}_thumbnail.png'\n    if image_name in default_dir:\n        tiles = get_segment(f'{TEST_THUMBNAILS_PATH}{image_name}', num_tiles=SAMPLE, zoom=ZOOM, size=SIZE, threshold=THRESHOLD)\n    else:\n        tiles = get_segment(f'{TEST_IMAGES_PATH}{row[\"image_id\"]}.png', num_tiles=SAMPLE, zoom=ZOOM, size=SIZE, threshold=THRESHOLD)\n    \n    if not len(tiles):\n        predict.append(row)\n        continue\n        \n    X_test_ = [preprocess_input(tile).reshape(1, *SIZE, 3) for tile in tiles]\n    y_pred_i = [model.predict(x) for x in X_test_]\n    predict_prev = np.sum(y_pred_i, axis=0)\n    \n    row['label'] = LABELS[np.argmax(predict_prev)]\n    predict.append(row)\n    \n    del X_test_\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:48:15.092282Z","iopub.execute_input":"2023-11-14T22:48:15.093128Z","iopub.status.idle":"2023-11-14T22:48:17.658572Z","shell.execute_reply.started":"2023-11-14T22:48:15.093096Z","shell.execute_reply":"2023-11-14T22:48:17.657644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_predict = pd.DataFrame(predict)\ndf_predict[['image_id', 'label']].to_csv(f\"{WORKING_PATH}submission.csv\", index=False)\n\n!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-11-14T22:48:26.317868Z","iopub.execute_input":"2023-11-14T22:48:26.31823Z","iopub.status.idle":"2023-11-14T22:48:27.401112Z","shell.execute_reply.started":"2023-11-14T22:48:26.318202Z","shell.execute_reply":"2023-11-14T22:48:27.39994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!rm -r /kaggle/working/*.h5","metadata":{"execution":{"iopub.status.busy":"2023-11-14T21:18:11.72311Z","iopub.status.idle":"2023-11-14T21:18:11.723447Z","shell.execute_reply.started":"2023-11-14T21:18:11.72328Z","shell.execute_reply":"2023-11-14T21:18:11.723296Z"},"trusted":true},"execution_count":null,"outputs":[]}]}