{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6917177,"sourceType":"datasetVersion","datasetId":3889865},{"sourceId":6984590,"sourceType":"datasetVersion","datasetId":4014175},{"sourceId":7029230,"sourceType":"datasetVersion","datasetId":4027203},{"sourceId":154210123,"sourceType":"kernelVersion"},{"sourceId":154352113,"sourceType":"kernelVersion"}],"dockerImageVersionId":30616,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport shutil\nimport os\nimport numpy as np\nimport glob\nimport random\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import classification_report\nimport tensorflow_datasets as tfds\n\nfrom tensorflow.keras.applications import EfficientNetB5","metadata":{"execution":{"iopub.status.busy":"2023-12-10T07:11:26.642791Z","iopub.execute_input":"2023-12-10T07:11:26.643085Z","iopub.status.idle":"2023-12-10T07:11:40.158608Z","shell.execute_reply.started":"2023-12-10T07:11:26.643058Z","shell.execute_reply":"2023-12-10T07:11:40.157777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '/kaggle/input/ubc-ocean-data0-1/train_dir/'\n\n# dirs = os.listdir('/kaggle/input/ubc-ocean-data/train_dir')\ndirs = os.listdir(train_dir)\n\n\nprint(dirs)\n\n# class_names = ['CC', 'EC', 'HGSC', 'LGSC', 'MC', 'Other']\n\nd2cnt = dict()\nclass_weight = dict()\nlabel2idx = {'CC': 0, 'EC': 1, 'HGSC': 2, 'LGSC': 3, 'MC': 4, 'Other': 5}\n# ['Other', 'LGSC', 'EC', 'CC', 'MC', 'HGSC']\n# label2idx = {'Other': 0, 'LGSC': 1, 'EC': 2, 'CC': 3, 'MC': 4, 'HGSC': 5}\n\nfor d in dirs:\n    d2cnt[d] = len(glob.glob(train_dir + d + '/*/*.png'))\n    print(d, d2cnt[d])\n    \ntot = 0\nfor d in d2cnt:\n    tot += d2cnt[d]\n    \nfor d in d2cnt:\n    class_weight[label2idx[d]] = tot / d2cnt[d]\n    \nprint(class_weight)\n\n\nmask_imgs_list = os.listdir(\"/kaggle/input/ubc-ocean-tiles-w-masks-2048px-scale-0-25/train_images\")\n# print(len(mask_imgs_list))\nmask_set = set(mask_imgs_list)\nprint(len(mask_set))\n# print('train.csv' in mask_set)\n\nall_imgs_list = os.listdir(\"/kaggle/input/tiles-of-cancer-2048px-scale-0-25\")\n# print(len(all_imgs_list))\nall_img_set = set(all_imgs_list)\nprint('train.csv' in all_img_set)\nall_img_set.remove('train.csv')\nprint(len(all_img_set))\n# print(all_img_set)\n\nno_mask_set = all_img_set - mask_set\nprint(len(no_mask_set))\n\n\ntrain_df = pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv')\n\nimg2lb = dict()\nfor i in range(len(train_df)):\n    img2lb[str(train_df.loc[i, 'image_id'])] = train_df.loc[i, 'label']\n    \n\ndef evaluate_single_img(model, d, class_names, img2lb, img_height, img_width, batch_size=32):\n    \n    img_dir = \"/kaggle/input/tiles-of-cancer-2048px-scale-0-25/\" + str(d) + \"/\"\n    \n    ds = tf.keras.utils.image_dataset_from_directory(\n        img_dir,\n        image_size=(img_height, img_width),\n        batch_size=batch_size,\n        shuffle=False,\n        labels=None,\n    )\n    \n    results = model.predict(ds, verbose=0)\n    \n    sm = tf.nn.softmax(\n        results, axis=None, name=None\n    )\n    \n    #####################################\n    # Discard \"Other\" logits\n    sm = sm[..., :5]\n#     sm = sm[..., 1:]\n    \n    #####################################\n\n    results1 = tf.math.reduce_max(sm, axis=0)\n\n    results2 = tf.math.argmax(\n        results1,\n    )\n    \n    prediction = class_names[results2]\n    label = img2lb[str(d)]\n    \n    return label, prediction\n\n##########################################\n# original size\n# img_height, img_width = 512, 512\n\n# efficient net B5\nimg_height, img_width = 456, 456\n##########################################\n\n# data_dir = '/kaggle/input/ubc-ocean-data/train_dir'\nbatch_size = 32\n\ntrain_ds = tf.keras.utils.image_dataset_from_directory(\n    train_dir,\n#     validation_split=0.02,\n#     subset=\"training\",\n#     seed=123,\n    image_size=(img_height, img_width),\n    batch_size=batch_size\n)\n\n# train_ds = train_ds.map(tfds.features.ClassLabel(names=class_names))\n\n# val_ds = tf.keras.utils.image_dataset_from_directory(\n#     data_dir,\n#     validation_split=0.02,\n#     subset=\"validation\",\n#     seed=123,\n#     image_size=(img_height, img_width),\n#     batch_size=batch_size\n# )\n\ntrain_size = train_ds.cardinality().numpy()\n# val_size = val_ds.cardinality().numpy()\nprint(train_size)\n# print(val_size)\n\n\nclass_names = train_ds.class_names\nprint(class_names)\n\nnum_classes = len(class_names)\n\n\n################################################\n# Simple CNN\n#################################################\n# model = Sequential([\n#     layers.Rescaling(1./255, input_shape=(img_height, img_width, 3)),\n#     layers.Conv2D(16, 3, padding='same', activation='relu'),\n#     layers.MaxPooling2D(),\n#     layers.Dropout(0.2),\n#     layers.Conv2D(32, 3, padding='same', activation='relu'),\n#     layers.MaxPooling2D(),\n#     layers.Conv2D(64, 3, padding='same', activation='relu'),\n#     layers.MaxPooling2D(),\n#     layers.Dropout(0.2),\n#     layers.Flatten(),\n#     layers.Dense(128, activation='relu'),\n#     layers.Dense(num_classes, name=\"outputs\")\n# ])\n\n# model.compile(\n#     optimizer='adam',\n#     loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n#     metrics=['accuracy']\n# )\n################################################\n# EfficientNetB5\n#################################################\n# inputs = layers.Input(shape=(img_height, img_width, 3))\n# model = EfficientNetB5(include_top=False, input_tensor=inputs, weights=\"imagenet\")\n# model.trainable = False\n\n# x = layers.GlobalAveragePooling2D(name=\"avg_pool\")(model.output)\n# x = layers.BatchNormalization()(x)\n\n# top_dropout_rate = 0.2\n# x = layers.Dropout(top_dropout_rate, name=\"top_dropout\")(x)\n# outputs = layers.Dense(num_classes)(x)\n\n# model = keras.Model(inputs, outputs, name=\"EfficientNet\")\n# optimizer = keras.optimizers.Adam(learning_rate=1e-2)\n# model.compile(\n#     optimizer=optimizer, \n#     loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n#     metrics=[\"accuracy\"]\n# )\n#####################################################\n# Load Old Model\n#####################################################\nmodel = keras.models.load_model(\"/kaggle/input/ubc-ocean-train0-1/my_model_28.keras\")\n\n\n####################\nsteps = 5\n# steps = 1\n####################\n\n#################################################\n# val_list = sorted(list(no_mask_set))[:100]\nval_list = sorted(list(no_mask_set))\n#################################################\n\nprint(\"val_list\", len(val_list))\noutside_epochs = 10\n\nstep_size = int(train_size / steps)\nprint(\"step_size\", step_size)\n\nfor j in tqdm(range(29, 60)):\n    for i in range(steps):\n        \n        ###################################\n#         if i > 0:\n#             continue\n        ###################################\n        \n        print(i, '/', steps)\n        \n        AUTOTUNE = tf.data.AUTOTUNE\n\n        train_ds_subset = train_ds.skip(i * step_size).take(step_size).cache().shuffle(step_size).prefetch(buffer_size=AUTOTUNE)\n#         train_ds_subset = train_ds.shuffle(1000).prefetch(buffer_size=AUTOTUNE)\n#         val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)\n\n        epochs=1\n        history = model.fit(\n            train_ds_subset,\n#             validation_data=val_ds,\n            epochs=epochs,\n            class_weight=class_weight,\n            \n            #####################################\n#             steps_per_epoch=5,\n            #####################################\n        )\n\n\n    if j % 3 == 1 or j == 59:\n#     if True:\n\n        model.save(\"my_model_\" + str(j) + \".keras\")\n    \n    \n        y_true, y_pred = [], []\n        for i, d in enumerate(val_list):\n            if i % 20 == 0:\n                print(i / len(val_list))\n            label, prediction = evaluate_single_img(model, d, class_names, img2lb, img_height, img_width, batch_size=32)\n            y_true.append(label2idx[label])\n            y_pred.append(label2idx[prediction])\n\n        print(classification_report(y_true, y_pred, target_names=class_names[:5]))\n        \n        with open('classification_report', 'a') as f_out:\n            f_out.writelines('epoch: ' + str(j + 1) + '\\n')\n            f_out.writelines(classification_report(y_true, y_pred))\n            f_out.writelines('\\n\\n')","metadata":{"execution":{"iopub.status.busy":"2023-12-10T07:11:40.160711Z","iopub.execute_input":"2023-12-10T07:11:40.16138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def evaluate_single_img(model, d, class_names, img2lb, img_height, img_width, batch_size=32):\n    \n#     img_dir = \"/kaggle/input/tiles-of-cancer-2048px-scale-0-25/\" + str(d) + \"/\"\n    \n#     ds = tf.keras.utils.image_dataset_from_directory(\n#         img_dir,\n#         image_size=(img_height, img_width),\n#         batch_size=batch_size,\n#         shuffle=False,\n#         labels=None,\n#     )\n    \n#     results = model.predict(ds, verbose=0)\n    \n#     sm = tf.nn.softmax(\n#         results, axis=None, name=None\n#     )\n    \n#     #####################################\n#     # Discard \"Other\" logits\n#     sm = sm[..., 1:]\n    \n#     #####################################\n\n#     results1 = tf.math.reduce_max(sm, axis=0)\n\n#     results2 = tf.math.argmax(\n#         results1,\n#     )\n    \n#     prediction = class_names[results2]\n#     label = img2lb[str(d)]\n    \n#     return label, prediction\n\n\n\n# train_df = pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv')\n\n# img2lb = dict()\n# for i in range(len(train_df)):\n#     img2lb[str(train_df.loc[i, 'image_id'])] = train_df.loc[i, 'label']\n    \n    \n# evaluate_single_img(model, \"10800\", ['CC', 'EC', 'HGSC', 'LGSC', 'MC', 'Other'], img2lb, 456, 456)","metadata":{"execution":{"iopub.status.busy":"2023-12-10T00:19:59.914926Z","iopub.execute_input":"2023-12-10T00:19:59.915701Z","iopub.status.idle":"2023-12-10T00:20:03.705663Z","shell.execute_reply.started":"2023-12-10T00:19:59.91567Z","shell.execute_reply":"2023-12-10T00:20:03.704805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_true","metadata":{"execution":{"iopub.status.busy":"2023-12-10T00:21:16.056038Z","iopub.execute_input":"2023-12-10T00:21:16.056749Z","iopub.status.idle":"2023-12-10T00:21:16.064293Z","shell.execute_reply.started":"2023-12-10T00:21:16.05672Z","shell.execute_reply":"2023-12-10T00:21:16.063337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_dir = '/kaggle/input/ubc-ocean-data0-1/train_dir/'\n\n# # dirs = os.listdir('/kaggle/input/ubc-ocean-data/train_dir')\n# dirs = os.listdir(train_dir)\n\n\n# print(dirs)\n\n\n# d2cnt = dict()\n# class_weight = dict()\n# label2idx = {'CC': 0, 'EC': 1, 'HGSC': 2, 'LGSC': 3, 'MC': 4, 'Other': 5}\n\n# for d in dirs:\n#     d2cnt[d] = len(glob.glob(train_dir + d + '/*/*.png'))\n#     print(d, d2cnt[d])","metadata":{"execution":{"iopub.status.busy":"2023-12-09T22:50:44.21575Z","iopub.execute_input":"2023-12-09T22:50:44.216791Z","iopub.status.idle":"2023-12-09T22:50:47.456274Z","shell.execute_reply.started":"2023-12-09T22:50:44.216744Z","shell.execute_reply":"2023-12-09T22:50:47.455265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# d = 'CC'\n# glob.glob('/kaggle/input/ubc-ocean-data0-1/train_dir/' + d + '/*/*.png')","metadata":{"execution":{"iopub.status.busy":"2023-12-09T22:49:09.884218Z","iopub.execute_input":"2023-12-09T22:49:09.884595Z","iopub.status.idle":"2023-12-09T22:49:10.660277Z","shell.execute_reply.started":"2023-12-09T22:49:09.884565Z","shell.execute_reply":"2023-12-09T22:49:10.659396Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}