{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip -q install tensorflow==2.3.0","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-03-04T18:01:21.521095Z","iopub.execute_input":"2022-03-04T18:01:21.521519Z","iopub.status.idle":"2022-03-04T18:01:29.922819Z","shell.execute_reply.started":"2022-03-04T18:01:21.521473Z","shell.execute_reply":"2022-03-04T18:01:29.92165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basics / Data manipulation\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport zipfile\nimport os\n\n# Visualization\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport cv2\nimport skimage.io\n\n# ML\nfrom sklearn import model_selection\nfrom sklearn.model_selection import train_test_split\n\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import models\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.applications import NASNetMobile\nfrom tensorflow.keras.applications.imagenet_utils import preprocess_input\n\n#Use this to check if the GPU is configured correctly\nfrom tensorflow.python.client import device_lib\nprint(device_lib.list_local_devices())\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:29.925775Z","iopub.execute_input":"2022-03-04T18:01:29.92627Z","iopub.status.idle":"2022-03-04T18:01:29.950884Z","shell.execute_reply.started":"2022-03-04T18:01:29.926227Z","shell.execute_reply":"2022-03-04T18:01:29.949742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data\n10k+ of .tiff images\n*    **80%** for training \n*    **20%** for internal testing\n            *  10% Validation\n            *  10% Testing","metadata":{"editable":false}},{"cell_type":"markdown","source":"# Checking if GPU is being used","metadata":{"editable":false}},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print(\"Running on TPU \", tpu.cluster_spec().as_dict()[\"worker\"])\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept ValueError:\n    print(\"Not connected to a TPU runtime. Using CPU/GPU strategy\")\n    strategy = tf.distribute.MirroredStrategy()","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-03-04T18:01:29.952217Z","iopub.execute_input":"2022-03-04T18:01:29.952785Z","iopub.status.idle":"2022-03-04T18:01:29.964243Z","shell.execute_reply.started":"2022-03-04T18:01:29.952749Z","shell.execute_reply":"2022-03-04T18:01:29.963109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model \nThe model will have the follow configuration:\n______________\n1st layer: NASNetMobile (224, 224, 3) input images\n______________\n2nd layer: GlobalMaxPooling2D\n______________\n3rd layer: Dropout with learning rate = 2e-5\n______________\n4th layer: Denser layer x 6 that will classify the image","metadata":{"editable":false}},{"cell_type":"code","source":"#NASNetMobile Image weights\n","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:29.965562Z","iopub.execute_input":"2022-03-04T18:01:29.966144Z","iopub.status.idle":"2022-03-04T18:01:29.985098Z","shell.execute_reply.started":"2022-03-04T18:01:29.966096Z","shell.execute_reply":"2022-03-04T18:01:29.983936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = NASNetMobile(weights='imagenet')\nconv_base = NASNetMobile(weights=\"imagenet\", include_top=False, input_shape=(224, 224, 3))\n# print(conv_base.summary())","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:02:00.658185Z","iopub.execute_input":"2022-03-04T18:02:00.658821Z","iopub.status.idle":"2022-03-04T18:02:18.646353Z","shell.execute_reply.started":"2022-03-04T18:02:00.65877Z","shell.execute_reply":"2022-03-04T18:02:18.64513Z"},"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x = conv_base.output\n# x = tf.keras.layers.GlobalAveragePooling2D()(x)\n\n# x = tf.keras.layers.Dense(1024, activation='relu')(x)\n# x = tf.keras.layers.Dense(1024, activation='relu')(x)\n# x = tf.keras.layers.Dense(1024, activation='relu')(x)\n# x = tf.keras.layers.Dense(512, activation='relu')(x)\n# preds = tf.keras.layers.Dense(2, activation ='softmax')(x)\n\n\nmodel = models.Sequential()\nmodel.add(conv_base)\nconv_base.trainable = False\nmodel.add(layers.GlobalMaxPooling2D(name=\"gap\"))\n# Avoid overfitting\nmodel.add(layers.Dropout(rate=0.5))\nmodel.add(layers.Dense(10, activation=\"softmax\", name=\"fc_out\"))\n\nmodel.compile(\n    loss=\"categorical_crossentropy\",\n    optimizer=optimizers.RMSprop(lr=2e-5),\n    metrics=[\"acc\"])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:46.609688Z","iopub.execute_input":"2022-03-04T17:50:46.609971Z","iopub.status.idle":"2022-03-04T17:50:50.975644Z","shell.execute_reply.started":"2022-03-04T17:50:46.609934Z","shell.execute_reply":"2022-03-04T17:50:50.974734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Unzipping The Files\nThe original images have been transformed into tiled mosaics. Each image_id has 8 mosaic variations; the variations have been grouped into their seperate own zips\nto work within a Kaggle restriction of  max 5 GB in ./kaggle/working & and max 20 GB in ./kaggle/tmp while training.csv + validation.csv + testing.csv are seperate & global. \n\nThe dataset has been split 90% Training, 7.5% Validation, and 2.5% Internal Testing. If you want to use all 10% of the data for Validation, just merge the appropriate dataframes & zipfile contents. \n\nOnly unzip a single variation at a time! The notebook will fail if you use up all the space in ./kaggle/working and your model will not be saved! If you need to move on to a different variation, delete the old files! You can unzip them again later, no problem.\n","metadata":{}},{"cell_type":"code","source":"### CAUTION ###\n\nvariations = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\", \"H\"]\n#variations = [\"A\", \"B\"]\n\ndef zippity(variant):\n    print(f'Variation {variant}')\n\n    with zipfile.ZipFile(f'../input/8-fold-pc-dataset-gen-{variations.index(variant) + 1}-8-{variant.lower()}/train{variant}.zip','r') as z:\n        z.extractall(\".\")\n    \n    with zipfile.ZipFile(f'../input/8-fold-pc-dataset-gen-{variations.index(variant) + 1}-8-{variant.lower()}/validation{variant}.zip','r') as z:\n        z.extractall(\".\")\n    \n#     with zipfile.ZipFile(\"../input/pc-data-dataset-gen/test.zip\",\"r\") as z:\n#         z.extractall(\".\")","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:50.977676Z","iopub.execute_input":"2022-03-04T17:50:50.978243Z","iopub.status.idle":"2022-03-04T17:50:50.985181Z","shell.execute_reply.started":"2022-03-04T17:50:50.978209Z","shell.execute_reply":"2022-03-04T17:50:50.984239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def zappity():\n    # Deleting image folders to avoid over-saturate the output\n    !rm -r train\n    !rm -r validation\n#     !rm -r test","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:50.986801Z","iopub.execute_input":"2022-03-04T17:50:50.987095Z","iopub.status.idle":"2022-03-04T17:50:51.003289Z","shell.execute_reply.started":"2022-03-04T17:50:50.987067Z","shell.execute_reply":"2022-03-04T17:50:51.002224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Additional Data Augmentation","metadata":{}},{"cell_type":"code","source":"image_gen = ImageDataGenerator(\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    rescale=1/255,\n    shear_range=0.2,\n    zoom_range=0.2,\n    fill_mode=\"nearest\",\n    preprocessing_function=tf.keras.applications.nasnet.preprocess_input)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:51.00621Z","iopub.execute_input":"2022-03-04T17:50:51.006627Z","iopub.status.idle":"2022-03-04T17:50:51.019949Z","shell.execute_reply.started":"2022-03-04T17:50:51.00658Z","shell.execute_reply":"2022-03-04T17:50:51.018817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sample = plt.imread(\"../input/panda2/train_images/0005f7aaab2800f6170c399693a96917.png\")\n\n#plt.imshow(image_gen.random_transform(sample))","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:51.021563Z","iopub.execute_input":"2022-03-04T17:50:51.02196Z","iopub.status.idle":"2022-03-04T17:50:51.034219Z","shell.execute_reply.started":"2022-03-04T17:50:51.021912Z","shell.execute_reply":"2022-03-04T17:50:51.033165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\n\ndef which_image_gen(which):\n    if(which == \"train\"):\n        which_gen = image_gen.flow_from_directory(\"./train\",\n                                                  target_size=(224, 224),\n                                                  batch_size=batch_size,\n                                                  class_mode=\"categorical\")\n        \n    \n    elif(which == \"valid\"):\n        which_gen = image_gen.flow_from_directory(\"./validation\",\n                                                  target_size=(224, 224),\n                                                  batch_size=batch_size,\n                                                  class_mode=\"categorical\")\n    \n#     elif(which == \"test\"):\n#         which_gen = image_gen.flow_from_directory(\"./test\",\n#                                                   target_size=(224, 224),\n#                                                   batch_size=batch_size,\n#                                                   class_mode=\"categorical\")\n    return which_gen\n","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:51.03541Z","iopub.execute_input":"2022-03-04T17:50:51.035676Z","iopub.status.idle":"2022-03-04T17:50:51.04817Z","shell.execute_reply.started":"2022-03-04T17:50:51.035641Z","shell.execute_reply":"2022-03-04T17:50:51.046167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUMBER_OF_TRAINING_IMAGES = len(pd.read_csv('../input/8-fold-pc-dataset-gen-0-8/training.csv'))\nNUMBER_OF_VALIDATION_IMAGES = len(pd.read_csv('../input/8-fold-pc-dataset-gen-0-8/validation.csv'))\nNUMBER_OF_TESTING_IMAGES = len(pd.read_csv('../input/8-fold-pc-dataset-gen-0-8/testing.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:51.049689Z","iopub.execute_input":"2022-03-04T17:50:51.049988Z","iopub.status.idle":"2022-03-04T17:50:51.13431Z","shell.execute_reply.started":"2022-03-04T17:50:51.049958Z","shell.execute_reply":"2022-03-04T17:50:51.133412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for variety in variations:\n    zippity(variety)\n    \n    train_image_gen = which_image_gen(\"train\")\n    validation_image_gen = which_image_gen(\"valid\")\n#     test_image_gen = which_image_gen(\"test\")\n\n#     Flowing through directories to see the classes and the number of images\n#     print(image_gen.flow_from_directory(\"./train\"))\n#     print(image_gen.flow_from_directory(\"./validation\"))\n#     print(image_gen.flow_from_directory(\"./test\"))\n\n#     train_image_gen.class_indices\n#     validation_image_gen.class_indices\n#     test_image_gen.class_indices\n\n    results = model.fit(\n        train_image_gen,\n        steps_per_epoch=NUMBER_OF_TRAINING_IMAGES // batch_size,\n        epochs=20,\n        validation_data=validation_image_gen,\n        validation_steps=NUMBER_OF_VALIDATION_IMAGES // batch_size,\n        verbose=1,\n        use_multiprocessing=True,\n        workers=4,\n    )\n    \n    # Saving the synaptic weights of the model\n    model.save(\"./NASNetMobile-model.h5\")\n    \n    zappity()\n","metadata":{"execution":{"iopub.status.busy":"2022-03-04T17:50:51.135794Z","iopub.execute_input":"2022-03-04T17:50:51.13615Z","iopub.status.idle":"2022-03-04T18:01:16.998024Z","shell.execute_reply.started":"2022-03-04T17:50:51.13611Z","shell.execute_reply":"2022-03-04T18:01:16.995589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_hist_acc(hist):\n    plt.plot(hist.history[\"acc\"])\n    plt.plot(hist.history[\"val_acc\"])\n    plt.title(\"Model Accuracy\")\n    plt.ylabel(\"Accuracy\")\n    plt.xlabel(\"Epoch\")\n    plt.legend([\"Accuracy\", \"Validation Accuracy\"], loc=\"upper left\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:17.000185Z","iopub.status.idle":"2022-03-04T18:01:17.001049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_hist_loss(hist):\n    plt.plot(hist.history[\"loss\"])\n    plt.plot(hist.history[\"val_loss\"])\n    plt.title(\"Model Loss\")\n    plt.ylabel(\"Accuracy\")\n    plt.xlabel(\"Epoch\")\n    plt.legend([\"Loss\", \"Validation Loss\"], loc=\"upper left\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:17.002714Z","iopub.status.idle":"2022-03-04T18:01:17.00367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Saving the synaptic weights of the model\nmodel.save(\"./NASNetMobile-model.h5\")","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-03-04T18:01:17.005447Z","iopub.status.idle":"2022-03-04T18:01:17.006108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results_df = pd.DataFrame({\"epoch\":[i + 1 for i in range(len(results.history[\"acc\"]))], \"acc\":results.history[\"acc\"], \"val_acc\":results.history[\"val_acc\"], \"loss\":results.history[\"loss\"], \"val_loss\":results.history[\"val_loss\"]})\nresults_df","metadata":{"editable":false,"execution":{"iopub.status.busy":"2022-03-04T18:01:17.007855Z","iopub.status.idle":"2022-03-04T18:01:17.008479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist_acc(results)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:17.010087Z","iopub.status.idle":"2022-03-04T18:01:17.01068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_hist_loss(results)","metadata":{"execution":{"iopub.status.busy":"2022-03-04T18:01:17.012638Z","iopub.status.idle":"2022-03-04T18:01:17.013331Z"},"trusted":true},"execution_count":null,"outputs":[]}]}