{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport zipfile\nimport random\nimport tensorflow as tf\nimport shutil\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom shutil import copyfile\nfrom os import getcwd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    path = \"../tiles\"\n    os.mkdir(path)\n    classes = ['gl3_yes', 'gl3_no', 'gl4_yes', 'gl4_no', 'gl5_yes', 'gl5_no']\n    t_types = ['training', 'testing']\n    for item in t_types:\n        path1 = os.path.join(path, item)\n        os.mkdir(path1)\n        for item in classes:\n            path2 = os.path.join(path1, item)\n            os.mkdir(path2)\nexcept OSError as error:\n    print(error)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfor item in classes[0:2]:\n    os.mkdir('../'+item)\n    with zipfile.ZipFile('../input/sorting-tiles-into-classes'+item[:3]+'/'+item+'.zip') as z:\n        z.extractall('../'+item)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for item in classes[0:2]:\n    print(len(os.listdir('../'+item)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sz = min([len(os.listdir('../'+item)) for item in classes[0:2]])\nfiles = os.listdir('../'+classes[1])\nshuffl = random.sample(files, len(files))\ndiscard = shuffl[sz:]\nfor item in discard:\n    os.remove('../'+classes[1]+'/'+item)\nprint(len(os.listdir('../'+classes[1])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def split_data(SOURCE, TRAINING, TESTING, SPLIT_SIZE):\n    source_files = os.listdir(SOURCE)\n    good_files = [f for f in source_files if os.path.getsize(os.path.join(SOURCE, f)) != 0]\n    shuffle = random.sample(good_files, len(good_files))\n    stop = int(len(good_files) * SPLIT_SIZE)\n    train_files = shuffle[:stop]\n    test_files = shuffle[stop:]\n    for item in train_files:\n        f_source  = os.path.join(SOURCE, item)\n        f_dest = os.path.join(TRAINING, item)\n        copyfile(f_source, f_dest)\n    for item in test_files:\n        f_source  = os.path.join(SOURCE, item)\n        f_dest = os.path.join(TESTING, item)\n        copyfile(f_source, f_dest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yes3_source_dir = '../gl3_yes/'\ntraining_yes3_dir = '../tiles/training/gl3_yes/'\ntesting_yes3_dir = '../tiles/testing/gl3_yes/'\n\nno3_source_dir = '../gl3_no/'\ntraining_no3_dir = '../tiles/training/gl3_no/'\ntesting_no3_dir = '../tiles/testing/gl3_no/'\n\nyes4_source_dir = '../gl4_yes/'\ntraining_yes4_dir = '../tiles/training/gl4_yes/'\ntesting_yes4_dir = '../tiles/testing/gl4_yes/'\n\nno4_source_dir = '../gl4_no/'\ntraining_no4_dir = '../tiles/training/gl4_no/'\ntesting_no4_dir = '../tiles/testing/gl4_no/'\n\nyes5_source_dir = '../gl5_yes/'\ntraining_yes5_dir = '../tiles/training/gl5_yes/'\ntesting_yes5_dir = '../tiles/testing/gl5_yes/'\n\nno5_source_dir = '../gl5_no/'\ntraining_no5_dir = '../tiles/training/gl5_no/'\ntesting_no5_dir = '../tiles/testing/gl5_no/'\n\nsplit_size = .8\nsplit_data(yes3_source_dir, training_yes3_dir, testing_yes3_dir, split_size)\nsplit_data(no3_source_dir, training_no3_dir, testing_no3_dir, split_size)\n\n#split_data(yes4_source_dir, training_yes4_dir, testing_yes4_dir, split_size)\n#split_data(no4_source_dir, training_no4_dir, testing_no4_dir, split_size)\n\n#split_data(yes5_source_dir, training_yes5_dir, testing_yes5_dir, split_size)\n#split_data(no5_source_dir, training_no5_dir, testing_no5_dir, split_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(os.listdir('../tiles/training/gl3_yes/')))\nprint(len(os.listdir('../tiles/training/gl3_no/')))\nprint(len(os.listdir('../tiles/testing/gl3_yes/')))\nprint(len(os.listdir('../tiles/testing/gl3_no/')))\n\n#print(len(os.listdir('../tiles/training/gl4_yes/')))\n#print(len(os.listdir('../tiles/training/gl4_no/')))\n#print(len(os.listdir('../tiles/testing/gl4_yes/')))\n#print(len(os.listdir('../tiles/testing/gl4_no/')))\n\n#print(len(os.listdir('../tiles/training/gl4_yes/')))\n#print(len(os.listdir('../tiles/training/gl4_no/')))\n#print(len(os.listdir('../tiles/testing/gl4_yes/')))\n#print(len(os.listdir('../tiles/testing/gl4_no/')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for item in t_types:\n    path = \"../tiles\"\n    path1 = os.path.join(path, item)\n    for item in classes:\n        path2 = os.path.join(path1, item)\n        if os.path.isdir(path2):\n            if len(os.listdir(path2)) == 0:\n                shutil.rmtree(path2)\n        else:\n            print(path2+ ' doesnae exist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import Model\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nimport cv2\nfrom tensorflow.keras.applications import ResNet50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.applications import ResNet50\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre_trained_model = ResNet50(include_top = False, pooling = 'avg', weights = 'imagenet', input_shape=(224, 224, 3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pre_trained_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for layer in pre_trained_model.layers:\n    layer.trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom tensorflow.keras.optimizers import RMSprop\nx = tf.keras.layers.Flatten()(pre_trained_model.output)\n# Add a fully connected layer with 1,024 hidden units and ReLU activation\nx = layers.Dense(128, activation='relu')(x)\n# Add a dropout rate of 0.2\nx = layers.Dropout(0.2)(x)                  \n# Add a final sigmoid layer for classification\nx = layers.Dense  (1, activation='sigmoid')(x)           \n\nmodel = Model( pre_trained_model.input, x) \n\nmodel.compile(optimizer = RMSprop(lr=0.0005), \n              loss = 'binary_crossentropy', \n              metrics = ['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ntrain_datagen = ImageDataGenerator()\ntest_datagen = ImageDataGenerator()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dir = '../tiles/training'\nvalidation_dir = '../tiles/testing'\ntrain_generator = train_datagen.flow_from_directory(train_dir,\n                                                    batch_size = 50,\n                                                    class_mode = 'binary', \n                                                    target_size = (224, 224))     \n\n# Flow validation images in batches of 20 using test_datagen generator\nvalidation_generator =  test_datagen.flow_from_directory( validation_dir,\n                                                          batch_size  = 50,\n                                                          class_mode  = 'binary', \n                                                          target_size = (224, 224))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.python.keras.callbacks import ModelCheckpoint\ncb_checkpointer = ModelCheckpoint(filepath = '../working/best.hdf5', monitor = 'val_loss', save_best_only = True, mode = 'auto')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n            train_generator,\n            validation_data = validation_generator,\n            steps_per_epoch = 2446,\n            epochs = 30,\n            validation_steps = 661,\n            verbose = 2,\n            callbacks=[cb_checkpointer])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save('gl3_model')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dirs = ['../gl3_no', '../gl3_yes', '../tiles']\nfor item in dirs:\n    shutil.rmtree(item)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}