{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install imagededup","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-22T05:39:14.68715Z","iopub.execute_input":"2023-07-22T05:39:14.687894Z","iopub.status.idle":"2023-07-22T05:39:29.743336Z","shell.execute_reply.started":"2023-07-22T05:39:14.687856Z","shell.execute_reply":"2023-07-22T05:39:29.74126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.simplefilter('ignore')\nwarnings.filterwarnings('ignore', module='imagededup')\n\n\nfrom imagededup.methods import CNN, PHash\nfrom multiprocessing import cpu_count\n\nnum_cpus = cpu_count()\nprint(\"Number of CPUs:\", num_cpus)\n\n\nimage_dir = \"/kaggle/input/contrail-png-image-mask/contrail\"","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:39:29.745338Z","iopub.execute_input":"2023-07-22T05:39:29.745674Z","iopub.status.idle":"2023-07-22T05:39:33.899697Z","shell.execute_reply.started":"2023-07-22T05:39:29.745645Z","shell.execute_reply":"2023-07-22T05:39:33.898534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NOTE: There will be cases where\n# {'1.png': ('2.png', 0.96),\n#  '2.png': ('1.png', 0.96)}","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:39:33.901432Z","iopub.execute_input":"2023-07-22T05:39:33.902473Z","iopub.status.idle":"2023-07-22T05:39:33.907998Z","shell.execute_reply.started":"2023-07-22T05:39:33.902432Z","shell.execute_reply":"2023-07-22T05:39:33.906909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN","metadata":{}},{"cell_type":"code","source":"cnn = CNN()\nencodings = cnn.encode_images(image_dir)","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:39:33.911271Z","iopub.execute_input":"2023-07-22T05:39:33.912096Z","iopub.status.idle":"2023-07-22T05:46:20.281219Z","shell.execute_reply.started":"2023-07-22T05:39:33.912054Z","shell.execute_reply":"2023-07-22T05:46:20.28007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# min_similarity_threshold (0-1) default 0.9\nduplicates = cnn.find_duplicates(\n    encoding_map=encodings,\n    scores=True,\n    min_similarity_threshold=0.95,\n    outfile='duplicates.json'\n)","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:46:20.283199Z","iopub.execute_input":"2023-07-22T05:46:20.283515Z","iopub.status.idle":"2023-07-22T05:58:42.03183Z","shell.execute_reply.started":"2023-07-22T05:46:20.283487Z","shell.execute_reply":"2023-07-22T05:58:42.030443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (img, curr_duplicates) in enumerate(duplicates.items()):\n    print(img, curr_duplicates)\n    if i == 15: break","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:58:42.033818Z","iopub.execute_input":"2023-07-22T05:58:42.034767Z","iopub.status.idle":"2023-07-22T05:58:42.041162Z","shell.execute_reply.started":"2023-07-22T05:58:42.034723Z","shell.execute_reply":"2023-07-22T05:58:42.040124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Custom Models","metadata":{}},{"cell_type":"code","source":"# from imagededup.methods import CNN\n\n# # Get CustomModel construct\n# from imagededup.utils import CustomModel\n\n# # Get the prepackaged models from imagededup\n# from imagededup.utils.models import ViT, MobilenetV3, EfficientNet\n\n\n# # Declare a custom config with CustomModel, the prepackaged models come with a name and transform function\n# custom_config = CustomModel(name=EfficientNet.name,\n#                             model=EfficientNet(), \n#                             transform=EfficientNet.transform)\n\n# # Use model_config argument to pass the custom config\n# cnn = CNN(model_config=custom_config)","metadata":{"execution":{"iopub.status.busy":"2023-07-22T05:58:42.04263Z","iopub.execute_input":"2023-07-22T05:58:42.043066Z","iopub.status.idle":"2023-07-22T05:58:42.053124Z","shell.execute_reply.started":"2023-07-22T05:58:42.043021Z","shell.execute_reply":"2023-07-22T05:58:42.052246Z"},"trusted":true},"execution_count":null,"outputs":[]}]}