{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nos.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,60).__str__()\nos.environ[\"CV_IO_MAX_IMAGE_PIXELS\"] = pow(2,60).__str__()\nimport time\nimport cv2","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-14T16:29:15.086153Z","iopub.execute_input":"2023-10-14T16:29:15.087486Z","iopub.status.idle":"2023-10-14T16:29:16.257491Z","shell.execute_reply.started":"2023-10-14T16:29:15.087447Z","shell.execute_reply":"2023-10-14T16:29:16.256328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read the train/test csvs","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/UBC-OCEAN/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-14T17:43:40.544181Z","iopub.execute_input":"2023-10-14T17:43:40.544633Z","iopub.status.idle":"2023-10-14T17:43:40.564849Z","shell.execute_reply.started":"2023-10-14T17:43:40.544588Z","shell.execute_reply":"2023-10-14T17:43:40.562835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initial Preprocessing","metadata":{}},{"cell_type":"markdown","source":"### Note: we are adding the thumbnails path as well, even if it does not exist for some images.","metadata":{}},{"cell_type":"code","source":"train['image_path'] = train['image_id'].apply(lambda x: \"/kaggle/input/UBC-OCEAN/train_images/\"+str(x)+\".png\")\ntrain['thumbnail_path'] = train['image_id'].apply(lambda x: \"/kaggle/input/UBC-OCEAN/train_thumbnails/\"+str(x)+\"_thumbnail.png\")","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:00:40.99065Z","iopub.execute_input":"2023-10-14T18:00:40.991999Z","iopub.status.idle":"2023-10-14T18:00:41.001681Z","shell.execute_reply.started":"2023-10-14T18:00:40.991938Z","shell.execute_reply":"2023-10-14T18:00:41.000401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Area is a good basic parameter to analyse the size of the images","metadata":{}},{"cell_type":"code","source":"train['image_area'] = train['image_width']*train['image_height']","metadata":{"execution":{"iopub.status.busy":"2023-10-14T17:48:17.074495Z","iopub.execute_input":"2023-10-14T17:48:17.074926Z","iopub.status.idle":"2023-10-14T17:48:17.085808Z","shell.execute_reply.started":"2023-10-14T17:48:17.074897Z","shell.execute_reply":"2023-10-14T17:48:17.084354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Basic analyses","metadata":{}},{"cell_type":"code","source":"train[['image_width','image_height','image_area']].describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-14T17:49:33.257234Z","iopub.execute_input":"2023-10-14T17:49:33.257639Z","iopub.status.idle":"2023-10-14T17:49:33.289482Z","shell.execute_reply.started":"2023-10-14T17:49:33.257612Z","shell.execute_reply":"2023-10-14T17:49:33.288337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('label')[['image_width','image_height','image_area']].mean().plot(kind = 'bar')\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2023-10-14T17:53:30.557326Z","iopub.execute_input":"2023-10-14T17:53:30.557861Z","iopub.status.idle":"2023-10-14T17:53:31.200038Z","shell.execute_reply.started":"2023-10-14T17:53:30.557821Z","shell.execute_reply":"2023-10-14T17:53:31.198508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('is_tma')[['image_width','image_height','image_area']].mean().plot(kind = 'bar')\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2023-10-14T17:54:39.964133Z","iopub.execute_input":"2023-10-14T17:54:39.964633Z","iopub.status.idle":"2023-10-14T17:54:40.592611Z","shell.execute_reply.started":"2023-10-14T17:54:39.964598Z","shell.execute_reply":"2023-10-14T17:54:40.590669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyses of the image sizes in the memory ","metadata":{}},{"cell_type":"code","source":"def get_size(path):\n    if os.path.isfile(path):\n        file_size_bytes = os.path.getsize(path)\n        file_size_mb = file_size_bytes / (1000 * 1000) \n        return file_size_mb\n    else:\n        return None\nget_size(\"/kaggle/input/UBC-OCEAN/test_images/41.png\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Note : some say that it should be $1024 \\times 1024$ , but the size shown in the data explorer on Kaggle shows the value when divided by $1000 \\times 1000$ \n![scr](https://i.imgur.com/w3RoJBt.png)","metadata":{}},{"cell_type":"code","source":"train['image_size'] = train['image_path'].apply(get_size)\ntrain['thumbnail_size'] = train['thumbnail_path'].apply(get_size)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:00:44.557425Z","iopub.execute_input":"2023-10-14T18:00:44.557829Z","iopub.status.idle":"2023-10-14T18:00:45.867765Z","shell.execute_reply.started":"2023-10-14T18:00:44.5578Z","shell.execute_reply":"2023-10-14T18:00:45.86621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['image_width','image_height','image_area', 'image_size', 'thumbnail_size']].describe()","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:01:22.949406Z","iopub.execute_input":"2023-10-14T18:01:22.949944Z","iopub.status.idle":"2023-10-14T18:01:22.986922Z","shell.execute_reply.started":"2023-10-14T18:01:22.949907Z","shell.execute_reply":"2023-10-14T18:01:22.985418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(train[['image_width','image_height','image_area', 'image_size', 'thumbnail_size']].corr(), annot = True)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:03:00.297319Z","iopub.execute_input":"2023-10-14T18:03:00.297937Z","iopub.status.idle":"2023-10-14T18:03:00.767445Z","shell.execute_reply.started":"2023-10-14T18:03:00.297896Z","shell.execute_reply":"2023-10-14T18:03:00.766185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Interesting that thumbnail size is negatively correlated 🤔","metadata":{}},{"cell_type":"code","source":"sorted_train = train.sort_values('image_area').reset_index(drop = True)\nsns.pairplot(sorted_train[['image_width','image_height','image_area', 'image_size', 'thumbnail_size']])","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:14:28.467259Z","iopub.execute_input":"2023-10-14T18:14:28.467755Z","iopub.status.idle":"2023-10-14T18:14:36.972145Z","shell.execute_reply.started":"2023-10-14T18:14:28.467721Z","shell.execute_reply":"2023-10-14T18:14:36.97104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Every column except thumbnail size is almost positively correlated. Correlation gets more scattered and unpredictable once the image size/area increases","metadata":{}},{"cell_type":"markdown","source":"# Analyses of Image loading times","metadata":{}},{"cell_type":"markdown","source":"### Using cv2 image opening as a benchmark, this can be extended to other ways of loading images and can be compared.","metadata":{}},{"cell_type":"code","source":"def get_image_load_time(path):\n    if os.path.isfile(path):\n        start_time = time.time()\n        img = cv2.imread(path,cv2.IMREAD_COLOR)\n        open_time = time.time() - start_time\n        del img\n        return open_time\n    else:\n        return None","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:11:28.969398Z","iopub.execute_input":"2023-10-14T18:11:28.969971Z","iopub.status.idle":"2023-10-14T18:11:28.982013Z","shell.execute_reply.started":"2023-10-14T18:11:28.969934Z","shell.execute_reply":"2023-10-14T18:11:28.980643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_times = sorted_train.iloc[::25]['image_path'].apply(get_image_load_time) \nthumbnail_times = sorted_train.iloc[::25]['thumbnail_path'].apply(get_image_load_time)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:15:13.458371Z","iopub.execute_input":"2023-10-14T18:15:13.459989Z","iopub.status.idle":"2023-10-14T18:39:33.094268Z","shell.execute_reply.started":"2023-10-14T18:15:13.459891Z","shell.execute_reply":"2023-10-14T18:39:33.092611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_train.loc[::25,'image_time'] = image_times\nsorted_train.loc[::25,'thumbnail_time'] = thumbnail_times","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:42:32.592947Z","iopub.execute_input":"2023-10-14T18:42:32.593423Z","iopub.status.idle":"2023-10-14T18:42:32.605995Z","shell.execute_reply.started":"2023-10-14T18:42:32.593391Z","shell.execute_reply":"2023-10-14T18:42:32.604621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_train['image_time'] = sorted_train['image_time'].interpolate()\nsorted_train['thumbnail_time'] = sorted_train['thumbnail_time'].interpolate()","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:43:59.24549Z","iopub.execute_input":"2023-10-14T18:43:59.246048Z","iopub.status.idle":"2023-10-14T18:43:59.256916Z","shell.execute_reply.started":"2023-10-14T18:43:59.24601Z","shell.execute_reply":"2023-10-14T18:43:59.255018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Note: I only calculated 1 fourth of all the images, and others were interpolated","metadata":{}},{"cell_type":"code","source":"sorted_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:44:34.425454Z","iopub.execute_input":"2023-10-14T18:44:34.426902Z","iopub.status.idle":"2023-10-14T18:44:34.44382Z","shell.execute_reply.started":"2023-10-14T18:44:34.426852Z","shell.execute_reply":"2023-10-14T18:44:34.442744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sorted_train.plot()\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:45:18.997019Z","iopub.execute_input":"2023-10-14T18:45:18.997479Z","iopub.status.idle":"2023-10-14T18:45:19.521803Z","shell.execute_reply.started":"2023-10-14T18:45:18.997449Z","shell.execute_reply":"2023-10-14T18:45:19.52075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(sorted_train[['image_width','image_height','image_area', 'image_size', 'thumbnail_size', 'image_time','thumbnail_time']].corr(), annot = True)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:50:03.082292Z","iopub.execute_input":"2023-10-14T18:50:03.08288Z","iopub.status.idle":"2023-10-14T18:50:03.622248Z","shell.execute_reply.started":"2023-10-14T18:50:03.08284Z","shell.execute_reply":"2023-10-14T18:50:03.621135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.pairplot(sorted_train[['image_width','image_height','image_area', 'image_size', 'thumbnail_size', 'image_time','thumbnail_time']])","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:46:46.945247Z","iopub.execute_input":"2023-10-14T18:46:46.945783Z","iopub.status.idle":"2023-10-14T18:47:03.513694Z","shell.execute_reply.started":"2023-10-14T18:46:46.945744Z","shell.execute_reply":"2023-10-14T18:47:03.5123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Thumbnais are really weird ahaha, no proper correlation or anything. Not even correlated with the respective loading time","metadata":{}},{"cell_type":"markdown","source":"# Dataframe saving in the output, if y'all wanna analyse it more!","metadata":{}},{"cell_type":"code","source":"sorted_train.to_csv('sorted_train.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:50:48.225942Z","iopub.execute_input":"2023-10-14T18:50:48.226367Z","iopub.status.idle":"2023-10-14T18:50:48.247275Z","shell.execute_reply.started":"2023-10-14T18:50:48.226338Z","shell.execute_reply":"2023-10-14T18:50:48.245883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thank you! \nFeel free to checkout my other work for this competition:\n* Tiles of the training dataset : \n\n    https://www.kaggle.com/datasets/pjmathematician/ucbo-tiles-256-1\n    \n    https://www.kaggle.com/datasets/pjmathematician/ucbo-tiles-256-2\n    \n    https://www.kaggle.com/datasets/pjmathematician/ucbo-tiles-256-3\n    \n    https://www.kaggle.com/datasets/pjmathematician/ucbo-tiles-256-4\n    \n    https://www.kaggle.com/datasets/pjmathematician/ucbo-tiles-256-5\n    \n    and the loader notebook:\n    \n    https://www.kaggle.com/code/pjmathematician/ucbo-256-tiles-loading\n* Keras Baseline Training/Inference:\n    https://www.kaggle.com/code/pjmathematician/ubco-keras-cnn-baseline-thumbnails-inference\n    https://www.kaggle.com/code/pjmathematician/ubco-keras-cnn-baseline-thumbnails-training\n    \nIf you found any of this helpful, consider giving upvote! ","metadata":{}},{"cell_type":"code","source":"print(\"😼\")","metadata":{"execution":{"iopub.status.busy":"2023-10-14T18:55:11.447956Z","iopub.execute_input":"2023-10-14T18:55:11.449747Z","iopub.status.idle":"2023-10-14T18:55:11.470361Z","shell.execute_reply.started":"2023-10-14T18:55:11.449581Z","shell.execute_reply":"2023-10-14T18:55:11.466537Z"},"trusted":true},"execution_count":null,"outputs":[]}]}