{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook is to share the C++ code I used to crop images into tiles. It is more straightforward to do it on local machines, because we have a 20 GB disk constraint on Kaggle and simply cannot process the whole ~700 GB dataset at once.","metadata":{}},{"cell_type":"markdown","source":"## setup env","metadata":{}},{"cell_type":"code","source":"!sudo apt-get update > /dev/null\n!sudo apt-get install libopencv-dev -y > /dev/null\n!g++ -v > /dev/null","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# compile","metadata":{}},{"cell_type":"code","source":"%%writefile image_tiling.cpp\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\nvoid cropAndSave(const cv::Mat& img, int x, int y, int d, const std::string& filename, const std::string& dirOut, std::atomic& savedCrops, std::atomic& skippedCrops) {\n cv::Rect roi(x, y, d, d);\n cv::Mat subImg = img(roi);\n\n // Convert the cropped image to grayscale\n cv::Mat graySubImg;\n cv::cvtColor(subImg, graySubImg, cv::COLOR_BGR2GRAY);\n\n // Calculate the number of non-blank pixels\n int nonBlankPixelCount = cv::countNonZero(graySubImg);\n float blankPixelRatio = 1.0f - static_cast(nonBlankPixelCount) / (d * d);\n\n // Only save the crop if less than 80% of the pixels are blank\n if (blankPixelRatio < 0.8) {\n std::string baseFilename = filename.substr(0, filename.find_last_of(\".\"));\n std::string outPath = dirOut + baseFilename + \"_\" + std::to_string(y) + \"_\" + std::to_string(x) + \".tiff\";\n \n cv::imwrite(outPath, subImg);\n savedCrops++; // Increment the saved crops count\n } else {\n skippedCrops++; // Increment the skipped crops count\n }\n \n}\n\nint main(int argc, char* argv[]) {\n if (argc < 2) {\n std::cerr << \"Usage: \" << argv[0] << \" \" << std::endl;\n return -1;\n }\n std::string filename = argv[1];\n \n std::string dirIn = \"/kaggle/input/UBC-OCEAN/train_images/\";\n std::string dirOut = \"./image_crop/\";\n int d = 1024; // desired width of each output image\n \n std::atomic savedCrops(0);\n std::atomic skippedCrops(0);\n \n namespace fs = std::filesystem;\n if (!fs::exists(dirOut)) {\n std::cout << \"Creating \" << dirOut << \" now.\" << std::endl;\n fs::create_directory(dirOut);\n }\n \n cv::Mat img = cv::imread(dirIn + filename);\n if (img.empty()) {\n std::cerr << \"Error opening image: \" << filename << \". Reason: \" << cv::getBuildInformation() << std::endl;\n return -1;\n }\n\n int width = img.cols;\n int height = img.rows;\n\n std::vector threads;\n for (int y = 0; y <= height - d; y += d) {\n for (int x = 0; x <= width - d; x += d) {\n threads.push_back(std::thread(cropAndSave, std::ref(img), x, y, d, filename, dirOut, std::ref(savedCrops), std::ref(skippedCrops)));\n }\n }\n\n for (auto& t : threads) {\n t.join();\n }\n \n // Print the results\n std::cout << \"Total crops saved: \" << savedCrops.load() << std::endl;\n std::cout << \"Total crops skipped: \" << skippedCrops.load() << std::endl;\n\n return 0;\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!g++ image_tiling.cpp -o image_tiling -I/usr/include/opencv4 -L/usr/lib/x86_64-linux-gnu -lopencv_core -lopencv_imgproc -lopencv_imgcodecs \\\n -std=c++17 -pthread","metadata":{"execution":{"iopub.status.busy":"2023-12-06T19:53:59.348893Z","iopub.execute_input":"2023-12-06T19:53:59.349851Z","iopub.status.idle":"2023-12-06T19:54:03.142571Z","shell.execute_reply.started":"2023-12-06T19:53:59.349812Z","shell.execute_reply":"2023-12-06T19:54:03.140291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['OPENCV_IO_MAX_IMAGE_PIXELS'] = str(pow(2, 70))","metadata":{"execution":{"iopub.status.busy":"2023-12-06T19:54:25.183242Z","iopub.execute_input":"2023-12-06T19:54:25.183707Z","iopub.status.idle":"2023-12-06T19:54:25.190287Z","shell.execute_reply.started":"2023-12-06T19:54:25.183673Z","shell.execute_reply":"2023-12-06T19:54:25.188843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# apply to one image (use a large 4.7 GB image as an example)\n\nOn local machines with sufficient disk space, you could simply add a loop to run it on all images.","metadata":{}},{"cell_type":"code","source":"!ls -lhS /kaggle/input/UBC-OCEAN/train_images/ | head -n 10","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!du -h /kaggle/input/UBC-OCEAN/train_images/45630.png","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n! ./image_tiling 45630.png","metadata":{"execution":{"iopub.status.busy":"2023-12-06T19:54:27.312531Z","iopub.execute_input":"2023-12-06T19:54:27.312959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show result","metadata":{}},{"cell_type":"code","source":"from PIL import Image\nImage.open(\"./image_crop/45630_21504_52224.tiff\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image.open(\"./image_crop/45630_34816_45056.tiff\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}