{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!go --version\n!wget https://go.dev/dl/go1.21.2.linux-amd64.tar.gz\n!rm -rf /usr/local/go && tar -C /usr/local -xzf go1.21.2.linux-amd64.tar.gz\nimport os\nos.environ['PATH'] += \":/usr/local/go/bin\"\n!echo $PATH","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-09T20:45:03.5898Z","iopub.execute_input":"2023-10-09T20:45:03.590176Z","iopub.status.idle":"2023-10-09T20:45:12.064396Z","shell.execute_reply.started":"2023-10-09T20:45:03.590147Z","shell.execute_reply":"2023-10-09T20:45:12.063167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_file(image_id,size):\n    file_string = \"\"\"\npackage main\n\nimport (\n\t\"fmt\"\n\t\"image\"\n\t\"image/jpeg\"\n\t\"image/png\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"sync\"\n    \"github.com/nfnt/resize\"\n)\n\n// cropAndSave crops the image based on provided coordinates and saves to the specified folder.\nfunc cropAndSave(img image.Image, x, y, d int, filename, dirOut string) {\n\trect := image.Rect(x, y, x+d, y+d)\n\tsubImg := img.(interface {\n\t\tSubImage(r image.Rectangle) image.Image\n\t}).SubImage(rect)\n    resizedImg := resize.Resize(SIZE, SIZE, subImg, resize.Lanczos3)\n\t\n\toutPath := filepath.Join(dirOut, fmt.Sprintf(\"%s_%d_%d%s\", strings.TrimSuffix(filename, filepath.Ext(filename)), y, x, filepath.Ext(filename)))\n\toutFile, err := os.Create(outPath)\n\tif err != nil {\n\t\tfmt.Println(\"Error creating file:\", err)\n\t\treturn\n\t}\n\tdefer outFile.Close()\n\n\tif filepath.Ext(filename) == \".jpg\" || filepath.Ext(filename) == \".jpeg\" {\n\t\tjpeg.Encode(outFile, resizedImg, nil)\n\t} else if filepath.Ext(filename) == \".png\" {\n\t\tpng.Encode(outFile, resizedImg)\n\t}\n}\n\nfunc main() {\n\tfilename := \"IMAGE_ID.png\" // Change this to your image file here\n\tdirIn := \"/kaggle/input/UBC-OCEAN/train_images/\"\n\tdirOut := \"/data/IMAGE_ID/\"\n\td := 1024  // desired width of each output image\n\n\timgFile, err := os.Open(filepath.Join(dirIn, filename))\n\tif err != nil {\n\t\tfmt.Println(\"Error opening image:\", err)\n\t\treturn\n\t}\n\tdefer imgFile.Close()\n\n\timg, _, err := image.Decode(imgFile)\n\tif err != nil {\n\t\tfmt.Println(\"Error decoding image:\", err)\n\t\treturn\n\t}\n\n\tbounds := img.Bounds()\n\twidth, height := bounds.Max.X, bounds.Max.Y\n\n\tvar wg sync.WaitGroup\n    \n\tfor y := 0; y <= height-d; y += d {\n\t\tfor x := 0; x <= width-d; x += d {\n\t\t\twg.Add(1)\n\t\t\tgo func(x, y int) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tcropAndSave(img, x, y, d, filename, dirOut)\n\t\t\t}(x, y)\n\t\t}\n\t}\n\n\twg.Wait()\n}\n\n    \"\"\"\n    file_string = (file_string.replace(\"IMAGE_ID\", str(image_id)).replace(\"SIZE\", str(size)))\n    with open(f\"tiler_{image_id}.go\", 'w') as f:\n        f.write(file_string)\n    file_string = \"\"\"\npackage main\n\nimport (\n\t\"fmt\"\n\t\"image\"\n\n\t// \"image/color\"\n\t_ \"image/jpeg\"\n\t_ \"image/png\"\n\t\"io/ioutil\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n)\n\n// Checks if the image is almost black\nfunc isAlmostBlack(img image.Image) bool {\n\tthreshold := 3 // Adjust this threshold as needed\n\ttotalPixels := 0\n\tdarkPixels := 0\n\n\tbounds := img.Bounds()\n\tfor y := bounds.Min.Y; y < bounds.Max.Y; y++ {\n\t\tfor x := bounds.Min.X; x < bounds.Max.X; x++ {\n\t\t\tr, g, b, _ := img.At(x, y).RGBA()\n\t\t\t// Check if the pixel is dark\n\t\t\tif int(r) < int(threshold) && int(g) < int(threshold) && int(b) < int(threshold) {\n\t\t\t\tdarkPixels++\n\t\t\t}\n\t\t\ttotalPixels++\n\t\t}\n\t}\n\t// Adjust this ratio as needed. For now, it considers the image as \"almost black\" if 90% of the pixels are dark.\n\treturn float64(darkPixels)/float64(totalPixels) > 0.99\n}\n\n// Processes the image\nfunc processImage(filename string) {\n\tfile, err := os.Open(filename)\n\tif err != nil {\n\t\treturn\n\t}\n\tdefer file.Close()\n\n\timg, _, err := image.Decode(file)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tif isAlmostBlack(img) {\n\t\terr := os.Remove(filename)\n        if err!= nil {\n            return\n        }\n\t}\n}\n\nfunc main() {\n\tdir := \"/data/IMAGE_ID/\" // Change this to your directory\n\n\tfiles, err := ioutil.ReadDir(dir)\n\tif err != nil {\n\t\tfmt.Println(\"Error reading directory:\", err)\n\t\treturn\n\t}\n\n\tvar wg sync.WaitGroup\n\tfor _, f := range files {\n\t\tif f.IsDir() {\n\t\t\tcontinue\n\t\t}\n\n\t\text := filepath.Ext(f.Name())\n\t\tif ext == \".jpg\" || ext == \".png\" { // You can add more extensions if needed\n\t\t\twg.Add(1)\n\t\t\tgo func(filename string) {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tprocessImage(filename)\n\t\t\t}(filepath.Join(dir, f.Name()))\n\t\t}\n\t}\n\twg.Wait()\n}\n    \"\"\"\n    file_string = (file_string.replace(\"IMAGE_ID\", str(image_id)))\n    with open(f\"remover_{image_id}.go\", 'w') as f:\n        f.write(file_string)\n    return (f\"tiler_{image_id}.go\", f\"remover_{image_id}.go\")","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:12.067372Z","iopub.execute_input":"2023-10-09T20:45:12.06782Z","iopub.status.idle":"2023-10-09T20:45:12.078629Z","shell.execute_reply.started":"2023-10-09T20:45:12.067788Z","shell.execute_reply":"2023-10-09T20:45:12.077405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!go mod init tiler\n!go get github.com/nfnt/resize","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:12.080314Z","iopub.execute_input":"2023-10-09T20:45:12.080948Z","iopub.status.idle":"2023-10-09T20:45:14.702428Z","shell.execute_reply.started":"2023-10-09T20:45:12.080901Z","shell.execute_reply":"2023-10-09T20:45:14.700991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir /data","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:14.70583Z","iopub.execute_input":"2023-10-09T20:45:14.706203Z","iopub.status.idle":"2023-10-09T20:45:15.821543Z","shell.execute_reply.started":"2023-10-09T20:45:14.706174Z","shell.execute_reply":"2023-10-09T20:45:15.81989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%env GOMAXPROCS=4","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:15.823595Z","iopub.execute_input":"2023-10-09T20:45:15.824268Z","iopub.status.idle":"2023-10-09T20:45:15.831993Z","shell.execute_reply.started":"2023-10-09T20:45:15.824215Z","shell.execute_reply":"2023-10-09T20:45:15.830821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimage_ids = pd.read_csv(\"/kaggle/input/UBC-OCEAN/train.csv\")['image_id'].tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:15.834095Z","iopub.execute_input":"2023-10-09T20:45:15.834486Z","iopub.status.idle":"2023-10-09T20:45:16.317567Z","shell.execute_reply.started":"2023-10-09T20:45:15.834455Z","shell.execute_reply":"2023-10-09T20:45:16.31618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.auto import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:16.320041Z","iopub.execute_input":"2023-10-09T20:45:16.320842Z","iopub.status.idle":"2023-10-09T20:45:16.433296Z","shell.execute_reply.started":"2023-10-09T20:45:16.320797Z","shell.execute_reply":"2023-10-09T20:45:16.432261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /data","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:16.435607Z","iopub.execute_input":"2023-10-09T20:45:16.436563Z","iopub.status.idle":"2023-10-09T20:45:17.729646Z","shell.execute_reply.started":"2023-10-09T20:45:16.43652Z","shell.execute_reply":"2023-10-09T20:45:17.728393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 256\nfor image_id in tqdm(image_ids[250:325]):\n    tiler, remover = make_file(image_id, 256)\n    !mkdir /data/$image_id\n    !go run $tiler\n    !go run $remover\n    zip_path = f\"/kaggle/working/256_{image_id}.zip\"\n    !cd /data/$image_id && zip -rq $zip_path .\n    !rm -rf /data/$image_id","metadata":{"execution":{"iopub.status.busy":"2023-10-09T20:45:17.731649Z","iopub.execute_input":"2023-10-09T20:45:17.732057Z","iopub.status.idle":"2023-10-09T20:51:09.269014Z","shell.execute_reply.started":"2023-10-09T20:45:17.732025Z","shell.execute_reply":"2023-10-09T20:51:09.266839Z"},"trusted":true},"execution_count":null,"outputs":[]}]}