{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!apt update -y\n!apt install -y libvips libvips-tools\n!pip install pyvips","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-31T10:55:25.025242Z","iopub.execute_input":"2023-12-31T10:55:25.02567Z","iopub.status.idle":"2023-12-31T10:56:20.451333Z","shell.execute_reply.started":"2023-12-31T10:55:25.025637Z","shell.execute_reply":"2023-12-31T10:56:20.45005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport cv2\nimport glob\nimport zipfile\nimport time\nimport threading\nimport pyvips\nimport random as rn\nfrom tqdm import tqdm\nfrom PIL import Image\n# from threading import Thread\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport os  \nfrom joblib import Parallel, delayed\n# 设置环境变量TMPDIR为/kaggle/output  \n# os.environ['TMPDIR'] = '/kaggle/working'","metadata":{"execution":{"iopub.status.busy":"2023-12-31T10:56:20.453517Z","iopub.execute_input":"2023-12-31T10:56:20.454437Z","iopub.status.idle":"2023-12-31T10:56:35.074528Z","shell.execute_reply.started":"2023-12-31T10:56:20.454382Z","shell.execute_reply":"2023-12-31T10:56:35.073595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.exists('/kaggle/working/tmp'): os.mkdir('/kaggle/working/tmp') \nos.environ['TMPDIR'] = '/kaggle/working/tmp' \n!export TMPDIR='/kaggle/working/tmp' \n!export VIPS_CONCURRENCY=1 \npyvips.cache_set_max(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv')\ndf['num_pix'] = df.apply(lambda x: x.image_width*x.image_height, axis=1)\ndf['longest_edge'] = df.apply(lambda x: max(x.image_width, x.image_height), axis=1)\ndf = df.sort_values(by=['num_pix'])\ndf = df[-100:].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-31T10:56:35.078379Z","iopub.execute_input":"2023-12-31T10:56:35.079422Z","iopub.status.idle":"2023-12-31T10:56:35.152152Z","shell.execute_reply.started":"2023-12-31T10:56:35.079368Z","shell.execute_reply":"2023-12-31T10:56:35.151265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_pyvips(f, longest_edge):\n    img = pyvips.Image.thumbnail(f, int(longest_edge))\n\n    img = np.ndarray(\n            buffer=img.write_to_memory(),\n            dtype=np.uint8,\n            shape=[img.height, img.width, img.bands]\n        )[..., :3]\n    \n    del img\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-31T10:58:28.503235Z","iopub.execute_input":"2023-12-31T10:58:28.503893Z","iopub.status.idle":"2023-12-31T10:58:28.514881Z","shell.execute_reply.started":"2023-12-31T10:58:28.503847Z","shell.execute_reply":"2023-12-31T10:58:28.512881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_= Parallel(n_jobs=2)(\n    delayed(load_pyvips)(f'/kaggle/input/UBC-OCEAN/train_images/{row.image_id}.png', row.longest_edge)\n    for i, row in tqdm(df.iterrows(), total=len(df))\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-31T10:58:30.809569Z","iopub.execute_input":"2023-12-31T10:58:30.81016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# chunk_idx = 0\n# fs_cell = fs[chunk_idx*chunk:(chunk_idx+1)*chunk]\n# for i, f in tqdm(enumerate(fs_cell)):\n#     img = pyvips.Image.new_from_file(f).numpy()\n#     del img\n#     gc.collect()\n#     print('**********************************')","metadata":{"execution":{"iopub.status.busy":"2023-12-31T10:58:09.882359Z","iopub.status.idle":"2023-12-31T10:58:09.882849Z","shell.execute_reply.started":"2023-12-31T10:58:09.882631Z","shell.execute_reply":"2023-12-31T10:58:09.882653Z"},"trusted":true},"execution_count":null,"outputs":[]}]}