{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":37333,"databundleVersionId":3949526,"sourceType":"competition"}],"dockerImageVersionId":30213,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!sudo apt-get update\n!sudo apt-get -y install libvips-dev\n!pip install pyvips","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.idle":"2022-07-25T21:26:15.966895Z","shell.execute_reply.started":"2022-07-25T21:23:42.696287Z","shell.execute_reply":"2022-07-25T21:26:15.965187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, rasterio, cv2, asyncio, warnings, pyvips\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom typing import Union","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-25T21:26:15.970466Z","iopub.execute_input":"2022-07-25T21:26:15.970886Z","iopub.status.idle":"2022-07-25T21:26:18.294245Z","shell.execute_reply.started":"2022-07-25T21:26:15.970852Z","shell.execute_reply":"2022-07-25T21:26:18.29327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = '../input/mayo-clinic-strip-ai/train.csv'\n\ntrain_df = pd.read_csv(train_path)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:26:18.295441Z","iopub.execute_input":"2022-07-25T21:26:18.296299Z","iopub.status.idle":"2022-07-25T21:26:18.352834Z","shell.execute_reply.started":"2022-07-25T21:26:18.296266Z","shell.execute_reply":"2022-07-25T21:26:18.351979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x='label', data=train_df)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:26:18.354935Z","iopub.execute_input":"2022-07-25T21:26:18.355505Z","iopub.status.idle":"2022-07-25T21:26:18.576705Z","shell.execute_reply.started":"2022-07-25T21:26:18.355471Z","shell.execute_reply":"2022-07-25T21:26:18.575258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_scale_write_image(image_path:str, write_dir:str='./output_images', scale:float=0.09):\n    warnings.filterwarnings(\"ignore\")\n    image_id = image_path.split('/')[-1].replace('.tif', '')\n    image = pyvips.Image.new_from_file(image_path).resize(scale)\n    \n    if not os.path.isdir(write_dir):os.mkdir(write_dir)\n    file_path = os.path.join(write_dir, f'{image_id}.png')\n    #cv2.imwrite(file_path, image)\n    image.write_to_file(file_path)\n    print(f'image: {file_path} downsampled and saved')\n\n\nasync def process_image(\n    image_path:str, semaphore:Union[asyncio.Semaphore, asyncio.BoundedSemaphore]):\n    loop = asyncio.get_running_loop()\n    \n    async with semaphore:\n        await loop.run_in_executor(None, read_scale_write_image, image_path)\n\n\nasync def main(paths:list, limit_concurrency:int=10):\n    semaphore = asyncio.Semaphore(value=limit_concurrency)\n    tasks = []\n    \n    for i in paths:\n        tasks.append(asyncio.ensure_future(process_image(i, semaphore)))\n    await asyncio.gather(*tasks)\n\n    \ntrain_image_path = '../input/mayo-clinic-strip-ai/train'\nfiles = os.listdir(train_image_path)\npaths = [os.path.join(train_image_path, i) for i in files]\nawait main(paths)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:26:18.702371Z","iopub.status.idle":"2022-07-25T21:26:18.702984Z","shell.execute_reply.started":"2022-07-25T21:26:18.702666Z","shell.execute_reply":"2022-07-25T21:26:18.702696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-25T21:26:18.705128Z","iopub.status.idle":"2022-07-25T21:26:18.705699Z","shell.execute_reply.started":"2022-07-25T21:26:18.70541Z","shell.execute_reply":"2022-07-25T21:26:18.705437Z"},"trusted":true},"execution_count":null,"outputs":[]}]}