{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Code that uses pyvips to read the very large png and convert it to a tif / tiff file, which is a pyramidal image file for fast efficient viewing, especially with whole slide image software such as QuPath.\n# With QuPath, the tif file can be annotated, to produce a mask for efficient tiling or handling of tiles to train or inference with.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-13T00:41:27.455889Z","iopub.execute_input":"2023-10-13T00:41:27.456266Z","iopub.status.idle":"2023-10-13T00:41:28.087089Z","shell.execute_reply.started":"2023-10-13T00:41:27.456238Z","shell.execute_reply":"2023-10-13T00:41:28.085655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install libvips with apt since pyvips needs it","metadata":{}},{"cell_type":"code","source":"!sudo apt-get update\n!sudo apt-get install -y libvips42","metadata":{"scrolled":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-13T00:41:35.346759Z","iopub.execute_input":"2023-10-13T00:41:35.347295Z","iopub.status.idle":"2023-10-13T00:42:14.344351Z","shell.execute_reply.started":"2023-10-13T00:41:35.347265Z","shell.execute_reply":"2023-10-13T00:42:14.342786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pyvips","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-13T00:42:22.536516Z","iopub.execute_input":"2023-10-13T00:42:22.536917Z","iopub.status.idle":"2023-10-13T00:42:39.104962Z","shell.execute_reply.started":"2023-10-13T00:42:22.536879Z","shell.execute_reply":"2023-10-13T00:42:39.103116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Select png training images to be converted.","metadata":{}},{"cell_type":"code","source":"ROOT_DIR = '/kaggle/input/UBC-OCEAN'\nTRAIN_DIR = '/kaggle/input/UBC-OCEAN/train_images'\n\ndf = pd.read_csv(f\"{ROOT_DIR}/train.csv\")\n# print(df.head())\n\n# make a list of images to be converted according to your criteria\n\n# here I only chose 2 specific ones.\n# png_list = [f\"{i}.png\" for i in [37190, 54506]]\n\n# here I chose all the TMA images\ntma_df = df.loc[df['is_tma']]\n# print (tma_df.head())\n\n# create a list of image_id from the filtered rows\nimage_id_list = tma_df['image_id'].tolist()\n# print(image_id_list)\n\npng_list = [f\"{i}.png\" for i in image_id_list]\n\nprint(png_list)\nprint(\"Number of images: \",len(png_list))","metadata":{"execution":{"iopub.status.busy":"2023-10-13T01:02:10.945508Z","iopub.execute_input":"2023-10-13T01:02:10.945907Z","iopub.status.idle":"2023-10-13T01:02:10.95867Z","shell.execute_reply.started":"2023-10-13T01:02:10.945877Z","shell.execute_reply":"2023-10-13T01:02:10.957399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convert the specified png training images into a pyramidal image file format such as TIF / TIFF using pyvips\nChange the quality of the tiles by changing the 'Q=' value. Values below 20 look pixelated.\nFile(s) will be in the output directory where they can be downloaded, and properly annotated to make appropriate masks to improve the input data quality.","metadata":{}},{"cell_type":"code","source":"import pyvips\n\n# loop through the png_list\nfor png_file in png_list:\n    # read a png file from the directory\n    image = pyvips.Image.new_from_file(f\"{TRAIN_DIR}/{png_file}\")\n    \n    # write to a tiff file with some options\n    image.tiffsave(f\"{png_file[:-4]}.tif\", compression=\"jpeg\", Q=95, tile=True, pyramid=True)\n\n# output for all the TMA images at Q=95 was 313.5 MB\n","metadata":{"execution":{"iopub.status.busy":"2023-10-13T01:08:25.537314Z","iopub.execute_input":"2023-10-13T01:08:25.537757Z","iopub.status.idle":"2023-10-13T01:08:48.559217Z","shell.execute_reply.started":"2023-10-13T01:08:25.537726Z","shell.execute_reply":"2023-10-13T01:08:48.557224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Zip the files for easier downloading","metadata":{}},{"cell_type":"code","source":"# Import the zipfile module\nimport zipfile\n\n# Create a ZipFile object with write mode\nwith zipfile.ZipFile('tif_files.zip', 'w') as zip:\n\n  # Loop through the files in the directory\n  for file in os.listdir('/kaggle/working'):\n\n    # Check if the file has the .tif extension\n    if file.endswith('.tif'):\n\n      # Add the file to the zip archive\n      zip.write(file)","metadata":{"execution":{"iopub.status.busy":"2023-10-13T01:18:04.373978Z","iopub.execute_input":"2023-10-13T01:18:04.374534Z","iopub.status.idle":"2023-10-13T01:18:05.578163Z","shell.execute_reply.started":"2023-10-13T01:18:04.374497Z","shell.execute_reply":"2023-10-13T01:18:05.576998Z"},"trusted":true},"execution_count":null,"outputs":[]}]}