{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## As displayed in my [previous notebook](https://www.kaggle.com/code/rainfalllove/big-difference-between-tma-and-wsi-thumbnail/notebook), using thumbnails for training is not a good way, here I decide to split each WSI into several \"TMAs\"(simulate), and these \"TMAs\" are used for training.\n\nFOR THE REASON:\n\n    WSI:20X\n\n    TMA:40X\n    \nUsing (average_tma_width//2,average_tma_height//2) as our TMA(cut form WSI) size, and then resize to (average_width,average_height)\n\n## Goal:\n```\nfor each WSI:\n\n    WSI--> TMAs\n```","metadata":{}},{"cell_type":"markdown","source":"# 1. Select WSIs for processing","metadata":{}},{"cell_type":"code","source":"import os\n\ntrain_images_folder = '/kaggle/input/UBC-OCEAN/train_images'\ntrain_thumbnails_folder = '/kaggle/input/UBC-OCEAN/train_thumbnails'\n\nimages_files = set(os.listdir(train_images_folder))\nthumbnails_files = set(os.listdir(train_thumbnails_folder))\n\nimages_filenames = set([filename.split('.')[0] for filename in images_files])\n\nthumbnails_filenames = set([filename.split('_')[0] for filename in thumbnails_files])\n\nmissing_thumbnails = images_filenames - thumbnails_filenames\nmissing_thumbnails=[name+'.png' for name in missing_thumbnails]\n\nprint(\"TMA list:\")\nfor file_name in missing_thumbnails:\n    print(file_name)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:34.276842Z","iopub.execute_input":"2023-10-24T10:15:34.27725Z","iopub.status.idle":"2023-10-24T10:15:34.425151Z","shell.execute_reply.started":"2023-10-24T10:15:34.277216Z","shell.execute_reply":"2023-10-24T10:15:34.423379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf=pd.read_csv('/kaggle/input/UBC-OCEAN/train.csv')\nis_tma_df=df[df['is_tma']]\nis_tma_df","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:35.440079Z","iopub.execute_input":"2023-10-24T10:15:35.44118Z","iopub.status.idle":"2023-10-24T10:15:35.898335Z","shell.execute_reply.started":"2023-10-24T10:15:35.441136Z","shell.execute_reply":"2023-10-24T10:15:35.897143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using (average_width//2,average_height//2) as our TMA(cut form WSI) size, and then resize to (average_width,average_height)","metadata":{}},{"cell_type":"code","source":"avg_width=int(is_tma_df['image_width'].mean())\navg_height=int(is_tma_df['image_height'].mean())\nprint(avg_width,avg_height)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:36.152614Z","iopub.execute_input":"2023-10-24T10:15:36.153344Z","iopub.status.idle":"2023-10-24T10:15:36.161224Z","shell.execute_reply.started":"2023-10-24T10:15:36.153295Z","shell.execute_reply":"2023-10-24T10:15:36.159827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"is_not_tma_df=df[df['is_tma']==False]\nis_not_tma_df","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:36.334961Z","iopub.execute_input":"2023-10-24T10:15:36.335348Z","iopub.status.idle":"2023-10-24T10:15:36.359633Z","shell.execute_reply.started":"2023-10-24T10:15:36.335318Z","shell.execute_reply":"2023-10-24T10:15:36.358492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. WSI-->TMAs","metadata":{}},{"cell_type":"code","source":"def wsi2tmas(wsi_path, label, save_dir, overlap=None, non_bg_ratio=1.0, width=avg_width//2, height=avg_height//2):\n    wsi_name = wsi_path.split('/')[-1].split('.')[0]\n\n    wsi_image = Image.open(wsi_path)\n    wsi_width, wsi_height = wsi_image.size\n\n    if overlap is not None:\n        num_tmas_width = wsi_width // (width - overlap)\n        num_tmas_height = wsi_height // (height - overlap)\n    else:\n        num_tmas_width = wsi_width // width\n        num_tmas_height = wsi_height // height\n\n    for i in range(num_tmas_width):\n        for j in range(num_tmas_height):\n            \n            left = i * (width - overlap) if overlap is not None else i * width\n            upper = j * (height - overlap) if overlap is not None else j * height\n            right = min((i + 1) * (width - overlap) + overlap, wsi_width) if overlap is not None else (i + 1) * width\n            lower = min((j + 1) * (height - overlap) + overlap, wsi_height) if overlap is not None else (j + 1) * height\n\n            tma_image = wsi_image.crop((left, upper, right, lower))\n\n            total_pixels = tma_image.size[0] * tma_image.size[1]\n            non_bg_pixels = sum(1 for pixel in tma_image.getdata() if pixel != (0, 0, 0))\n\n            if non_bg_pixels / total_pixels >= non_bg_ratio:\n                # 将图像 resize 到目标尺寸\n#                 tma_image = tma_image.resize((avg_width, avg_height))\n                tma_image.save(os.path.join(save_dir, f'{wsi_name}_{label}_{i}_{j}.jpg'), quality=jpeg_quality)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:37.827186Z","iopub.execute_input":"2023-10-24T10:15:37.827569Z","iopub.status.idle":"2023-10-24T10:15:37.840714Z","shell.execute_reply.started":"2023-10-24T10:15:37.827539Z","shell.execute_reply":"2023-10-24T10:15:37.839657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nImage.MAX_IMAGE_PIXELS = None\njpeg_quality=80\n\nsave_dir='/kaggle/working/images'\nif not os.path.exists(save_dir):\n    os.makedirs(save_dir)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:39.185827Z","iopub.execute_input":"2023-10-24T10:15:39.186229Z","iopub.status.idle":"2023-10-24T10:15:39.194376Z","shell.execute_reply.started":"2023-10-24T10:15:39.186196Z","shell.execute_reply":"2023-10-24T10:15:39.192886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Single WSI sample\n# wsi_path='/kaggle/input/UBC-OCEAN/train_images/10077.png'\n# wsi2tmas(wsi_path,'label',save_dir)\n\n# x=Image.open('/kaggle/working/images/10077_label_7_11.jpg')\n# x","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:40.031319Z","iopub.execute_input":"2023-10-24T10:15:40.03185Z","iopub.status.idle":"2023-10-24T10:15:40.03991Z","shell.execute_reply.started":"2023-10-24T10:15:40.031803Z","shell.execute_reply":"2023-10-24T10:15:40.038424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now process all WSIs\nfor i,row in is_not_tma_df[50:100].iterrows():\n    print(i)\n    image_id=str(row['image_id'])\n    label=row['label']\n    WSI_path=os.path.join('/kaggle/input/UBC-OCEAN/train_images',image_id+'.png')\n    wsi2tmas(WSI_path,label,save_dir)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T10:15:43.249362Z","iopub.execute_input":"2023-10-24T10:15:43.249809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export all images into a zipped file\nimport tarfile\nimport gzip\nimport os\nimport shutil\n\ndef pack_images_to_tar_gz(input_folder, output_tar_gz):\n    with tarfile.open(output_tar_gz, 'w:gz') as tar_gz:\n        for root, dirs, files in os.walk(input_folder):\n            for file in files:\n                file_path = os.path.join(root, file)\n                arcname = os.path.relpath(file_path, input_folder)\n                tar_gz.add(file_path, arcname=arcname)\n\ninput_folder = save_dir\noutput_tar_gz =os.path.join('/kaggle/working/','output_images.tar.gz') \n\npack_images_to_tar_gz(input_folder, output_tar_gz)\n\nshutil.rmtree(input_folder)","metadata":{"execution":{"iopub.status.busy":"2023-10-24T02:58:12.174179Z","iopub.execute_input":"2023-10-24T02:58:12.174617Z","iopub.status.idle":"2023-10-24T02:58:17.476787Z","shell.execute_reply.started":"2023-10-24T02:58:12.174574Z","shell.execute_reply":"2023-10-24T02:58:17.475557Z"},"trusted":true},"execution_count":null,"outputs":[]}]}