{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":18647,"databundleVersionId":1126921,"sourceType":"competition"}],"dockerImageVersionId":29926,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport skimage.io\nimport cv2\nimport zipfile\nfrom tqdm import tqdm\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nimport tempfile\nimport shutil\n\ntile_size = 256\nn_tiles = 14\ntile_mode = 0\n\ndef get_tiles(img, tile_size, n_tiles, mode=0):\n    \"\"\"\n    Generate tiles from an image.\n    \"\"\"\n    result = []\n    h, w, c = img.shape\n    pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n    pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n\n    img_padded = np.pad(img, [[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2, pad_w - pad_w // 2], [0, 0]], constant_values=255)\n    img_reshaped = img_padded.reshape(\n        img_padded.shape[0] // tile_size,\n        tile_size,\n        img_padded.shape[1] // tile_size,\n        tile_size,\n        c\n    )\n\n    img_reshaped = img_reshaped.transpose(0, 2, 1, 3, 4).reshape(-1, tile_size, tile_size, c)\n    if len(img_reshaped) < n_tiles:\n        img_reshaped = np.pad(img_reshaped, [[0, n_tiles - len(img_reshaped)], [0, 0], [0, 0], [0, 0]], constant_values=255)\n    idxs = np.argsort(img_reshaped.reshape(img_reshaped.shape[0], -1).sum(-1))[:n_tiles]\n    img_reshaped = img_reshaped[idxs]\n\n    for i in range(len(img_reshaped)):\n        result.append({'img': img_reshaped[i], 'idx': i})\n    return result\n        \ndef save_tiles_to_zip(tiles, img_id, zip_path=\"train_patches.zip\"):\n    \"\"\"\n    Save the tiles as .png files directly into a ZIP file.\n    \"\"\"\n    with zipfile.ZipFile(zip_path, 'a') as zipf:  # Open the ZIP file in append mode\n        for tile in tiles:\n            idx = tile['idx']\n            img = tile['img']\n            filename = f\"{img_id}_{idx}.png\"  # Naming format to ensure uniqueness\n            # Convert the image to PNG format in memory\n            _, img_encoded = cv2.imencode('.png', cv2.cvtColor(img, cv2.COLOR_RGB2BGR))\n            # Save the encoded image directly into the ZIP file\n            zipf.writestr(filename, img_encoded.tobytes())\n\ndef process_image_and_save_tiles(img_id, tiff_file):\n    \"\"\"\n    Process an image to generate tiles and save them to a temporary directory.\n    Returns the path to the temporary directory containing the tiles.\n    \"\"\"\n    image = skimage.io.MultiImage(tiff_file)[1]  # Load the image at the appropriate level\n    tiles = get_tiles(image, tile_size, n_tiles, tile_mode)\n    temp_dir = tempfile.mkdtemp()  # Create a temporary directory for this image's tiles\n    \n    stats = []  # To store statistics (mean, std) for each tile\n    for tile in tiles:\n        idx = tile['idx']\n        img = tile['img']\n        filename = f\"{img_id}_{idx}.png\"\n        cv2.imwrite(os.path.join(temp_dir, filename), cv2.cvtColor(img, cv2.COLOR_RGB2BGR))\n        \n        # Calculate and store channel-wise statistics\n        tile_mean = np.mean(img, axis=(0, 1))\n        tile_var = np.var(img, axis=(0, 1))\n        stats.append((tile_mean, tile_var))\n           \n    return temp_dir, stats\n\ndef aggregate_tiles_into_zip(temp_dirs, zip_path=\"train_patches.zip\"):\n    \"\"\"\n    Aggregate tiles from temporary directories into a single ZIP file.\n    \"\"\"\n    with zipfile.ZipFile(zip_path, 'w') as zipf:\n        for temp_dir in temp_dirs:\n            for filename in os.listdir(temp_dir):\n                file_path = os.path.join(temp_dir, filename)\n                zipf.write(file_path, filename)\n            shutil.rmtree(temp_dir)  # Clean up the temporary directory\n\nif __name__ == \"__main__\":\n    data_dir = '../input/prostate-cancer-grade-assessment'\n    df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\n    image_folder = os.path.join(data_dir, 'train_images')\n\n    temp_dirs = []  # List to collect temporary directories\n    all_stats = []  # To collect statistics from all tiles\n\n    with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:\n        futures = [executor.submit(process_image_and_save_tiles, row['image_id'], os.path.join(image_folder, f'{row[\"image_id\"]}.tiff')) for index, row in df_train.iterrows()]\n        \n        for future in tqdm(as_completed(futures), total=len(futures)):\n            temp_dir, stats = future.result()\n            temp_dirs.append(temp_dir)  # Collect the path to the temporary directory\n            all_stats.extend(stats)  # Aggregate statistics\n    \n    # After collecting all stats\n    overall_means = np.mean([stat[0] for stat in all_stats], axis=0)\n    # Calculate overall variance per channel\n    overall_vars = np.mean([stat[1] for stat in all_stats], axis=0)\n    # The correct overall std per channel is the sqrt of the overall variance\n    overall_stds = np.sqrt(overall_vars)\n\n    print(f\"Overall Means per Channel: {overall_means}, Overall Stds per Channel: {overall_stds}\")\n\n\n    aggregate_tiles_into_zip(temp_dirs)  \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}