{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Config","metadata":{"id":"2cfc5d52","papermill":{"duration":0.015884,"end_time":"2023-02-25T23:54:33.789992","exception":false,"start_time":"2023-02-25T23:54:33.774108","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## from Kaggle  notebook to Colab","metadata":{"id":"508f9924","papermill":{"duration":0.012269,"end_time":"2023-02-25T23:54:33.814823","exception":false,"start_time":"2023-02-25T23:54:33.802554","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import sys,os,json\nfrom os.path import join as j\n\n\n# Common\nIS_KAGGLE = False\ntry:\n    !pip install kaggle\n    from google.colab import drive\n\n    drive.mount(\"/content/drive\")\n    # Kaggle API\n    kaggle_json_path=\"/content/drive/MyDrive/Kaichis-folder/api/kaggle.json\"\n    f = open(kaggle_json_path, \"r\")\n    json_data = json.load(f)\n    os.environ[\"KAGGLE_USERNAME\"] = json_data[\"username\"]\n    os.environ[\"KAGGLE_KEY\"] = json_data[\"key\"]\nexcept:\n    IS_KAGGLE = True\n    print(\"You are running in Kaggle notebook.\")\n    sys.path.append(\"../input/timm-pytorch-image-models/pytorch-image-models-master\")\n\nos.environ[\"WANDB_MODE\"] = \"online\"\nif os.environ[\"WANDB_MODE\"] == \"online\":\n    if IS_KAGGLE:\n        from kaggle_secrets import UserSecretsClient\n\n        \nINPUT_DIR = '/kaggle/input'\n\nif not IS_KAGGLE:\n    print(\"You are running in Colab.\")\n    INPUT_DIR = '/content/drive/MyDrive/Dataset'\n    sys.path.append(\"/content/drive/MyDrive/Dataset/timm-pytorch-image-models/\")\n    \n\n","metadata":{"id":"48a2d516","outputId":"a007e199-caaa-4145-83da-ab244fc8fc2a","papermill":{"duration":30.582623,"end_time":"2023-02-25T23:55:04.409749","exception":false,"start_time":"2023-02-25T23:54:33.827126","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:33:30.520441Z","iopub.execute_input":"2023-02-26T08:33:30.520881Z","iopub.status.idle":"2023-02-26T08:34:00.269271Z","shell.execute_reply.started":"2023-02-26T08:33:30.520772Z","shell.execute_reply":"2023-02-26T08:34:00.267925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\n\ndef download_kaggle_ds(user_name_slash_ds_name, input_dir):\n    \"\"\"\n    ## An Example of Use\n    `download_kaggle_ds('garudakai/rsna-convnext-5models-stkfold',INPUT_DIR)`\n    \"\"\"\n    assert \"/\" in user_name_slash_ds_name\n    ds_name = user_name_slash_ds_name.split(\"/\")[-1]\n    ds_dir = j(input_dir, ds_name)\n    os.makedirs(ds_dir, exist_ok=True)\n\n    !kaggle datasets download -d {user_name_slash_ds_name} -p {ds_dir}\n\n    zips = glob.glob(j(ds_dir, \"*.zip\"))\n    for zip_path in zips:\n        !unzip {zip_path} -d {ds_dir}\n        !rm {zip_path}\n    %ls {ds_dir}","metadata":{"id":"1963babd","papermill":{"duration":0.036606,"end_time":"2023-02-25T23:55:04.460215","exception":false,"start_time":"2023-02-25T23:55:04.423609","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:34:00.272471Z","iopub.execute_input":"2023-02-26T08:34:00.272798Z","iopub.status.idle":"2023-02-26T08:34:00.286687Z","shell.execute_reply.started":"2023-02-26T08:34:00.272769Z","shell.execute_reply":"2023-02-26T08:34:00.285058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Install Keras CV Attention Model Pip Package for ConvNextV2 Models\n!pip install --no-deps {j(INPUT_DIR,'keras-cv-attention-models/keras_cv_attention_models-1.3.9-py3-none-any.whl')}","metadata":{"id":"c8e34f4d","outputId":"ad79c16d-7ce1-4f35-a0cd-cac28fb167c8","papermill":{"duration":22.358303,"end_time":"2023-02-25T23:55:26.831969","exception":false,"start_time":"2023-02-25T23:55:04.473666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:34:00.287885Z","iopub.execute_input":"2023-02-26T08:34:00.28817Z","iopub.status.idle":"2023-02-26T08:34:22.337574Z","shell.execute_reply.started":"2023-02-26T08:34:00.288144Z","shell.execute_reply":"2023-02-26T08:34:22.336352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q timm==0.6.5 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}\n!pip install -q albumentations==1.2.1 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}\n!pip install -q pylibjpeg-libjpeg==1.3.1 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}\n!pip install -q pydicom==2.0.0 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}\n!pip install -q python-gdcm==3.0.20 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}\n!pip install -q dicomsdl==0.109.1 --no-index --find-links={j(INPUT_DIR,'rsna-bc-pip-requirements')}","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"22374683","outputId":"897879be-c75a-433a-a281-3b4424aa8182","papermill":{"duration":61.452299,"end_time":"2023-02-25T23:56:28.297485","exception":false,"start_time":"2023-02-25T23:55:26.845186","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:34:22.341172Z","iopub.execute_input":"2023-02-26T08:34:22.341596Z","iopub.status.idle":"2023-02-26T08:35:18.700573Z","shell.execute_reply.started":"2023-02-26T08:34:22.341555Z","shell.execute_reply":"2023-02-26T08:35:18.699375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we install the latest DALI packaging which we will use for GPU decoding","metadata":{"id":"33032720","papermill":{"duration":0.012938,"end_time":"2023-02-25T23:56:28.323889","exception":false,"start_time":"2023-02-25T23:56:28.310951","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q {j(INPUT_DIR,'nvidia-dali-nightly-cuda110-1230dev/nvidia_dali_nightly_cuda110-1.23.0.dev20230203-7187866-py3-none-manylinux2014_x86_64.whl')}","metadata":{"id":"702b09da","papermill":{"duration":41.537998,"end_time":"2023-02-25T23:57:09.875076","exception":false,"start_time":"2023-02-25T23:56:28.337078","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:18.702372Z","iopub.execute_input":"2023-02-26T08:35:18.705899Z","iopub.status.idle":"2023-02-26T08:35:48.145089Z","shell.execute_reply.started":"2023-02-26T08:35:18.705866Z","shell.execute_reply":"2023-02-26T08:35:48.143849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we import all the packages we need and patch a function to allow for INT16 support","metadata":{"id":"d41acf5e","papermill":{"duration":0.013399,"end_time":"2023-02-25T23:57:09.90208","exception":false,"start_time":"2023-02-25T23:57:09.888681","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import timm\nimport numpy as np  # linear algebra\nimport pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom copy import copy\nimport gc\nimport shutil\n\nimport glob\nfrom scipy.special import expit\n\nimport albumentations as A\nimport cv2\n\ncv2.setNumThreads(0)\n\nif IS_KAGGLE:\n    import dicomsdl\nimport pydicom\nfrom pydicom.filebase import DicomBytesIO\n\nfrom os.path import join\n\nfrom tqdm import tqdm\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\n\nfrom types import SimpleNamespace\nfrom typing import Any, Dict\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import GradScaler, autocast\n\n\nimport nvidia.dali.fn as fn\nimport nvidia.dali.types as types\nfrom nvidia.dali import pipeline_def\nfrom nvidia.dali.types import DALIDataType","metadata":{"id":"7355d5ed","papermill":{"duration":5.043879,"end_time":"2023-02-25T23:57:14.959378","exception":false,"start_time":"2023-02-25T23:57:09.915499","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:48.146958Z","iopub.execute_input":"2023-02-26T08:35:48.147363Z","iopub.status.idle":"2023-02-26T08:35:50.351901Z","shell.execute_reply.started":"2023-02-26T08:35:48.147324Z","shell.execute_reply":"2023-02-26T08:35:50.350709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we need to patch DALI for Int16 support\n\n\nfrom nvidia.dali.backend import TensorGPU, TensorListGPU\nfrom nvidia.dali.pipeline import Pipeline\nimport nvidia.dali.ops as ops\nfrom nvidia.dali import types\nfrom nvidia.dali.plugin.base_iterator import _DaliBaseIterator\nfrom nvidia.dali.plugin.base_iterator import LastBatchPolicy\nimport torch\nimport torch.utils.dlpack as torch_dlpack\nimport ctypes\nimport numpy as np\nimport torch.nn.functional as F\nimport pydicom\n\nto_torch_type = {\n    types.DALIDataType.FLOAT: torch.float32,\n    types.DALIDataType.FLOAT64: torch.float64,\n    types.DALIDataType.FLOAT16: torch.float16,\n    types.DALIDataType.UINT8: torch.uint8,\n    types.DALIDataType.INT8: torch.int8,\n    types.DALIDataType.UINT16: torch.int16,\n    types.DALIDataType.INT16: torch.int16,\n    types.DALIDataType.INT32: torch.int32,\n    types.DALIDataType.INT64: torch.int64,\n}\n\n\ndef feed_ndarray(dali_tensor, arr, cuda_stream=None):\n    \"\"\"\n    Copy contents of DALI tensor to PyTorch's Tensor.\n\n    Parameters\n    ----------\n    `dali_tensor` : nvidia.dali.backend.TensorCPU or nvidia.dali.backend.TensorGPU\n                    Tensor from which to copy\n    `arr` : torch.Tensor\n            Destination of the copy\n    `cuda_stream` : torch.cuda.Stream, cudaStream_t or any value that can be cast to cudaStream_t.\n                    CUDA stream to be used for the copy\n                    (if not provided, an internal user stream will be selected)\n                    In most cases, using pytorch's current stream is expected (for example,\n                    if we are copying to a tensor allocated with torch.zeros(...))\n    \"\"\"\n    dali_type = to_torch_type[dali_tensor.dtype]\n\n    assert dali_type == arr.dtype, (\n        \"The element type of DALI Tensor/TensorList\"\n        \" doesn't match the element type of the target PyTorch Tensor: \"\n        \"{} vs {}\".format(dali_type, arr.dtype)\n    )\n    assert dali_tensor.shape() == list(\n        arr.size()\n    ), \"Shapes do not match: DALI tensor has size {0}, but PyTorch Tensor has size {1}\".format(\n        dali_tensor.shape(), list(arr.size())\n    )\n    cuda_stream = types._raw_cuda_stream(cuda_stream)\n\n    # turn raw int to a c void pointer\n    c_type_pointer = ctypes.c_void_p(arr.data_ptr())\n    if isinstance(dali_tensor, (TensorGPU, TensorListGPU)):\n        stream = None if cuda_stream is None else ctypes.c_void_p(cuda_stream)\n        dali_tensor.copy_to_external(c_type_pointer, stream, non_blocking=True)\n    else:\n        dali_tensor.copy_to_external(c_type_pointer)\n    return arr","metadata":{"id":"8a414728","papermill":{"duration":0.029626,"end_time":"2023-02-25T23:57:15.002711","exception":false,"start_time":"2023-02-25T23:57:14.973085","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.35387Z","iopub.execute_input":"2023-02-26T08:35:50.354632Z","iopub.status.idle":"2023-02-26T08:35:50.368791Z","shell.execute_reply.started":"2023-02-26T08:35:50.354586Z","shell.execute_reply":"2023-02-26T08:35:50.367825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next I set major variables which handle the public run and the re-run on the hidden test set, and also allow for simulating the size of the hidden test set by setting RAM_CHECK = True","metadata":{"id":"8387cd56","papermill":{"duration":0.012829,"end_time":"2023-02-25T23:57:15.028809","exception":false,"start_time":"2023-02-25T23:57:15.01598","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Read DataFrame\n","metadata":{"id":"e05870ac","papermill":{"duration":0.01319,"end_time":"2023-02-25T23:57:15.055317","exception":false,"start_time":"2023-02-25T23:57:15.042127","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Params\nCROP_IMAGE_1st = False\nCROP_IMAGE_2nd = True\n\nCOMP_FOLDER = j(INPUT_DIR, \"rsna-breast-cancer-detection\")\nDATA_FOLDER = j(COMP_FOLDER, \"test_images\")\n\nsample_submission = pd.read_csv(j(COMP_FOLDER, \"sample_submission.csv\"))\n\nPUBLIC_RUN = len(sample_submission) == 2\n\nN_CORES = mp.cpu_count()\nMIXED_PRECISION = False\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nRAM_CHECK = False\nDEBUG = False\n\ntest_df = pd.read_csv(j(COMP_FOLDER, \"test.csv\"))\ntest_df[\"cancer\"] = 0  # dummy value\n\n\nif PUBLIC_RUN is False:\n    RAM_CHECK = False\n    DEBUG = False\n\nif RAM_CHECK is True:\n    test_df = pd.read_csv(j(COMP_FOLDER, \"train.csv\"))\n    patient_filter = (\n        list(sorted((set(test_df.patient_id.unique()))))[:8000]\n        if IS_KAGGLE\n        else [10042, 10006]\n    )\n    test_df = test_df[test_df.patient_id.isin(patient_filter)]\n    DATA_FOLDER = DATA_FOLDER.replace(\"test\", \"train\")\n\nif DEBUG is True:\n    test_df = test_df.head(100)\n\ntest_df","metadata":{"id":"e7156e2a","outputId":"1eef71c6-7945-4ee4-93dd-182596cdb60d","papermill":{"duration":0.122613,"end_time":"2023-02-25T23:57:15.190993","exception":false,"start_time":"2023-02-25T23:57:15.06838","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.370118Z","iopub.execute_input":"2023-02-26T08:35:50.370725Z","iopub.status.idle":"2023-02-26T08:35:50.658811Z","shell.execute_reply.started":"2023-02-26T08:35:50.370687Z","shell.execute_reply":"2023-02-26T08:35:50.657762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Len df : {len(test_df)}\")\ntest_df[\"patient_id\"].nunique()","metadata":{"id":"a6157191","outputId":"5f53e05e-ffdb-4ef7-b663-ba36418e83f6","papermill":{"duration":0.029355,"end_time":"2023-02-25T23:57:15.234081","exception":false,"start_time":"2023-02-25T23:57:15.204726","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.660345Z","iopub.execute_input":"2023-02-26T08:35:50.660747Z","iopub.status.idle":"2023-02-26T08:35:50.669164Z","shell.execute_reply.started":"2023-02-26T08:35:50.660706Z","shell.execute_reply":"2023-02-26T08:35:50.667778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"fns\"] = (\n    test_df[\"patient_id\"].astype(str) + \"/\" + test_df[\"image_id\"].astype(str) + \".dcm\"\n)","metadata":{"id":"cdd1f8ed","papermill":{"duration":0.023886,"end_time":"2023-02-25T23:57:15.271586","exception":false,"start_time":"2023-02-25T23:57:15.2477","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.676646Z","iopub.execute_input":"2023-02-26T08:35:50.67704Z","iopub.status.idle":"2023-02-26T08:35:50.683227Z","shell.execute_reply.started":"2023-02-26T08:35:50.676996Z","shell.execute_reply":"2023-02-26T08:35:50.682144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Blend notebook \"SE-ResNeXt50 full GPU decoding\" and \"RSNA ConvNextV2 Inference Tensorflow\"\n## \"SE-ResNeXt50 full GPU decoding\" is:\nhttps://www.kaggle.com/code/christofhenkel/se-resnext50-full-gpu-decoding/notebook\n\n## \"RSNA ConvNextV2 Inference Tensorflow\" is:\nhttps://www.kaggle.com/code/markwijkhuizen/rsna-convnextv2-inference-tensorflow/notebook\n\n**Thank you Dieter (@christofhenkel)** for public pretrained model `rsna-seresnext50-5fold`\n\nand\n\n**Thank you Mark Wijkhuizen (@markwijkhuizen)** for public pretrained model `rsna-efficientnetv2-training-tensorflow-tpu-ds`","metadata":{"id":"6b0459d8","papermill":{"duration":0.013383,"end_time":"2023-02-25T23:57:15.29831","exception":false,"start_time":"2023-02-25T23:57:15.284927","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Next, we define the function for GPU-based decoding using DALI and processing the dicom images","metadata":{"id":"09edb173","papermill":{"duration":0.013171,"end_time":"2023-02-25T23:57:15.325188","exception":false,"start_time":"2023-02-25T23:57:15.312017","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode=\"same\")\n\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[: int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n\n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n\n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n\n    if isinstance(debug, np.ndarray):\n        debug[1].imshow(image)\n        debug[1].set_title(\"X Offset\")\n        vv_scale = H / vv.max() * 0.90\n        # Values\n        debug[1].plot(H - vv * vv_scale, c=\"red\", label=\"vv\")\n        # Threshold\n        debug[1].hlines(\n            H - vv_threshold * vv_scale, 0, W - 1, colors=\"orange\", label=\"threshold\"\n        )\n        # Max Value\n        debug[1].scatter(\n            vv_argmax,\n            H - vv[vv_argmax] * vv_scale,\n            c=\"blue\",\n            s=100,\n            label=\"Max\",\n            zorder=np.PINF,\n        )\n        # First Column Below Threshold\n        debug[1].scatter(\n            offset,\n            H - vv[offset] * vv_scale,\n            c=\"purple\",\n            s=100,\n            label=\"Offset\",\n            zorder=np.PINF,\n        )\n        debug[1].set_ylim(H, 0)\n        debug[1].legend()\n        debug[1].axis(\"off\")\n\n    return offset\n\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = (\n        int(image.shape[0] * 0.25)\n        + vv[int(image.shape[0] * 0.25) : int(image.shape[0] * 0.75)].argmax()\n    )\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n\n    if isinstance(debug, np.ndarray):\n        debug[2].imshow(image)\n        debug[2].set_title(\"Y Bottom Offset\")\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[2].plot(vv * vv_scale, np.arange(H), c=\"red\", label=\"vv\")\n        # Threshold\n        debug[2].vlines(\n            vv_threshold * vv_scale, 0, H - 1, colors=\"orange\", label=\"threshold\"\n        )\n        # Max Value\n        debug[2].scatter(\n            vv[vv_argmax] * vv_scale,\n            vv_argmax,\n            c=\"blue\",\n            s=100,\n            label=\"Max\",\n            zorder=np.PINF,\n        )\n        # First Column Below Threshold\n        debug[2].scatter(\n            vv[offset_bottom] * vv_scale,\n            offset_bottom,\n            c=\"purple\",\n            s=100,\n            label=\"Offset\",\n            zorder=np.PINF,\n        )\n        debug[2].set_ylim(H, 0)\n        debug[2].legend()\n        debug[2].axis(\"off\")\n\n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n\n    if isinstance(debug, np.ndarray):\n        debug[3].imshow(image)\n        debug[3].set_title(\"Y Top Offset\")\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[3].plot(vv * vv_scale, np.arange(H), c=\"red\", label=\"vv\")\n        # Threshold\n        debug[3].vlines(\n            vv_threshold * vv_scale, 0, H - 1, colors=\"orange\", label=\"threshold\"\n        )\n        # Max Value\n        debug[3].scatter(\n            vv[vv_argmax] * vv_scale,\n            vv_argmax,\n            c=\"blue\",\n            s=100,\n            label=\"Max\",\n            zorder=np.PINF,\n        )\n        # First Column Below Threshold\n        debug[3].scatter(\n            vv[offset_top] * vv_scale,\n            offset_top,\n            c=\"purple\",\n            s=100,\n            label=\"Offset\",\n            zorder=np.PINF,\n        )\n        debug[2].set_ylim(H, 0)\n        debug[3].legend()\n        debug[3].axis(\"off\")\n\n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:, :x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n\n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n\n    # Crop Image\n    image = image[offset_bottom:offset_top:, :x_offset]\n\n    return image","metadata":{"id":"fe965844","papermill":{"duration":0.042619,"end_time":"2023-02-25T23:57:15.38129","exception":false,"start_time":"2023-02-25T23:57:15.338671","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.684935Z","iopub.execute_input":"2023-02-26T08:35:50.685654Z","iopub.status.idle":"2023-02-26T08:35:50.714142Z","shell.execute_reply.started":"2023-02-26T08:35:50.685618Z","shell.execute_reply":"2023-02-26T08:35:50.712982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_dicom_to_jpg(file, save_folder=\"\"):\n    patient = file.split(\"/\")[-2]\n    image = file.split(\"/\")[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == \"1.2.840.10008.1.2.4.90\":\n        with open(file, \"rb\") as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(\n            b\"\\x00\\x00\\x00\\x0C\"\n        )  # <---- the jpeg2000 header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n    if dcmfile.file_meta.TransferSyntaxUID == \"1.2.840.10008.1.2.4.70\":\n        with open(file, \"rb\") as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(\n            b\"\\xff\\xd8\\xff\\xe0\"\n        )  # <---- the jpeg lossless header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n\n@pipeline_def\ndef jpg_decode_pipeline(jpgfiles):\n    jpegs, _ = fn.readers.file(files=jpgfiles)\n    images = fn.experimental.decoders.image(\n        jpegs, device=\"mixed\", output_type=types.ANY_DATA, dtype=DALIDataType.UINT16\n    )\n    return images\n\n\ndef parse_window_element(elem):\n    if type(elem) == list:\n        return float(elem[0])\n    if type(elem) == str:\n        return float(elem)\n    if type(elem) == float:\n        return elem\n    if type(elem) == pydicom.dataelem.DataElement:\n        try:\n            return float(elem[0])\n        except:\n            return float(elem.value)\n    return None\n\n\ndef linear_window(data, center, width):\n    lower, upper = center - width // 2, center + width // 2\n    data = torch.clamp(data, min=lower, max=upper)\n    return data\n\n\ndef process_dicom(img, dicom):\n    try:\n        invert = getattr(dicom, \"PhotometricInterpretation\", None) == \"MONOCHROME1\"\n    except:\n        invert = False\n\n    center = parse_window_element(dicom[\"WindowCenter\"])\n    width = parse_window_element(dicom[\"WindowWidth\"])\n\n    if (center is not None) & (width is not None):\n        img = linear_window(img, center, width)\n\n    img = (img - img.min()) / (img.max() - img.min())\n    if invert:\n        img = 1 - img\n    return img","metadata":{"id":"247cdd5f","papermill":{"duration":0.029898,"end_time":"2023-02-25T23:57:15.424613","exception":false,"start_time":"2023-02-25T23:57:15.394715","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.715743Z","iopub.execute_input":"2023-02-26T08:35:50.716488Z","iopub.status.idle":"2023-02-26T08:35:50.733563Z","shell.execute_reply.started":"2023-02-26T08:35:50.716447Z","shell.execute_reply":"2023-02-26T08:35:50.732563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = SimpleNamespace(**{})\ncfg.img_size = 1024\ncfg.backbone = \"seresnext50_32x4d\"\ncfg.pretrained = False\ncfg.in_channels = 1\ncfg.classes = [\"cancer\"]\ncfg.batch_size = 8\ncfg.data_folder = \"/tmp/output/\"\ncfg.val_aug = A.CenterCrop(\n    always_apply=False, p=1.0, height=cfg.img_size, width=cfg.img_size\n)\ncfg.device = DEVICE","metadata":{"id":"a499a6b5","papermill":{"duration":0.022444,"end_time":"2023-02-25T23:57:15.460855","exception":false,"start_time":"2023-02-25T23:57:15.438411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.734986Z","iopub.execute_input":"2023-02-26T08:35:50.735443Z","iopub.status.idle":"2023-02-26T08:35:50.746356Z","shell.execute_reply.started":"2023-02-26T08:35:50.735405Z","shell.execute_reply":"2023-02-26T08:35:50.745472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will process the dicoms in chunks so the disk space does not become an issue. ","metadata":{"execution":{"iopub.execute_input":"2023-02-05T07:14:35.534267Z","iopub.status.busy":"2023-02-05T07:14:35.533224Z","iopub.status.idle":"2023-02-05T07:14:35.545681Z","shell.execute_reply":"2023-02-05T07:14:35.544691Z","shell.execute_reply.started":"2023-02-05T07:14:35.534231Z"},"id":"fdf2b0bf","papermill":{"duration":0.013197,"end_time":"2023-02-25T23:57:15.487362","exception":false,"start_time":"2023-02-25T23:57:15.474165","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SAVE_SIZE = int(cfg.img_size * 1.125)\nSAVE_FOLDER = cfg.data_folder\nos.makedirs(SAVE_FOLDER, exist_ok=True)\nN_CHUNKS = len(test_df[\"fns\"]) // 2000 if len(test_df[\"fns\"]) > 2000 else 1\nCHUNKS = [\n    (len(test_df[\"fns\"]) / N_CHUNKS * k, len(test_df[\"fns\"]) / N_CHUNKS * (k + 1))\n    for k in range(N_CHUNKS)\n]\nCHUNKS = np.array(CHUNKS).astype(int)\nJPG_FOLDER = \"/tmp/jpg/\"","metadata":{"id":"b05610d0","papermill":{"duration":0.023492,"end_time":"2023-02-25T23:57:15.524536","exception":false,"start_time":"2023-02-25T23:57:15.501044","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.747993Z","iopub.execute_input":"2023-02-26T08:35:50.748493Z","iopub.status.idle":"2023-02-26T08:35:50.758428Z","shell.execute_reply.started":"2023-02-26T08:35:50.748455Z","shell.execute_reply":"2023-02-26T08:35:50.757514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ttt, chunk in enumerate(CHUNKS):\n    print(f\"chunk {ttt} of {len(CHUNKS)} chunks\")\n    os.makedirs(JPG_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_jpg)(f\"{DATA_FOLDER}/{img}\", save_folder=JPG_FOLDER)\n        for img in test_df[\"fns\"].tolist()[chunk[0] : chunk[1]]\n    )\n\n    jpgfiles = glob.glob(JPG_FOLDER + \"*.jpg\")\n\n    pipe = jpg_decode_pipeline(jpgfiles, batch_size=1, num_threads=2, device_id=0)\n    pipe.build()\n\n    for i, f in enumerate(tqdm(jpgfiles)):\n        patient, dicom_id = f.split(\"/\")[-1][:-4].split(\"_\")\n        dicom = pydicom.dcmread(DATA_FOLDER + f\"/{patient}/{dicom_id}.dcm\")\n        try:\n            out = pipe.run()\n            # Dali -> Torch\n            img = out[0][0]\n            img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n            feed_ndarray(\n                img, img_torch, cuda_stream=torch.cuda.current_stream(device=0)\n            )\n            img = img_torch.float()\n\n            del out, img_torch\n\n            # apply dicom preprocessing\n            img = process_dicom(img, dicom)\n\n            # resize the torch image\n            img = F.interpolate(\n                img.view(1, 1, img.size(0), img.size(1)),\n                (SAVE_SIZE * 2, SAVE_SIZE * 2),\n                mode=\"bilinear\",\n            )[0, 0]\n\n            img = (img * 255).clip(0, 255).to(torch.uint8).cpu().numpy()\n            out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n            cv2.imwrite(out_file_name, img)\n\n            del out_file_name, img, dicom\n\n        except Exception as e:\n            print(i, e)\n            pipe = jpg_decode_pipeline(\n                jpgfiles[i + 1 :], batch_size=1, num_threads=2, device_id=0\n            )\n            pipe.build()\n            continue\n\n    shutil.rmtree(JPG_FOLDER)\nprint(f\"DALI Raw image load complete\")","metadata":{"id":"f2999431","outputId":"da6ef9ad-bc19-4896-d7c0-a1d4135799c4","papermill":{"duration":4.155746,"end_time":"2023-02-25T23:57:19.69416","exception":false,"start_time":"2023-02-25T23:57:15.538414","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:35:50.76011Z","iopub.execute_input":"2023-02-26T08:35:50.760556Z","iopub.status.idle":"2023-02-26T08:36:11.216044Z","shell.execute_reply.started":"2023-02-26T08:35:50.760488Z","shell.execute_reply":"2023-02-26T08:36:11.2148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns = glob.glob(f\"{SAVE_FOLDER}/*.png\")\nn_saved = len(fns)\nprint(f\"Image on disk count : {n_saved}\")","metadata":{"id":"85dc892e","outputId":"410790d6-c9e7-4540-f6ef-184cfe971f27","papermill":{"duration":0.025094,"end_time":"2023-02-25T23:57:19.73463","exception":false,"start_time":"2023-02-25T23:57:19.709536","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.21767Z","iopub.execute_input":"2023-02-26T08:36:11.218437Z","iopub.status.idle":"2023-02-26T08:36:11.227998Z","shell.execute_reply.started":"2023-02-26T08:36:11.218396Z","shell.execute_reply":"2023-02-26T08:36:11.227007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A few hidden test set images might not be decoded via DALI, so we fallback to CPU for those","metadata":{"id":"49a99ce6","papermill":{"duration":0.013946,"end_time":"2023-02-25T23:57:19.763778","exception":false,"start_time":"2023-02-25T23:57:19.749832","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gpu_processed_files = [\n    fn.split(\"/\")[-1].replace(\"_\", \"/\").replace(\"png\", \"dcm\") for fn in fns\n]\nto_process = [f for f in test_df[\"fns\"].values if f not in gpu_processed_files]\nlen(gpu_processed_files), len(to_process)","metadata":{"id":"62c53218","outputId":"b84a4833-1fbd-4654-eec4-66902ad96f59","papermill":{"duration":0.02567,"end_time":"2023-02-25T23:57:19.80368","exception":false,"start_time":"2023-02-25T23:57:19.77801","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.229259Z","iopub.execute_input":"2023-02-26T08:36:11.230073Z","iopub.status.idle":"2023-02-26T08:36:11.303909Z","shell.execute_reply.started":"2023-02-26T08:36:11.230031Z","shell.execute_reply":"2023-02-26T08:36:11.302813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, save_folder=\"\"):\n    patient = f.split(\"/\")[-2]\n    dicom_id = f.split(\"/\")[-1][:-4]\n\n    dicom = dicomsdl.open(f)\n    img = dicom.pixelData()\n\n    img = torch.from_numpy(img)\n    img = process_dicom(img, dicom)\n\n    img = F.interpolate(\n        img.view(1, 1, img.size(0), img.size(1)),\n        (SAVE_SIZE * 2, SAVE_SIZE * 2),\n        mode=\"bilinear\",\n    )[0, 0]\n\n    img = (img * 255).clip(0, 255).to(torch.uint8).cpu().numpy()\n    out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n    cv2.imwrite(out_file_name, img)\n    return out_file_name","metadata":{"id":"158b9628","papermill":{"duration":0.024616,"end_time":"2023-02-25T23:57:19.842348","exception":false,"start_time":"2023-02-25T23:57:19.817732","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.30621Z","iopub.execute_input":"2023-02-26T08:36:11.307272Z","iopub.status.idle":"2023-02-26T08:36:11.315659Z","shell.execute_reply.started":"2023-02-26T08:36:11.307233Z","shell.execute_reply":"2023-02-26T08:36:11.314676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_processed_filenames = Parallel(n_jobs=2)(\n    delayed(process)(f\"{DATA_FOLDER}/{img}\", save_folder=SAVE_FOLDER)\n    for img in tqdm(to_process)\n)\ncpu_processed_filenames = [f for f in cpu_processed_filenames if f]\nprint(f\"CPU Raw image load complete with {len(cpu_processed_filenames)} loaded\")","metadata":{"id":"c6bc25cf","outputId":"e8677d1c-7c26-41b0-9d3e-f7432520b6ef","papermill":{"duration":0.03057,"end_time":"2023-02-25T23:57:19.886976","exception":false,"start_time":"2023-02-25T23:57:19.856406","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.31771Z","iopub.execute_input":"2023-02-26T08:36:11.318558Z","iopub.status.idle":"2023-02-26T08:36:11.331309Z","shell.execute_reply.started":"2023-02-26T08:36:11.318521Z","shell.execute_reply":"2023-02-26T08:36:11.330377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"id":"4ef62522","papermill":{"duration":0.206943,"end_time":"2023-02-25T23:57:20.109039","exception":false,"start_time":"2023-02-25T23:57:19.902096","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.332765Z","iopub.execute_input":"2023-02-26T08:36:11.333446Z","iopub.status.idle":"2023-02-26T08:36:11.536473Z","shell.execute_reply.started":"2023-02-26T08:36:11.333412Z","shell.execute_reply":"2023-02-26T08:36:11.535399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_saved = len(glob.glob(f\"{SAVE_FOLDER}/*.png\"))\nprint(f\"Image on disk count : {n_saved}\")","metadata":{"id":"9ba909ca","outputId":"f8468385-e6df-45dd-ffc1-5256956021c6","papermill":{"duration":0.025523,"end_time":"2023-02-25T23:57:20.149787","exception":false,"start_time":"2023-02-25T23:57:20.124264","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.538088Z","iopub.execute_input":"2023-02-26T08:36:11.538551Z","iopub.status.idle":"2023-02-26T08:36:11.546751Z","shell.execute_reply.started":"2023-02-26T08:36:11.538403Z","shell.execute_reply":"2023-02-26T08:36:11.545738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert n_saved == len(test_df)","metadata":{"id":"22625e93","papermill":{"duration":0.022121,"end_time":"2023-02-25T23:57:20.187336","exception":false,"start_time":"2023-02-25T23:57:20.165215","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.548348Z","iopub.execute_input":"2023-02-26T08:36:11.548697Z","iopub.status.idle":"2023-02-26T08:36:11.555171Z","shell.execute_reply.started":"2023-02-26T08:36:11.548663Z","shell.execute_reply":"2023-02-26T08:36:11.552931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We finished with preprocessing all the dicoms to images. So next, we set-up the dataloading and model","metadata":{"id":"ce7f7337","papermill":{"duration":0.014197,"end_time":"2023-02-25T23:57:20.216185","exception":false,"start_time":"2023-02-25T23:57:20.201988","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# SE-ResNeXt50 Part","metadata":{"id":"66470c51","papermill":{"duration":0.013984,"end_time":"2023-02-25T23:57:20.244679","exception":false,"start_time":"2023-02-25T23:57:20.230695","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Switchers\n# Reject {REJECT_N_PERCENT_SERES}% in Inference of the other 4 CV models\nREJECT_N_PERCENT_SERES = 75\nW1_SERES = 0.2 # W1_SERES = 0.2  if ensamble equally\nW2_SERES = 1 - W1_SERES\nassert W2_SERES >= 0","metadata":{"id":"CDiZShv2YHMz","execution":{"iopub.status.busy":"2023-02-26T08:36:11.556734Z","iopub.execute_input":"2023-02-26T08:36:11.557112Z","iopub.status.idle":"2023-02-26T08:36:11.563445Z","shell.execute_reply.started":"2023-02-26T08:36:11.557078Z","shell.execute_reply":"2023-02-26T08:36:11.562287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def batch_to_device(batch, device):\n    batch_dict = {key: batch[key].to(device) for key in batch}\n    return batch_dict\n\n\nclass CustomDataset(Dataset):\n    def __init__(self, df, cfg, aug):\n        self.cfg = cfg\n        self.df = df.copy()\n        # self.df = self.df[self.df[\"image_id\"].astype(str) != \"1942326353\"]\n        self.labels = self.df[self.cfg.classes].values\n        self.df[\"fns\"] = (\n            self.df[\"patient_id\"].astype(str)\n            + \"_\"\n            + self.df[\"image_id\"].astype(str)\n            + \".png\"\n        )\n        self.fns = self.df[\"fns\"].astype(str).values\n        self.aug = aug\n        self.data_folder = cfg.data_folder\n\n    def __getitem__(self, idx):\n        label = self.labels[idx]\n        img = self.load_one(idx)\n\n        if self.aug:\n            img = self.augment(img)\n\n        img = self.normalize_img(img)\n        torch_img = torch.tensor(img).float().permute(2, 0, 1)\n\n        feature_dict = {\n            \"input\": torch_img,\n            \"target\": torch.tensor(label),\n        }\n        return feature_dict\n\n    def __len__(self):\n        return len(self.fns)\n\n    def load_one(self, idx):\n        path = self.data_folder + self.fns[idx]\n        try:\n            img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n            shape = img.shape\n\n            if CROP_IMAGE_1st:\n                if len(img.shape) == 2:\n                    image_c = crop(img)\n                    if min(*image_c.shape) > 100:\n                        img = image_c\n\n            img = cv2.resize(img, (SAVE_SIZE, SAVE_SIZE))\n\n            if len(img.shape) == 2:\n                img = img[:, :, None]\n\n        except Exception as e:\n            print(e)\n        return img\n\n    def augment(self, img):\n        img = img.astype(np.float32)\n        transformed = self.aug(image=img)\n        trans_img = transformed[\"image\"]\n        return trans_img\n\n    def normalize_img(self, img):\n        img = img / 255\n        return img","metadata":{"id":"33a6bad1","papermill":{"duration":0.030606,"end_time":"2023-02-25T23:57:20.289606","exception":false,"start_time":"2023-02-25T23:57:20.259","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.565114Z","iopub.execute_input":"2023-02-26T08:36:11.565556Z","iopub.status.idle":"2023-02-26T08:36:11.5801Z","shell.execute_reply.started":"2023-02-26T08:36:11.565523Z","shell.execute_reply":"2023-02-26T08:36:11.579004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (\n            self.__class__.__name__\n            + f\"(p={self.p.data.tolist()[0]:.4f},eps={self.eps})\"\n        )\n\n\nclass Net(nn.Module):\n    def __init__(self, cfg: Any):\n        super(Net, self).__init__()\n\n        self.cfg = cfg\n        self.n_classes = len(cfg.classes)\n        self.backbone = timm.create_model(\n            cfg.backbone,\n            pretrained=cfg.pretrained,\n            num_classes=0,\n            global_pool=\"\",\n            in_chans=self.cfg.in_channels,\n        )\n\n        backbone_out = self.backbone.feature_info[-1][\"num_chs\"]\n\n        self.global_pool = GeM(p_trainable=False)\n        self.head = torch.nn.Linear(backbone_out, self.n_classes)\n        self.loss_fn = nn.BCEWithLogitsLoss()\n\n    def forward(self, batch):\n        x = batch[\"input\"]\n\n        x = self.backbone(x)\n        x = self.global_pool(x)\n        x = x[:, :, 0, 0]\n\n        logits = self.head(x)\n\n        outputs = {}\n\n        if self.training:\n            loss = self.loss_fn(logits, batch[\"target\"].float())\n            outputs[\"loss\"] = loss\n        else:\n            outputs[\"logits\"] = logits\n\n        return outputs","metadata":{"id":"d4608483","papermill":{"duration":0.029617,"end_time":"2023-02-25T23:57:20.333734","exception":false,"start_time":"2023-02-25T23:57:20.304117","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.581484Z","iopub.execute_input":"2023-02-26T08:36:11.582047Z","iopub.status.idle":"2023-02-26T08:36:11.596082Z","shell.execute_reply.started":"2023-02-26T08:36:11.582013Z","shell.execute_reply":"2023-02-26T08:36:11.595169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dl(test_df, cfg):\n    test_ds = CustomDataset(test_df, cfg, cfg.val_aug)\n    test_dl = DataLoader(\n        test_ds,\n        shuffle=False,\n        batch_size=cfg.batch_size,\n        num_workers=N_CORES,\n        pin_memory=True,\n    )\n\n    return test_dl, batch_to_device\n\n\ndef get_state_dict(sd_fp):\n    sd = torch.load(sd_fp, map_location=\"cpu\")[\"model\"]\n    sd = {k.replace(\"module.\", \"\"): v for k, v in sd.items()}\n    return sd\n\n\ndef get_nets(cfg, state_dicts):\n    nets = []\n\n    for i, state_dict in enumerate(state_dicts):\n        net = Net(cfg).eval().to(DEVICE)\n        print(\"loading dict\")\n        sd = get_state_dict(state_dict)\n        net.load_state_dict(sd, strict=True)\n        nets += [net]\n        del sd\n        gc.collect()\n    return nets","metadata":{"id":"fc499319","papermill":{"duration":0.024354,"end_time":"2023-02-25T23:57:20.372627","exception":false,"start_time":"2023-02-25T23:57:20.348273","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.597685Z","iopub.execute_input":"2023-02-26T08:36:11.598067Z","iopub.status.idle":"2023-02-26T08:36:11.609295Z","shell.execute_reply.started":"2023-02-26T08:36:11.598032Z","shell.execute_reply":"2023-02-26T08:36:11.608303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"state_dicts = sorted(glob.glob(j(INPUT_DIR, \"rsna-seresnext50-5fold/check*.pth\")))\nprint(state_dicts)\n\nnets = get_nets(cfg, state_dicts)\ntype(nets[0])","metadata":{"id":"1c59dada","outputId":"930913d8-e7d1-4519-bea9-94283a675669","papermill":{"duration":8.269597,"end_time":"2023-02-25T23:57:28.656795","exception":false,"start_time":"2023-02-25T23:57:20.387198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:11.610601Z","iopub.execute_input":"2023-02-26T08:36:11.611086Z","iopub.status.idle":"2023-02-26T08:36:15.072888Z","shell.execute_reply.started":"2023-02-26T08:36:11.611051Z","shell.execute_reply":"2023-02-26T08:36:15.071261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_dl0, batch_to_device = get_dl(test_df, cfg)\nprint(f\"Dataloader length : {len(sub_dl0.dataset)}\")","metadata":{"id":"7eb08e61","outputId":"f1f3b8dc-b783-4a7b-83b6-564e40aeeb91","papermill":{"duration":0.030327,"end_time":"2023-02-25T23:57:28.702495","exception":false,"start_time":"2023-02-25T23:57:28.672168","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:15.079949Z","iopub.execute_input":"2023-02-26T08:36:15.080247Z","iopub.status.idle":"2023-02-26T08:36:15.08905Z","shell.execute_reply.started":"2023-02-26T08:36:15.080221Z","shell.execute_reply":"2023-02-26T08:36:15.087993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict with the first model and reject data\ntest_df0 = test_df.copy().reset_index()\nwith torch.inference_mode():\n    preds0 = [[] for i in range(len(nets[:1]))]\n    for batch in tqdm(sub_dl0):\n        batch = batch_to_device(batch, cfg.device)\n        for i, net in enumerate(nets[:1]):\n            logits = net(batch)[\"logits\"].sigmoid().float().detach().cpu().numpy()\n            preds0[i] += [logits]\n\npreds0 = np.array([np.concatenate(p, axis=0) for p in preds0])\npreds0 = preds0.mean(0)  # average fold predictions\npreds0 = preds0[:, 0]\nprint(f\"shape: {preds0.shape}\")\nt = np.percentile(preds0, REJECT_N_PERCENT_SERES)  # reject REJECT_N_PERCENT_SERES%\nprint(f\"{REJECT_N_PERCENT_SERES}% percentile point is {t}\")\n\ntest_df0[\"cancer_p0\"] = preds0\nprint(\"first model inference is completed\")\ntest_df0","metadata":{"id":"2530d431","outputId":"9c650204-169a-4f01-8e8b-568d14fb9eea","papermill":{"duration":6.478903,"end_time":"2023-02-25T23:57:35.196726","exception":false,"start_time":"2023-02-25T23:57:28.717823","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:15.090711Z","iopub.execute_input":"2023-02-26T08:36:15.091272Z","iopub.status.idle":"2023-02-26T08:36:23.599289Z","shell.execute_reply.started":"2023-02-26T08:36:15.091234Z","shell.execute_reply":"2023-02-26T08:36:23.598142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict with rest of the models\ntest_df1 = test_df0[test_df0.cancer_p0 > t]  # .reset_index(drop=True)\n\nsub_dl1, batch_to_device = get_dl(test_df1, cfg)\n\nwith torch.inference_mode():\n    preds1 = [[] for i in range(len(nets[1:]))]\n    for batch in tqdm(sub_dl1):\n        batch = batch_to_device(batch, cfg.device)\n        for i, net in enumerate(nets[1:]):\n            logits = net(batch)[\"logits\"].sigmoid().float().detach().cpu().numpy()\n            preds1[i] += [logits]\n\npreds1 = np.array([np.concatenate(p, axis=0) for p in preds1])\npreds1 = preds1.mean(0)  # average fold predictions\npreds1 = preds1[:, 0]\nprint(f\"shape: {preds1.shape}\")\nprint(\"The rest of the models inference is completed\")\ngc.collect()","metadata":{"id":"b50dc4ca","outputId":"80f988b8-c61b-4227-9327-ef65bf53cb24","papermill":{"duration":0.693639,"end_time":"2023-02-25T23:57:35.906895","exception":false,"start_time":"2023-02-25T23:57:35.213256","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:23.601208Z","iopub.execute_input":"2023-02-26T08:36:23.601856Z","iopub.status.idle":"2023-02-26T08:36:30.652456Z","shell.execute_reply.started":"2023-02-26T08:36:23.601796Z","shell.execute_reply":"2023-02-26T08:36:30.651354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# indice0 = test_df0.index\nindice1 = test_df1.index\npreds = preds0.copy()\npreds[indice1] = W1_SERES * preds0[indice1] + W2_SERES * preds1","metadata":{"id":"1a326b40","papermill":{"duration":0.023693,"end_time":"2023-02-25T23:57:35.946633","exception":false,"start_time":"2023-02-25T23:57:35.92294","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:36:30.654237Z","iopub.execute_input":"2023-02-26T08:36:30.654632Z","iopub.status.idle":"2023-02-26T08:36:30.66159Z","shell.execute_reply.started":"2023-02-26T08:36:30.654592Z","shell.execute_reply":"2023-02-26T08:36:30.66052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if PUBLIC_RUN or RAM_CHECK:\n    df = pd.DataFrame({\"preds0\": preds0})\n    df.loc[indice1, \"preds1\"] = preds1\n    df[\"preds\"] = preds\n    display(df)\n    print(df.info())\n    del df\n    test_df0.loc[test_df0.cancer_p0 > t,'cancer_p1'] = preds1\n    test_df0.to_csv('/kaggle/working/seres-pred-df.csv',index=False)","metadata":{"id":"3004a1a0","outputId":"afd92c60-57fe-41bc-e864-2c2b9d42d3e6","papermill":{"duration":0.032277,"end_time":"2023-02-25T23:57:35.994671","exception":false,"start_time":"2023-02-25T23:57:35.962394","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:14.58824Z","iopub.execute_input":"2023-02-26T08:38:14.588654Z","iopub.status.idle":"2023-02-26T08:38:14.619724Z","shell.execute_reply.started":"2023-02-26T08:38:14.588622Z","shell.execute_reply":"2023-02-26T08:38:14.618742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del (\n    batch,\n    logits,\n    i,\n    net,\n    nets,\n    batch_to_device,\n    state_dicts,\n    cpu_processed_filenames,\n    ttt,\n    chunk,\n    jpgfiles,\n    pipe,\n    gpu_processed_files,\n    to_process,\n)","metadata":{"id":"33605250","papermill":{"duration":0.043823,"end_time":"2023-02-25T23:57:36.054702","exception":false,"start_time":"2023-02-25T23:57:36.010879","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:25.592948Z","iopub.execute_input":"2023-02-26T08:38:25.593329Z","iopub.status.idle":"2023-02-26T08:38:25.654879Z","shell.execute_reply.started":"2023-02-26T08:38:25.593299Z","shell.execute_reply":"2023-02-26T08:38:25.653767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"id":"62019b60","papermill":{"duration":0.224985,"end_time":"2023-02-25T23:57:36.295771","exception":false,"start_time":"2023-02-25T23:57:36.070786","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:25.657513Z","iopub.execute_input":"2023-02-26T08:38:25.657972Z","iopub.status.idle":"2023-02-26T08:38:25.927077Z","shell.execute_reply.started":"2023-02-26T08:38:25.657931Z","shell.execute_reply":"2023-02-26T08:38:25.925904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ConvNext Part","metadata":{"id":"11770c6f","papermill":{"duration":0.015397,"end_time":"2023-02-25T23:57:36.327509","exception":false,"start_time":"2023-02-25T23:57:36.312112","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import tensorflow as tf\nfrom keras_cv_attention_models import convnext","metadata":{"id":"9d856e9a","papermill":{"duration":4.94454,"end_time":"2023-02-25T23:57:41.28798","exception":false,"start_time":"2023-02-25T23:57:36.34344","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:25.928759Z","iopub.execute_input":"2023-02-26T08:38:25.929483Z","iopub.status.idle":"2023-02-26T08:38:30.668Z","shell.execute_reply.started":"2023-02-26T08:38:25.929438Z","shell.execute_reply":"2023-02-26T08:38:30.66697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_HEIGHT = 1344\nTARGET_WIDTH = 768\nN_CHANNELS = 1\nINPUT_SHAPE = (TARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS)\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH\n\n\n# Inference configs\nNUM_OF_MODELS_CONV1 = 6\nNUM_OF_MODELS_CONV2 = 6\nNUM_OF_TTA_CONV = 1\n# rejection proportion[%]\nREJECT_N_PERCENT_CONV_1st = 70\n# Actual proportion is REJECT_N_PERCENT_CONV_1st*REJECT_N_PERCENT_CONV_2nd\nREJECT_N_PERCENT_CONV_2nd = 1\nTTA_N_PERCENT_CONV = 80\n\n# ensemble weights: W1_CONV+W2_CONV+W3_CONV=1\nW1_CONV = 0.3  # original model LB:0.52\nW2_CONV = 0.4  # CV models\nW3_CONV = 1 - W1_CONV - W2_CONV\nassert W3_CONV >= 0\n\n# Ensemble\nAB_RATE = 0.73\nSCALE_CONV = False","metadata":{"id":"141ba7a1","papermill":{"duration":0.024599,"end_time":"2023-02-25T23:57:41.328866","exception":false,"start_time":"2023-02-25T23:57:41.304267","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:30.669638Z","iopub.execute_input":"2023-02-26T08:38:30.670041Z","iopub.status.idle":"2023-02-26T08:38:30.682091Z","shell.execute_reply.started":"2023-02-26T08:38:30.670002Z","shell.execute_reply":"2023-02-26T08:38:30.680305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(image):\n    # Repeat channels to create 3 channel images required by pretrained ConvNextV2 models\n    image = tf.repeat(image, repeats=3, axis=3)\n    # Cast to float 32\n    image = tf.cast(image, tf.float32)\n    # Normalize with respect to ImageNet mean/std\n    image = tf.keras.applications.imagenet_utils.preprocess_input(image, mode=\"torch\")\n\n    return image","metadata":{"id":"e8f29e0c","papermill":{"duration":0.024657,"end_time":"2023-02-25T23:57:41.369578","exception":false,"start_time":"2023-02-25T23:57:41.344921","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:30.685668Z","iopub.execute_input":"2023-02-26T08:38:30.68608Z","iopub.status.idle":"2023-02-26T08:38:30.705526Z","shell.execute_reply.started":"2023-02-26T08:38:30.686038Z","shell.execute_reply":"2023-02-26T08:38:30.704538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(\n    model_path=\"/kaggle/input/rsna-efficientnetv2-training-tensorflow-tpu-ds/model.h5\",\n):\n    # Inputs, note the names are equal to the dictionary keys in the dataset\n    image = tf.keras.layers.Input(INPUT_SHAPE, name=\"image\", dtype=tf.uint8)\n\n    # Normalize Input\n    image_norm = normalize(image)\n\n    # CNN Feature Maps\n    x = convnext.ConvNeXtV2Tiny(\n        input_shape=(TARGET_HEIGHT, TARGET_WIDTH, 3),\n        pretrained=None,\n        num_classes=0,\n    )(image_norm)\n\n    # Average Pooling BxHxWxC -> BxC\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    # Dropout to prevent Overfitting\n    x = tf.keras.layers.Dropout(0.30)(x)\n    # Output value between [0, 1] using Sigmoid function\n    outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n\n    # Define model with inputs and outputs\n    model = tf.keras.models.Model(inputs=image, outputs=outputs)\n\n    # Load pretrained Model Weights\n    model.load_weights(model_path)\n\n    # Set model non-trainable\n    model.trainable = False\n\n    # Compile model\n    model.compile()\n\n    return model","metadata":{"id":"40c62a65","papermill":{"duration":0.026446,"end_time":"2023-02-25T23:57:41.411679","exception":false,"start_time":"2023-02-25T23:57:41.385233","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:30.707134Z","iopub.execute_input":"2023-02-26T08:38:30.70757Z","iopub.status.idle":"2023-02-26T08:38:30.716734Z","shell.execute_reply.started":"2023-02-26T08:38:30.707535Z","shell.execute_reply":"2023-02-26T08:38:30.715712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained File Path: '/kaggle/input/sartorius-training-dataset/model.h5'\ntf.keras.backend.clear_session()\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\n# model = get_model()\nmodels = []\nmodel_paths = [\n    j(INPUT_DIR, \"rsna-efficientnetv2-training-tensorflow-tpu-ds/model.h5\")\n] + glob.glob(j(INPUT_DIR, \"rsna-convnext-5models-stkfold/results/models/*.h5\"))\nfor model_path in model_paths:\n    model = get_model(model_path)\n    models.append(model)\n\n# models = models[:NUM_OF_MODELS_CONV]\nmodels","metadata":{"id":"72ca729a","outputId":"0024d0d3-83e0-4395-f4ac-a0253ec22fc9","papermill":{"duration":27.223445,"end_time":"2023-02-25T23:58:08.650969","exception":false,"start_time":"2023-02-25T23:57:41.427524","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:30.718439Z","iopub.execute_input":"2023-02-26T08:38:30.718797Z","iopub.status.idle":"2023-02-26T08:38:57.171491Z","shell.execute_reply.started":"2023-02-26T08:38:30.718763Z","shell.execute_reply":"2023-02-26T08:38:57.170471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test = sub_dl.dataset.df\n# SUBMISSION_ROWS = []\n# # Iterate over all patient_id/laterality combinations groups\n# for idx, ((patient_id, laterality), g) in enumerate(tqdm(test.groupby(['patient_id', 'laterality']))):\n#     # Cancer target is mean of predicted cancer values\n#     cancer = 0\n#     # Iterate over all scans in group\n#     for row_idx, row in g.iterrows():\n#         # Load Image\n#         image_id = row['image_id']\n#         image = cv2.imread(f'/tmp/output/{patient_id}_{image_id}.png', -1)\n\n#         if CROP_IMAGE_2nd:\n#             if len(image.shape) == 2:\n#                 image_c = crop(image)\n#                 if min(*image_c.shape) > 100:\n#                     image = image_c\n\n#         image = cv2.resize(image, (TARGET_WIDTH, TARGET_HEIGHT))\n\n#         # Expand to Batch HxW -> 1xHxWx1\n#         image = np.expand_dims(image, [0, 3])\n#         # Make Prediction\n#         cancer += model.predict_on_batch(image).squeeze() / len(g)\n\n#     # Add Submission Row\n#     SUBMISSION_ROWS.append({\n#         'prediction_id': f'{patient_id}_{laterality}',\n#         'cancer': np.float32(cancer),\n#     })\n\n#     if np.random.rand() > 0.99:\n#         gc.collect()","metadata":{"id":"fc725a40","papermill":{"duration":0.024028,"end_time":"2023-02-25T23:58:08.691487","exception":false,"start_time":"2023-02-25T23:58:08.667459","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:57.173143Z","iopub.execute_input":"2023-02-26T08:38:57.173789Z","iopub.status.idle":"2023-02-26T08:38:57.179728Z","shell.execute_reply.started":"2023-02-26T08:38:57.173747Z","shell.execute_reply":"2023-02-26T08:38:57.17871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.errors import InvalidArgumentError\nimport time\n\n\n# short Tensorflow randin integer function\ndef tf_rand_int(minval, maxval, dtype=tf.int64):\n    minval = tf.cast(minval, dtype)\n    maxval = tf.cast(maxval, dtype)\n    return tf.random.uniform(shape=(), minval=minval, maxval=maxval, dtype=dtype)\n\n\ndef get_pred_df(\n    test: pd.DataFrame, models: list, show_images=False, n_aug=0, verbose=1000\n) -> pd.DataFrame:\n    start_time = time.time()\n    n_images = 0\n    preds_prob = []\n    # Iterate over all patient_id/laterality combinations groups\n    for idx, ((patient_id, laterality), g) in enumerate(\n        tqdm(test.groupby([\"patient_id\", \"laterality\"]))\n    ):\n        # Cancer target is mean of predicted cancer values\n        cancer = 0\n        # Iterate over all scans in group\n        for row_idx, row in g.iterrows():\n            # Load Image\n            image_id = row[\"image_id\"]\n            image = cv2.imread(f\"/tmp/output/{patient_id}_{image_id}.png\", -1)\n\n            if CROP_IMAGE_2nd:\n                if len(image.shape) == 2:\n                    image_c = crop(image)\n                    if min(*image_c.shape) > 100:\n                        image = image_c\n\n            image = cv2.resize(image, (TARGET_WIDTH, TARGET_HEIGHT))\n\n            assert image is not None\n            aug_images = []\n            if n_aug > 0:\n                for _ in range(n_aug):\n                    # Random Brightness\n                    aug_image = tf.image.random_brightness(image, 0.10)\n\n                    # Random Contrast\n                    try:\n                        aug_image = tf.image.random_contrast(aug_image, 0.90, 1.10)\n                    except InvalidArgumentError as e:\n                        aug_image = np.expand_dims(aug_image, [2])\n                        aug_image = tf.image.random_contrast(aug_image, 0.90, 1.10)\n\n                    # Random JPEG Quality\n                    aug_image = tf.image.random_jpeg_quality(aug_image, 75, 100)\n\n                    # Random crop aug_image with maximum of 10%\n                    ratio = tf.random.uniform([], 0.75, 1.00)\n                    aug_image_height_crop = tf.cast(ratio * TARGET_HEIGHT, tf.int32)\n                    aug_image_width_crop = tf.cast(ratio * TARGET_WIDTH, tf.int32)\n                    # Random offset for crop\n                    aug_image_height_offset = tf_rand_int(\n                        0, TARGET_HEIGHT - aug_image_height_crop\n                    )\n                    aug_image_width_offset = 0\n                    # Crop And Resize\n                    aug_image = tf.slice(\n                        aug_image,\n                        [aug_image_height_offset, aug_image_width_offset, 0],\n                        [aug_image_height_crop, aug_image_width_crop, N_CHANNELS],\n                    )\n                    aug_image = tf.image.resize(\n                        aug_image,\n                        [TARGET_HEIGHT, TARGET_WIDTH],\n                        method=tf.image.ResizeMethod.BILINEAR,\n                    )\n                    # Clip pixel values in range [0,255] to prevent underflow/overflow\n                    aug_image = tf.clip_by_value(aug_image, 0, 255)\n                    aug_image = tf.cast(aug_image, tf.uint8)\n                    aug_images.append(aug_image.numpy().squeeze())\n\n            # Show First Few Images\n            if show_images and idx < 16:\n                plt.figure(figsize=(5, 8))\n                plt.imshow(image)\n                plt.show()\n\n            # Expand to Batch HxW -> 1xHxWx1\n            assert image.shape == (TARGET_HEIGHT, TARGET_WIDTH)\n            image = np.expand_dims(image, [0, 3])\n            # Make Prediction\n            for i, model in enumerate(models):\n                prediction = model.predict_on_batch(image).squeeze()\n                # if IS_INTERACTIVE: print(f'{patient_id}_{laterality}: image_id: {image_id}, model: {i}, pred: {prediction}')\n                cancer += prediction / len(g) / len(models) / (n_aug + 1)\n                n_images += 1\n                # TTA\n                for aug_image in aug_images:\n                    assert aug_image.shape == image.squeeze().shape\n                    aug_image = np.expand_dims(aug_image, [0, 3])\n                    prediction = model.predict_on_batch(aug_image).squeeze()\n                    # if IS_INTERACTIVE: print(f'{patient_id}_{laterality}: image_id: {image_id}, model: {i}, pred: {prediction}')\n                    cancer += prediction / len(g) / len(models) / (n_aug + 1)\n                    n_images += 1\n\n            # Remove Image\n            # os.remove(f'{image_id}.{IMAGE_FORMAT}')\n            del image, aug_images\n            gc.collect()\n\n        # Add Submission Row\n        preds_prob.append(\n            {\n                \"prediction_id\": f\"{patient_id}_{laterality}\",\n                \"cancer\": cancer if cancer > 0 else np.nan,\n            }\n        )\n\n        if np.random.rand() > 0.99:\n            gc.collect()\n        if n_images > 0 and n_images % verbose == 0:\n            print(\n                f\"idx: {idx}, num of image: {n_images}, avg time: {(time.time()-start_time)/n_images}sec.\"\n            )\n\n    return pd.DataFrame(preds_prob)","metadata":{"id":"fa8347a2","papermill":{"duration":0.040707,"end_time":"2023-02-25T23:58:08.748368","exception":false,"start_time":"2023-02-25T23:58:08.707661","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:57.181435Z","iopub.execute_input":"2023-02-26T08:38:57.181793Z","iopub.status.idle":"2023-02-26T08:38:57.20522Z","shell.execute_reply.started":"2023-02-26T08:38:57.181757Z","shell.execute_reply":"2023-02-26T08:38:57.2043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = sub_dl0.dataset.df\nprint(f\"shape: {test.shape}\")\ntest.head()","metadata":{"id":"e61c9ce1","outputId":"a54cd79b-773f-4ef5-f131-151596e37eb6","papermill":{"duration":0.035005,"end_time":"2023-02-25T23:58:08.799174","exception":false,"start_time":"2023-02-25T23:58:08.764169","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:57.206778Z","iopub.execute_input":"2023-02-26T08:38:57.20716Z","iopub.status.idle":"2023-02-26T08:38:57.231139Z","shell.execute_reply.started":"2023-02-26T08:38:57.207125Z","shell.execute_reply":"2023-02-26T08:38:57.230041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict with the first model and reject data\n\n# Reject Using SERES results\n\n\nt0 = np.percentile(\n    preds0, REJECT_N_PERCENT_CONV_1st\n)  # reject REJECT_N_PERCENT_CONV_1st%\nprint(f\"{REJECT_N_PERCENT_CONV_1st}% percentile point is {t0}\")\n\ntest_df0_conv = (\n    test_df0[test_df0.cancer_p0 > t0].reset_index(drop=True).drop(columns=\"cancer_p0\")\n)\npreds_prob_df0 = get_pred_df(test_df0_conv, models[:1], n_aug=0)\npreds_prob_df0.rename(columns={\"cancer\": \"cancer_p0\"}, inplace=True)\nprobability0 = preds_prob_df0[\"cancer_p0\"].values\n\n# Rejection points\nt1 = np.percentile(\n    probability0, REJECT_N_PERCENT_CONV_2nd\n)  # reject REJECT_N_PERCENT_CONV_2nd%\nprint(f\"{REJECT_N_PERCENT_CONV_2nd}% percentile point is {t1}\")\nt2 = np.percentile(\n    probability0, TTA_N_PERCENT_CONV\n)  # reject REJECT_N_PERCENT_CONV_2nd%\nprint(f\"{TTA_N_PERCENT_CONV}% percentile point is {t2}\")\n\ntest_df0_conv[\"prediction_id\"] = (\n    test_df0_conv[\"patient_id\"].astype(\"string\") + \"_\" + test_df0_conv[\"laterality\"]\n)\ntest_df0_conv = pd.merge(test_df0_conv, preds_prob_df0, on=\"prediction_id\", how=\"left\")\nprint(\"first model inference is completed\")\ntest_df0_conv","metadata":{"id":"4a364b81","outputId":"7e731fef-cc8e-43e7-abdc-f1180c806fe0","papermill":{"duration":8.646766,"end_time":"2023-02-25T23:58:17.462339","exception":false,"start_time":"2023-02-25T23:58:08.815573","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:38:57.232889Z","iopub.execute_input":"2023-02-26T08:38:57.233617Z","iopub.status.idle":"2023-02-26T08:39:26.341089Z","shell.execute_reply.started":"2023-02-26T08:38:57.233575Z","shell.execute_reply":"2023-02-26T08:39:26.34Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict with rest of the models\ntest_df1 = test_df0_conv[\n    (test_df0_conv.cancer_p0 > t1) & (test_df0_conv.cancer_p0 < t2)\n].reset_index(drop=True)\nif len(test_df1) == 0:\n    test_df1 = test_df0_conv.copy()\n%time preds_prob_df1 = get_pred_df(test_df1,models[1:NUM_OF_MODELS_CONV1],n_aug=0)\npreds_prob_df1.rename(columns={\"cancer\": \"cancer_p1\"}, inplace=True)\n\npreds_prob_df01 = pd.merge(\n    preds_prob_df0, preds_prob_df1, on=\"prediction_id\", how=\"left\"\n)\npreds_prob_df01.loc[\n    preds_prob_df01.cancer_p1.isna(), \"cancer_p1\"\n] = preds_prob_df01.cancer_p0\nprint(f\"The rest of the {NUM_OF_MODELS_CONV1-1} models inference is completed\")\ngc.collect()","metadata":{"id":"9ade8b5a","outputId":"3215fed1-2d58-48f6-de11-9ae6873ee2bb","papermill":{"duration":1.776841,"end_time":"2023-02-25T23:58:19.257852","exception":false,"start_time":"2023-02-25T23:58:17.481011","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:39:26.34296Z","iopub.execute_input":"2023-02-26T08:39:26.34366Z","iopub.status.idle":"2023-02-26T08:40:01.065455Z","shell.execute_reply.started":"2023-02-26T08:39:26.34362Z","shell.execute_reply":"2023-02-26T08:40:01.064429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TTA\ntest_df2 = test_df0_conv[test_df0_conv.cancer_p0 >= t2].reset_index(drop=True)\npreds_prob_df2 = get_pred_df(\n    test_df2, models[:NUM_OF_MODELS_CONV2], n_aug=NUM_OF_TTA_CONV\n)\npreds_prob_df2.rename(columns={\"cancer\": \"cancer_p2\"}, inplace=True)\nprint(\"TTA is completed.\")\nprint(f\"TTA info: n_aug={NUM_OF_TTA_CONV}, 3 models\")\n# Merger\npreds_prob_df_merged = pd.merge(\n    preds_prob_df01, preds_prob_df2, on=\"prediction_id\", how=\"left\"\n)\npreds_prob_df_merged.loc[\n    preds_prob_df_merged.cancer_p2.isna(), \"cancer_p2\"\n] = preds_prob_df_merged.cancer_p0\n\npreds_prob_df_merged[\"cancer\"] = (\n    W1_CONV * preds_prob_df_merged.cancer_p0.values\n    + W2_CONV * preds_prob_df_merged.cancer_p1.values\n    + W3_CONV * preds_prob_df_merged.cancer_p2.values\n)\nprint(\"Merger is completed.\")\nprint(f\"Merger info: W1_CONV={W1_CONV},W2_CONV={W2_CONV},W3_CONV={W3_CONV}\")\npreds_prob_df_merged","metadata":{"id":"c5740fc3","outputId":"af27d26d-f888-40c6-9156-4b3d377319d7","papermill":{"duration":0.887617,"end_time":"2023-02-25T23:58:20.165552","exception":false,"start_time":"2023-02-25T23:58:19.277935","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:40:01.067204Z","iopub.execute_input":"2023-02-26T08:40:01.067905Z","iopub.status.idle":"2023-02-26T08:40:04.963165Z","shell.execute_reply.started":"2023-02-26T08:40:01.067863Z","shell.execute_reply":"2023-02-26T08:40:04.962034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id = sub_dl0.dataset.df[\"patient_id\"].values\nlaterality = sub_dl0.dataset.df[\"laterality\"].values\n\nprediction_id_list = [f\"{i}_{j}\" for i, j in zip(patient_id, laterality)]\n\npreds_prob_df_merged = pd.merge(\n    pd.DataFrame({\"prediction_id\": prediction_id_list}),\n    preds_prob_df_merged,\n    on=\"prediction_id\",\n    how=\"left\",\n)\n\npreds_prob_df_merged","metadata":{"id":"YGcq7V1-cz61","outputId":"5ee18341-3f5c-4baa-e177-de0b575339d5","execution":{"iopub.status.busy":"2023-02-26T08:40:04.967961Z","iopub.execute_input":"2023-02-26T08:40:04.968416Z","iopub.status.idle":"2023-02-26T08:40:04.990177Z","shell.execute_reply.started":"2023-02-26T08:40:04.968378Z","shell.execute_reply":"2023-02-26T08:40:04.989088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SUBMISSION_ROWS = []\n# Add Submission Row\nfor prediction_id, cancer in zip(\n    preds_prob_df_merged.prediction_id.values, preds_prob_df_merged.cancer.values\n):\n    SUBMISSION_ROWS.append(\n        {\n            \"prediction_id\": prediction_id,\n            \"cancer\": np.float32(cancer),\n        }\n    )\nif RAM_CHECK:\n    preds_prob_df_merged.to_csv(\"convnext-predictions-df.csv\", index=False)\ndel preds_prob_df_merged\ngc.collect()","metadata":{"id":"5aebd358","papermill":{"duration":0.304162,"end_time":"2023-02-25T23:58:20.48795","exception":false,"start_time":"2023-02-25T23:58:20.183788","status":"completed"},"tags":[],"outputId":"8ed4cc51-a7e6-421d-ae0f-fbf8c3ffb9ac","execution":{"iopub.status.busy":"2023-02-26T08:40:04.991944Z","iopub.execute_input":"2023-02-26T08:40:04.992382Z","iopub.status.idle":"2023-02-26T08:40:05.337232Z","shell.execute_reply.started":"2023-02-26T08:40:04.992339Z","shell.execute_reply":"2023-02-26T08:40:05.336046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble ","metadata":{"id":"iynPbhdKStqG"}},{"cell_type":"code","source":"if PUBLIC_RUN or RAM_CHECK:\n    display(pd.DataFrame(SUBMISSION_ROWS))\n    print(pd.DataFrame(SUBMISSION_ROWS).info())","metadata":{"id":"XFmeb8Teiin-","outputId":"9ef8024b-07d4-4b96-b77e-c0293640afe3","execution":{"iopub.status.busy":"2023-02-26T08:40:05.338941Z","iopub.execute_input":"2023-02-26T08:40:05.339527Z","iopub.status.idle":"2023-02-26T08:40:05.363463Z","shell.execute_reply.started":"2023-02-26T08:40:05.33949Z","shell.execute_reply":"2023-02-26T08:40:05.362363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1_std = np.std(preds)\nsub1_mean = np.mean(preds)\nsub2_std = np.std([r[\"cancer\"] for r in SUBMISSION_ROWS if not np.isnan(r[\"cancer\"])])\nsub2_mean = np.mean([r[\"cancer\"] for r in SUBMISSION_ROWS if not np.isnan(r[\"cancer\"])])\nassert not np.isnan(sub1_std)\nassert not np.isnan(sub1_mean)\nassert not np.isnan(sub2_std)\nassert not np.isnan(sub2_mean)\nprint(\n    f\"sub1_std: {sub1_std}, sub1_mean: {sub1_mean}, sub2_std: {sub2_std}, sub2_mean: {sub2_mean}\"\n)","metadata":{"id":"c5f2323e","papermill":{"duration":0.026685,"end_time":"2023-02-25T23:58:20.532556","exception":false,"start_time":"2023-02-25T23:58:20.505871","status":"completed"},"tags":[],"outputId":"4f1abb7f-1602-4d14-cb79-b25e3c389330","execution":{"iopub.status.busy":"2023-02-26T08:40:05.365121Z","iopub.execute_input":"2023-02-26T08:40:05.365732Z","iopub.status.idle":"2023-02-26T08:40:05.374442Z","shell.execute_reply.started":"2023-02-26T08:40:05.365693Z","shell.execute_reply":"2023-02-26T08:40:05.372969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if SCALE_CONV:\n\n    submission_2 = {\n        r[\"prediction_id\"]: sub1_std * ((r[\"cancer\"] - sub2_mean) / sub2_std) + sub1_mean\n        for r in SUBMISSION_ROWS\n    }\nelse:\n    submission_2 = {\n        r[\"prediction_id\"]: r[\"cancer\"]\n        for r in SUBMISSION_ROWS\n    }\n    preds = sub2_std *(preds-sub1_mean)/ sub1_std + sub2_mean\n\nsubmission_2","metadata":{"id":"8aa37eb6","papermill":{"duration":0.028216,"end_time":"2023-02-25T23:58:20.578185","exception":false,"start_time":"2023-02-25T23:58:20.549969","status":"completed"},"tags":[],"outputId":"22586c60-cf8b-4165-bd32-8756596eb4f7","execution":{"iopub.status.busy":"2023-02-26T08:40:05.375907Z","iopub.execute_input":"2023-02-26T08:40:05.37636Z","iopub.status.idle":"2023-02-26T08:40:05.390289Z","shell.execute_reply.started":"2023-02-26T08:40:05.376325Z","shell.execute_reply":"2023-02-26T08:40:05.389212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ab_rate = AB_RATE\nassert set(submission_2.keys()) == set(prediction_id_list)\n\npreds_merged = [\n    sub_1 * ab_rate + submission_2[p] * (1.0 - ab_rate)\n    if not np.isnan(submission_2[p])\n    else sub_1\n    for sub_1, p in zip(preds, prediction_id_list)\n]\n\npred_df = pd.DataFrame(\n    {\"prediction_id\": prediction_id_list, \"cancer_raw\": preds_merged}\n)\nif PUBLIC_RUN or RAM_CHECK:\n    pred_df[\"seres_pred\"] = preds\n    display(pred_df)","metadata":{"id":"fq9ig7aniFhW","outputId":"9fad5eff-3560-4e28-8ea1-22973a7b3f36","execution":{"iopub.status.busy":"2023-02-26T08:40:05.391645Z","iopub.execute_input":"2023-02-26T08:40:05.39235Z","iopub.status.idle":"2023-02-26T08:40:05.410459Z","shell.execute_reply.started":"2023-02-26T08:40:05.392312Z","shell.execute_reply":"2023-02-26T08:40:05.409297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# aggregate by prediction_id , i.e. by patient_laterality\nsub = pred_df.groupby(\"prediction_id\")[[\"cancer_raw\"]].agg(\"mean\")\n\n# binarize predictions\nth = np.quantile(sub[\"cancer_raw\"].values, 0.97935)\nsub[\"cancer\"] = (sub[\"cancer_raw\"].values > th).astype(int)","metadata":{"id":"Et9r_ubReW9a","execution":{"iopub.status.busy":"2023-02-26T08:40:05.412319Z","iopub.execute_input":"2023-02-26T08:40:05.412686Z","iopub.status.idle":"2023-02-26T08:40:05.425209Z","shell.execute_reply.started":"2023-02-26T08:40:05.41265Z","shell.execute_reply":"2023-02-26T08:40:05.424169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[[\"cancer\"]].to_csv(\"submission.csv\")\npd.read_csv(\"submission.csv\")","metadata":{"id":"1122d115","papermill":{"duration":0.034528,"end_time":"2023-02-25T23:58:20.680037","exception":false,"start_time":"2023-02-25T23:58:20.645509","status":"completed"},"tags":[],"outputId":"4a775730-09c7-4d9f-efaf-e35f31f32261","execution":{"iopub.status.busy":"2023-02-26T08:40:05.42686Z","iopub.execute_input":"2023-02-26T08:40:05.427244Z","iopub.status.idle":"2023-02-26T08:40:05.44614Z","shell.execute_reply.started":"2023-02-26T08:40:05.427203Z","shell.execute_reply":"2023-02-26T08:40:05.445251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for debugging purpose we can calculate the pF1 score if we infered on the train data by setting RAM_CHECK=True in the beginning","metadata":{"id":"22a627c0","papermill":{"duration":0.018583,"end_time":"2023-02-25T23:58:20.717135","exception":false,"start_time":"2023-02-25T23:58:20.698552","status":"completed"},"tags":[]}},{"cell_type":"code","source":"if RAM_CHECK and IS_KAGGLE:\n\n    def pfbeta(labels, predictions, beta):\n        # official implementation\n        y_true_count = 0\n        ctp = 0\n        cfp = 0\n\n        for idx in range(len(labels)):\n            prediction = min(max(predictions[idx], 0), 1)\n            if labels[idx]:\n                y_true_count += 1\n                ctp += prediction\n            #             cfp += 1 - prediction #bugfix\n            else:\n                cfp += prediction\n\n        beta_squared = beta * beta\n        c_precision = ctp / (ctp + cfp)\n        c_recall = ctp / y_true_count\n        if c_precision > 0 and c_recall > 0:\n            result = (\n                (1 + beta_squared)\n                * (c_precision * c_recall)\n                / (beta_squared * c_precision + c_recall)\n            )\n            return result\n        else:\n            return 0\n\n    # aggregate by prediction_id , i.e. by patient_laterality\n    test_df[\"prediction_id\"] = test_df.apply(\n        lambda x: f\"{x.patient_id}_{x.laterality}\", 1\n    )\n    test_df_gr = test_df.groupby(\"prediction_id\")[[\"cancer\"]].agg(\"mean\")\n\n    # Sort both the same\n    test_df_gr = test_df_gr.loc[sub.index]\n\n    y = test_df_gr[\"cancer\"].values  # .astype(np.float32)\n    y_pred = sub[\"cancer\"].values\n\n    #     print(y.shape, y_pred.shape)\n\n    score = pfbeta(y, y_pred, 1)\n    print(th, score)","metadata":{"id":"82f2953e","papermill":{"duration":0.032803,"end_time":"2023-02-25T23:58:20.76824","exception":false,"start_time":"2023-02-25T23:58:20.735437","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:40:05.447616Z","iopub.execute_input":"2023-02-26T08:40:05.448227Z","iopub.status.idle":"2023-02-26T08:40:05.466261Z","shell.execute_reply.started":"2023-02-26T08:40:05.44819Z","shell.execute_reply":"2023-02-26T08:40:05.464352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if RAM_CHECK and IS_KAGGLE:\n    import optuna\n\n    def opt(trial):\n        ab_rate = trial.suggest_uniform(\"ab_rate\", 0.1, 0.9)\n        th_rate = trial.suggest_uniform(\"th_rate\", 0.0, 0.1)\n\n        preds_merged = [\n            sub_1 * ab_rate + submission_2[p] * (1.0 - ab_rate)\n            for sub_1, p in zip(preds, prediction_id)\n        ]\n\n        pred_df = pd.DataFrame(\n            {\"prediction_id\": prediction_id, \"cancer_raw\": preds_merged}\n        )\n\n        # aggregate by prediction_id , i.e. by patient_laterality\n        sub = pred_df.groupby(\"prediction_id\")[[\"cancer_raw\"]].agg(\"mean\")\n\n        # binarize predictions\n        th = np.quantile(sub[\"cancer_raw\"].values, 0.9 + th_rate)\n        sub[\"cancer\"] = (sub[\"cancer_raw\"].values > th).astype(int)\n\n        y = test_df_gr[\"cancer\"].values  # .astype(np.float32)\n        y_pred = sub[\"cancer\"].values\n\n        score = pfbeta(y, y_pred, 1)\n\n        return score\n\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(opt, n_trials=100)\n    print(study.best_params)","metadata":{"id":"a7cfe707","papermill":{"duration":0.028884,"end_time":"2023-02-25T23:58:20.815095","exception":false,"start_time":"2023-02-25T23:58:20.786211","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-26T08:40:05.468421Z","iopub.execute_input":"2023-02-26T08:40:05.468936Z","iopub.status.idle":"2023-02-26T08:40:05.934025Z","shell.execute_reply.started":"2023-02-26T08:40:05.468826Z","shell.execute_reply":"2023-02-26T08:40:05.932747Z"},"trusted":true},"execution_count":null,"outputs":[]}]}