{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Blend notebook \"SE-ResNeXt50 full GPU decoding\" and \"RSNA ConvNextV2 Inference Tensorflow\"\n## \"SE-ResNeXt50 full GPU decoding\" is:\nhttps://www.kaggle.com/code/christofhenkel/se-resnext50-full-gpu-decoding/notebook\n\n## \"RSNA ConvNextV2 Inference Tensorflow\" is:\nhttps://www.kaggle.com/code/markwijkhuizen/rsna-convnextv2-inference-tensorflow/notebook\n\n**Thank you Dieter (@christofhenkel)** for public pretrained model `rsna-seresnext50-5fold`\n\nand\n\n**Thank you Mark Wijkhuizen (@markwijkhuizen)** for public pretrained model `rsna-efficientnetv2-training-tensorflow-tpu-ds`","metadata":{"papermill":{"duration":0.016431,"end_time":"2023-02-22T01:55:13.866155","exception":false,"start_time":"2023-02-22T01:55:13.849724","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Install Keras CV Attention Model Pip Package for ConvNextV2 Models\n!pip install --no-deps /kaggle/input/keras-cv-attention-models/keras_cv_attention_models-1.3.9-py3-none-any.whl","metadata":{"papermill":{"duration":22.932976,"end_time":"2023-02-22T01:55:36.810344","exception":false,"start_time":"2023-02-22T01:55:13.877368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:36:00.39324Z","iopub.execute_input":"2023-02-23T07:36:00.393685Z","iopub.status.idle":"2023-02-23T07:36:23.191109Z","shell.execute_reply.started":"2023-02-23T07:36:00.393609Z","shell.execute_reply":"2023-02-23T07:36:23.189941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q timm==0.6.5 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q albumentations==1.2.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q pylibjpeg-libjpeg==1.3.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q pydicom==2.0.0 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q python-gdcm==3.0.20 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements\n!pip install -q dicomsdl==0.109.1 --no-index --find-links=/kaggle/input/rsna-bc-pip-requirements","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":62.110564,"end_time":"2023-02-22T01:56:38.931343","exception":false,"start_time":"2023-02-22T01:55:36.820779","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:36:23.19345Z","iopub.execute_input":"2023-02-23T07:36:23.194133Z","iopub.status.idle":"2023-02-23T07:37:24.437843Z","shell.execute_reply.started":"2023-02-23T07:36:23.194092Z","shell.execute_reply":"2023-02-23T07:37:24.436657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we install the latest DALI packaging which we will use for GPU decoding","metadata":{"papermill":{"duration":0.010244,"end_time":"2023-02-22T01:56:38.952726","exception":false,"start_time":"2023-02-22T01:56:38.942482","status":"completed"},"tags":[]}},{"cell_type":"code","source":"!pip install -q /kaggle/input/nvidia-dali-nightly-cuda110-1230dev/nvidia_dali_nightly_cuda110-1.23.0.dev20230203-7187866-py3-none-manylinux2014_x86_64.whl","metadata":{"papermill":{"duration":42.283887,"end_time":"2023-02-22T01:57:21.246875","exception":false,"start_time":"2023-02-22T01:56:38.962988","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:37:24.439854Z","iopub.execute_input":"2023-02-23T07:37:24.440238Z","iopub.status.idle":"2023-02-23T07:38:06.22742Z","shell.execute_reply.started":"2023-02-23T07:37:24.4402Z","shell.execute_reply":"2023-02-23T07:38:06.226242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we import all the packages we need and patch a function to allow for INT16 support","metadata":{"papermill":{"duration":0.010426,"end_time":"2023-02-22T01:57:21.268071","exception":false,"start_time":"2023-02-22T01:57:21.257645","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import timm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfrom copy import copy\nimport gc\nimport shutil \n\nimport glob\nfrom scipy.special import expit\n\nimport albumentations as A\nimport cv2\ncv2.setNumThreads(0)\n\nimport dicomsdl\nimport pydicom\nfrom pydicom.filebase import DicomBytesIO\n\nfrom os.path import join\n\nfrom tqdm import tqdm\n\nfrom joblib import Parallel, delayed\nimport multiprocessing as mp\n\nfrom types import SimpleNamespace\nfrom typing import Any, Dict\n\nimport torch\nimport torch.nn.functional as F\nfrom torch import nn\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.cuda.amp import GradScaler, autocast\n\n\nimport nvidia.dali.fn as fn\nimport nvidia.dali.types as types\nfrom nvidia.dali import pipeline_def\nfrom nvidia.dali.types import DALIDataType","metadata":{"papermill":{"duration":5.20679,"end_time":"2023-02-22T01:57:26.485272","exception":false,"start_time":"2023-02-22T01:57:21.278482","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:06.230649Z","iopub.execute_input":"2023-02-23T07:38:06.231282Z","iopub.status.idle":"2023-02-23T07:38:10.980847Z","shell.execute_reply.started":"2023-02-23T07:38:06.231241Z","shell.execute_reply":"2023-02-23T07:38:10.979784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we need to patch DALI for Int16 support\n\n\nfrom nvidia.dali.backend import TensorGPU, TensorListGPU\nfrom nvidia.dali.pipeline import Pipeline\nimport nvidia.dali.ops as ops\nfrom nvidia.dali import types\nfrom nvidia.dali.plugin.base_iterator import _DaliBaseIterator\nfrom nvidia.dali.plugin.base_iterator import LastBatchPolicy\nimport torch\nimport torch.utils.dlpack as torch_dlpack\nimport ctypes\nimport numpy as np\nimport torch.nn.functional as F\nimport pydicom\n\nto_torch_type = {\n    types.DALIDataType.FLOAT:   torch.float32,\n    types.DALIDataType.FLOAT64: torch.float64,\n    types.DALIDataType.FLOAT16: torch.float16,\n    types.DALIDataType.UINT8:   torch.uint8,\n    types.DALIDataType.INT8:    torch.int8,\n    types.DALIDataType.UINT16:  torch.int16,\n    types.DALIDataType.INT16:   torch.int16,\n    types.DALIDataType.INT32:   torch.int32,\n    types.DALIDataType.INT64:   torch.int64\n}\n\n\ndef feed_ndarray(dali_tensor, arr, cuda_stream=None):\n    \"\"\"\n    Copy contents of DALI tensor to PyTorch's Tensor.\n\n    Parameters\n    ----------\n    `dali_tensor` : nvidia.dali.backend.TensorCPU or nvidia.dali.backend.TensorGPU\n                    Tensor from which to copy\n    `arr` : torch.Tensor\n            Destination of the copy\n    `cuda_stream` : torch.cuda.Stream, cudaStream_t or any value that can be cast to cudaStream_t.\n                    CUDA stream to be used for the copy\n                    (if not provided, an internal user stream will be selected)\n                    In most cases, using pytorch's current stream is expected (for example,\n                    if we are copying to a tensor allocated with torch.zeros(...))\n    \"\"\"\n    dali_type = to_torch_type[dali_tensor.dtype]\n\n    assert dali_type == arr.dtype, (\"The element type of DALI Tensor/TensorList\"\n                                    \" doesn't match the element type of the target PyTorch Tensor: \"\n                                    \"{} vs {}\".format(dali_type, arr.dtype))\n    assert dali_tensor.shape() == list(arr.size()), \\\n        (\"Shapes do not match: DALI tensor has size {0}, but PyTorch Tensor has size {1}\".\n            format(dali_tensor.shape(), list(arr.size())))\n    cuda_stream = types._raw_cuda_stream(cuda_stream)\n\n    # turn raw int to a c void pointer\n    c_type_pointer = ctypes.c_void_p(arr.data_ptr())\n    if isinstance(dali_tensor, (TensorGPU, TensorListGPU)):\n        stream = None if cuda_stream is None else ctypes.c_void_p(cuda_stream)\n        dali_tensor.copy_to_external(c_type_pointer, stream, non_blocking=True)\n    else:\n        dali_tensor.copy_to_external(c_type_pointer)\n    return arr\n\n\n\n","metadata":{"papermill":{"duration":0.027617,"end_time":"2023-02-22T01:57:26.52392","exception":false,"start_time":"2023-02-22T01:57:26.496303","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:10.982305Z","iopub.execute_input":"2023-02-23T07:38:10.982893Z","iopub.status.idle":"2023-02-23T07:38:11.000346Z","shell.execute_reply.started":"2023-02-23T07:38:10.982857Z","shell.execute_reply":"2023-02-23T07:38:10.999302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next I set major variables which handle the public run and the re-run on the hidden test set, and also allow for simulating the size of the hidden test set by setting RAM_CHECK = True","metadata":{"papermill":{"duration":0.010484,"end_time":"2023-02-22T01:57:26.544909","exception":false,"start_time":"2023-02-22T01:57:26.534425","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Params\n\nCROP_IMAGE_1st = False\nCROP_IMAGE_2nd = True\n\nCOMP_FOLDER = '/kaggle/input/rsna-breast-cancer-detection/'\nDATA_FOLDER = COMP_FOLDER + 'test_images/'\n\nsample_submission = pd.read_csv(COMP_FOLDER + 'sample_submission.csv')\n\nPUBLIC_RUN = len(sample_submission) == 2\n\nN_CORES = mp.cpu_count()\nMIXED_PRECISION = False\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nRAM_CHECK = True\nDEBUG = True\n\ntest_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_df['cancer'] = 0 #dummy value\n\n\nif PUBLIC_RUN is False:\n    RAM_CHECK = False\n    DEBUG = False\n\nif RAM_CHECK is True:\n    test_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n    patient_filter = list(sorted((set(test_df.patient_id.unique()))))[:8000]\n    test_df = test_df[test_df.patient_id.isin(patient_filter)]\n    DATA_FOLDER = DATA_FOLDER.replace('test','train')\n\nif DEBUG is True:\n    test_df = test_df.head(1500)\n\ntest_df","metadata":{"papermill":{"duration":0.246848,"end_time":"2023-02-22T01:57:26.802133","exception":false,"start_time":"2023-02-22T01:57:26.555285","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.00351Z","iopub.execute_input":"2023-02-23T07:38:11.003779Z","iopub.status.idle":"2023-02-23T07:38:11.243946Z","shell.execute_reply.started":"2023-02-23T07:38:11.003755Z","shell.execute_reply":"2023-02-23T07:38:11.243034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Len df : {len(test_df)}')\ntest_df['patient_id'].nunique()","metadata":{"papermill":{"duration":0.023586,"end_time":"2023-02-22T01:57:26.836873","exception":false,"start_time":"2023-02-22T01:57:26.813287","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.245292Z","iopub.execute_input":"2023-02-23T07:38:11.245744Z","iopub.status.idle":"2023-02-23T07:38:11.25397Z","shell.execute_reply.started":"2023-02-23T07:38:11.245708Z","shell.execute_reply":"2023-02-23T07:38:11.252881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"fns\"] = test_df['patient_id'].astype(str) + '/' + test_df['image_id'].astype(str) + '.dcm'","metadata":{"papermill":{"duration":0.022434,"end_time":"2023-02-22T01:57:26.870906","exception":false,"start_time":"2023-02-22T01:57:26.848472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.255885Z","iopub.execute_input":"2023-02-23T07:38:11.256596Z","iopub.status.idle":"2023-02-23T07:38:11.267642Z","shell.execute_reply.started":"2023-02-23T07:38:11.256562Z","shell.execute_reply":"2023-02-23T07:38:11.266667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we define the function for GPU-based decoding using DALI and processing the dicom images","metadata":{"papermill":{"duration":0.010724,"end_time":"2023-02-22T01:57:26.892743","exception":false,"start_time":"2023-02-22T01:57:26.882019","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode='same')\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[:int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n    \n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n        \n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[1].imshow(image)\n        debug[1].set_title('X Offset')\n        vv_scale = H / vv.max() * 0.90\n        # Values\n        debug[1].plot(H - vv * vv_scale , c='red', label='vv')\n        # Threshold\n        debug[1].hlines(H - vv_threshold * vv_scale, 0, W -1, colors='orange', label='threshold')\n        # Max Value\n        debug[1].scatter(vv_argmax, H - vv[vv_argmax] * vv_scale, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[1].scatter(offset, H - vv[offset] * vv_scale, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[1].set_ylim(H, 0)\n        debug[1].legend()\n        debug[1].axis('off')\n        \n    return offset\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = int(image.shape[0] * 0.25) + vv[int(image.shape[0] * 0.25):int(image.shape[0] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n    \n    if isinstance(debug, np.ndarray):\n        debug[2].imshow(image)\n        debug[2].set_title('Y Bottom Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[2].plot(vv * vv_scale, np.arange(H), c='red', label='vv')\n        # Threshold\n        debug[2].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[2].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[2].scatter(vv[offset_bottom] * vv_scale, offset_bottom, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[2].legend()\n        debug[2].axis('off')\n            \n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[3].imshow(image)\n        debug[3].set_title('Y Top Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[3].plot(vv * vv_scale, np.arange(H) , c='red', label='vv')\n        # Threshold\n        debug[3].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[3].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[3].scatter(vv[offset_top] * vv_scale, offset_top, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[3].legend()\n        debug[3].axis('off')\n            \n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:,:x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n    \n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n        \n    # Crop Image\n    image = image[offset_bottom:offset_top:,:x_offset]\n        \n    return image","metadata":{"papermill":{"duration":0.039877,"end_time":"2023-02-22T01:57:26.943799","exception":false,"start_time":"2023-02-22T01:57:26.903922","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.269421Z","iopub.execute_input":"2023-02-23T07:38:11.269919Z","iopub.status.idle":"2023-02-23T07:38:11.297889Z","shell.execute_reply.started":"2023-02-23T07:38:11.269769Z","shell.execute_reply":"2023-02-23T07:38:11.296929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_dicom_to_jpg(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")  #<---- the jpeg2000 header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n            \n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.70':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\xff\\xd8\\xff\\xe0\")  #<---- the jpeg lossless header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jpg\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n            \n@pipeline_def\ndef jpg_decode_pipeline(jpgfiles):\n    jpegs, _ = fn.readers.file(files=jpgfiles)\n    images = fn.experimental.decoders.image(jpegs, device='mixed', output_type=types.ANY_DATA, dtype=DALIDataType.UINT16)\n    return images\n\ndef parse_window_element(elem):\n    if type(elem)==list:\n        return float(elem[0])\n    if type(elem)==str:\n        return float(elem)\n    if type(elem)==float:\n        return elem\n    if type(elem)==pydicom.dataelem.DataElement:\n        try:\n            return float(elem[0])\n        except:\n            return float(elem.value)\n    return None\n\ndef linear_window(data, center, width):\n    lower, upper = center - width // 2, center + width // 2\n    data = torch.clamp(data, min=lower, max=upper)\n    return data \n\ndef process_dicom(img, dicom):\n    try:\n        invert = getattr(dicom, \"PhotometricInterpretation\", None) == \"MONOCHROME1\"\n    except:\n        invert = False\n        \n    center = parse_window_element(dicom[\"WindowCenter\"]) \n    width = parse_window_element(dicom[\"WindowWidth\"])\n        \n    if (center is not None) & (width is not None):\n        img = linear_window(img, center, width)\n\n    img = (img - img.min()) / (img.max() - img.min())\n    if invert:\n        img = 1 - img\n    return img","metadata":{"papermill":{"duration":0.028329,"end_time":"2023-02-22T01:57:26.983178","exception":false,"start_time":"2023-02-22T01:57:26.954849","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.302873Z","iopub.execute_input":"2023-02-23T07:38:11.303534Z","iopub.status.idle":"2023-02-23T07:38:11.318743Z","shell.execute_reply.started":"2023-02-23T07:38:11.3035Z","shell.execute_reply":"2023-02-23T07:38:11.317823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.011007,"end_time":"2023-02-22T01:57:27.005124","exception":false,"start_time":"2023-02-22T01:57:26.994117","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cfg = SimpleNamespace(**{})\ncfg.img_size = 1024\ncfg.backbone = 'seresnext50_32x4d'\ncfg.pretrained=False\ncfg.in_channels = 1\ncfg.classes = ['cancer']\ncfg.batch_size = 8\ncfg.data_folder = \"/tmp/output/\"\ncfg.val_aug = A.CenterCrop(always_apply=False, p=1.0, height=cfg.img_size, width=cfg.img_size)\ncfg.device = DEVICE","metadata":{"papermill":{"duration":0.020708,"end_time":"2023-02-22T01:57:27.036775","exception":false,"start_time":"2023-02-22T01:57:27.016067","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.31995Z","iopub.execute_input":"2023-02-23T07:38:11.320868Z","iopub.status.idle":"2023-02-23T07:38:11.332889Z","shell.execute_reply.started":"2023-02-23T07:38:11.320832Z","shell.execute_reply":"2023-02-23T07:38:11.331921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will process the dicoms in chunks so the disk space does not become an issue. ","metadata":{"execution":{"iopub.execute_input":"2023-02-05T07:14:35.534267Z","iopub.status.busy":"2023-02-05T07:14:35.533224Z","iopub.status.idle":"2023-02-05T07:14:35.545681Z","shell.execute_reply":"2023-02-05T07:14:35.544691Z","shell.execute_reply.started":"2023-02-05T07:14:35.534231Z"},"papermill":{"duration":0.010758,"end_time":"2023-02-22T01:57:27.059281","exception":false,"start_time":"2023-02-22T01:57:27.048523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"SAVE_SIZE = int(cfg.img_size * 1.125)\nSAVE_FOLDER = cfg.data_folder\nos.makedirs(SAVE_FOLDER, exist_ok=True)\nN_CHUNKS = len(test_df[\"fns\"]) // 2000 if len(test_df[\"fns\"]) > 2000 else 1\nCHUNKS = [(len(test_df[\"fns\"]) / N_CHUNKS * k, len(test_df[\"fns\"]) / N_CHUNKS * (k + 1)) for k in range(N_CHUNKS)]\nCHUNKS = np.array(CHUNKS).astype(int)\nJPG_FOLDER = \"/tmp/jpg/\"","metadata":{"papermill":{"duration":0.021535,"end_time":"2023-02-22T01:57:27.091777","exception":false,"start_time":"2023-02-22T01:57:27.070242","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.334227Z","iopub.execute_input":"2023-02-23T07:38:11.334744Z","iopub.status.idle":"2023-02-23T07:38:11.344401Z","shell.execute_reply.started":"2023-02-23T07:38:11.334703Z","shell.execute_reply":"2023-02-23T07:38:11.343437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfor ttt, chunk in enumerate(CHUNKS):\n    print(f'chunk {ttt} of {len(CHUNKS)} chunks')\n    os.makedirs(JPG_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_jpg)(f'{DATA_FOLDER}/{img}', save_folder=JPG_FOLDER)\n        for img in test_df[\"fns\"].tolist()[chunk[0]: chunk[1]]\n    )\n    \n    jpgfiles = glob.glob(JPG_FOLDER + \"*.jpg\")\n\n\n    pipe = jpg_decode_pipeline(jpgfiles, batch_size=1, num_threads=2, device_id=0)\n    pipe.build()\n\n    for i, f in enumerate(tqdm(jpgfiles)):\n        \n        patient, dicom_id = f.split('/')[-1][:-4].split('_')\n        dicom = pydicom.dcmread(DATA_FOLDER + f\"/{patient}/{dicom_id}.dcm\")\n        try:\n            out = pipe.run()\n            # Dali -> Torch\n            img = out[0][0]\n            img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n            feed_ndarray(img, img_torch, cuda_stream=torch.cuda.current_stream(device=0))\n            img = img_torch.float()\n\n            \n            del out, img_torch\n\n            #apply dicom preprocessing\n            img = process_dicom(img, dicom)\n\n            #resize the torch image\n            img = F.interpolate(img.view(1, 1, img.size(0), img.size(1)), (SAVE_SIZE*2, SAVE_SIZE*2), mode=\"bilinear\")[0, 0]\n\n            img = (img * 255).clip(0,255).to(torch.uint8).cpu().numpy()\n            out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n            cv2.imwrite(out_file_name, img)\n\n            del out_file_name, img, dicom\n\n        except Exception as e:\n            print(i, e)\n            pipe = jpg_decode_pipeline(jpgfiles[i+1:], batch_size=1, num_threads=2, device_id=0)\n            pipe.build()\n            continue\n\n    shutil.rmtree(JPG_FOLDER)\nprint(f'DALI Raw image load complete')","metadata":{"papermill":{"duration":339.321501,"end_time":"2023-02-22T02:03:06.424379","exception":false,"start_time":"2023-02-22T01:57:27.102878","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:38:11.345712Z","iopub.execute_input":"2023-02-23T07:38:11.346183Z","iopub.status.idle":"2023-02-23T07:44:08.035468Z","shell.execute_reply.started":"2023-02-23T07:38:11.346148Z","shell.execute_reply":"2023-02-23T07:44:08.03178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fns = glob.glob(f'{SAVE_FOLDER}/*.png')\nn_saved = len(fns)\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.124494,"end_time":"2023-02-22T02:03:06.746432","exception":false,"start_time":"2023-02-22T02:03:06.621938","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.048006Z","iopub.execute_input":"2023-02-23T07:44:08.051631Z","iopub.status.idle":"2023-02-23T07:44:08.100388Z","shell.execute_reply.started":"2023-02-23T07:44:08.051567Z","shell.execute_reply":"2023-02-23T07:44:08.096375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A few hidden test set images might not be decoded via DALI, so we fallback to CPU for those","metadata":{"papermill":{"duration":0.065136,"end_time":"2023-02-22T02:03:06.876934","exception":false,"start_time":"2023-02-22T02:03:06.811798","status":"completed"},"tags":[]}},{"cell_type":"code","source":"gpu_processed_files = [fn.split('/')[-1].replace('_','/').replace('png','dcm') for fn in fns]\nto_process = [f for f in test_df[\"fns\"].values if f not in gpu_processed_files]\nlen(gpu_processed_files), len(to_process)","metadata":{"papermill":{"duration":0.105964,"end_time":"2023-02-22T02:03:07.047916","exception":false,"start_time":"2023-02-22T02:03:06.941952","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.110494Z","iopub.execute_input":"2023-02-23T07:44:08.114122Z","iopub.status.idle":"2023-02-23T07:44:08.152076Z","shell.execute_reply.started":"2023-02-23T07:44:08.114062Z","shell.execute_reply":"2023-02-23T07:44:08.151145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef process(f, save_folder=\"\"):\n    patient = f.split('/')[-2]\n    dicom_id = f.split('/')[-1][:-4]\n    \n    dicom = dicomsdl.open(f)\n    img = dicom.pixelData()\n    \n    img = torch.from_numpy(img)\n    img = process_dicom(img, dicom)\n    \n    img = F.interpolate(img.view(1, 1, img.size(0), img.size(1)), (SAVE_SIZE*2, SAVE_SIZE*2), mode=\"bilinear\")[0, 0]\n\n    img = (img * 255).clip(0,255).to(torch.uint8).cpu().numpy()\n    out_file_name = SAVE_FOLDER + f\"{patient}_{dicom_id}.png\"\n    cv2.imwrite(out_file_name, img)\n    return out_file_name","metadata":{"papermill":{"duration":0.077021,"end_time":"2023-02-22T02:03:07.189655","exception":false,"start_time":"2023-02-22T02:03:07.112634","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.153382Z","iopub.execute_input":"2023-02-23T07:44:08.154434Z","iopub.status.idle":"2023-02-23T07:44:08.162692Z","shell.execute_reply.started":"2023-02-23T07:44:08.154399Z","shell.execute_reply":"2023-02-23T07:44:08.161721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_processed_filenames = Parallel(n_jobs=2)(\n    delayed(process)(f'{DATA_FOLDER}/{img}', save_folder=SAVE_FOLDER)\n    for img in tqdm(to_process)\n)\ncpu_processed_filenames = [f for f in cpu_processed_filenames if f]\nprint(f'CPU Raw image load complete with {len(cpu_processed_filenames)} loaded')","metadata":{"papermill":{"duration":0.085704,"end_time":"2023-02-22T02:03:07.340772","exception":false,"start_time":"2023-02-22T02:03:07.255068","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.163831Z","iopub.execute_input":"2023-02-23T07:44:08.164662Z","iopub.status.idle":"2023-02-23T07:44:08.181658Z","shell.execute_reply.started":"2023-02-23T07:44:08.164621Z","shell.execute_reply":"2023-02-23T07:44:08.18055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.480798,"end_time":"2023-02-22T02:03:07.891169","exception":false,"start_time":"2023-02-22T02:03:07.410371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.183006Z","iopub.execute_input":"2023-02-23T07:44:08.183748Z","iopub.status.idle":"2023-02-23T07:44:08.486396Z","shell.execute_reply.started":"2023-02-23T07:44:08.183714Z","shell.execute_reply":"2023-02-23T07:44:08.485124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_saved = len(glob.glob(f'{SAVE_FOLDER}/*.png'))\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.080734,"end_time":"2023-02-22T02:03:08.038649","exception":false,"start_time":"2023-02-22T02:03:07.957915","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.487731Z","iopub.execute_input":"2023-02-23T07:44:08.488686Z","iopub.status.idle":"2023-02-23T07:44:08.526809Z","shell.execute_reply.started":"2023-02-23T07:44:08.488657Z","shell.execute_reply":"2023-02-23T07:44:08.525728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert n_saved == len(test_df)","metadata":{"papermill":{"duration":0.074316,"end_time":"2023-02-22T02:03:08.178642","exception":false,"start_time":"2023-02-22T02:03:08.104326","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.531258Z","iopub.execute_input":"2023-02-23T07:44:08.531691Z","iopub.status.idle":"2023-02-23T07:44:08.569507Z","shell.execute_reply.started":"2023-02-23T07:44:08.531642Z","shell.execute_reply":"2023-02-23T07:44:08.568359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We finished with preprocessing all the dicoms to images. So next, we set-up the dataloading and model","metadata":{"papermill":{"duration":0.064517,"end_time":"2023-02-22T02:03:08.309678","exception":false,"start_time":"2023-02-22T02:03:08.245161","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ndef batch_to_device(batch, device):\n    batch_dict = {key: batch[key].to(device) for key in batch}\n    return batch_dict\n\n\nclass CustomDataset(Dataset):\n    def __init__(self, df, cfg, aug):\n\n        self.cfg = cfg\n        self.df = df.copy()\n        self.df = self.df[self.df['image_id'].astype(str) != '1942326353']\n        self.labels = self.df[self.cfg.classes].values\n        self.df[\"fns\"] = self.df['patient_id'].astype(str) + '_' + self.df['image_id'].astype(str) + '.png'\n        self.fns = self.df[\"fns\"].astype(str).values\n        self.aug = aug\n        self.data_folder = cfg.data_folder\n\n    def __getitem__(self, idx):\n\n        label = self.labels[idx]\n        img = self.load_one(idx)\n\n        if self.aug:\n            img = self.augment(img)\n\n        img = self.normalize_img(img)\n        torch_img = torch.tensor(img).float().permute(2,0,1)\n        \n        feature_dict = {\n            \"input\": torch_img,\n            \"target\": torch.tensor(label),\n        }\n        return feature_dict\n\n    def __len__(self):\n        return len(self.fns)\n\n    def load_one(self, idx):\n        path = self.data_folder + self.fns[idx]\n        try:\n            img = cv2.imread(path, cv2.IMREAD_UNCHANGED)\n            shape = img.shape\n\n            if CROP_IMAGE_1st:\n                if len(img.shape) == 2:\n                    image_c = crop(img)\n                    if min(*image_c.shape) > 100:\n                        img = image_c\n        \n            img = cv2.resize(img, (SAVE_SIZE, SAVE_SIZE))\n            \n            if len(img.shape) == 2:\n                img = img[:,:,None]\n                    \n        except Exception as e:\n            print(e)\n        return img\n\n    def augment(self, img):\n        img = img.astype(np.float32)\n        transformed = self.aug(image=img)\n        trans_img = transformed[\"image\"]\n        return trans_img\n\n    def normalize_img(self, img):\n        img = img / 255\n        return img\n","metadata":{"papermill":{"duration":0.159071,"end_time":"2023-02-22T02:03:08.534714","exception":false,"start_time":"2023-02-22T02:03:08.375643","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.573481Z","iopub.execute_input":"2023-02-23T07:44:08.573776Z","iopub.status.idle":"2023-02-23T07:44:08.599604Z","shell.execute_reply.started":"2023-02-23T07:44:08.57375Z","shell.execute_reply":"2023-02-23T07:44:08.598509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (self.__class__.__name__  + f\"(p={self.p.data.tolist()[0]:.4f},eps={self.eps})\")\n\n\n\nclass Net(nn.Module):\n\n    def __init__(self, cfg: Any):\n        super(Net, self).__init__()\n\n        self.cfg = cfg\n        self.n_classes = len(cfg.classes)\n        self.backbone = timm.create_model(cfg.backbone, \n                                          pretrained=cfg.pretrained, \n                                          num_classes=0, \n                                          global_pool=\"\", \n                                          in_chans=self.cfg.in_channels)\n    \n        backbone_out = self.backbone.feature_info[-1]['num_chs']\n\n        self.global_pool = GeM(p_trainable=False)\n        self.head = torch.nn.Linear(backbone_out, self.n_classes)\n        self.loss_fn = nn.BCEWithLogitsLoss()\n\n    def forward(self, batch):\n\n        x = batch['input']\n\n        x = self.backbone(x)\n        x = self.global_pool(x)\n        x = x[:,:,0,0]\n\n        logits = self.head(x)\n        \n        \n        outputs = {}\n        \n        \n        if self.training:\n            loss = self.loss_fn(logits,batch[\"target\"].float())\n            outputs['loss'] = loss\n        else:\n            outputs[\"logits\"] = logits\n        \n \n\n        return outputs\n","metadata":{"papermill":{"duration":0.118273,"end_time":"2023-02-22T02:03:08.751229","exception":false,"start_time":"2023-02-22T02:03:08.632956","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.601254Z","iopub.execute_input":"2023-02-23T07:44:08.60164Z","iopub.status.idle":"2023-02-23T07:44:08.617341Z","shell.execute_reply.started":"2023-02-23T07:44:08.601578Z","shell.execute_reply":"2023-02-23T07:44:08.616069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dl(test_df, cfg):\n\n    test_ds = CustomDataset(test_df, cfg, cfg.val_aug)\n    test_dl = DataLoader(test_ds, shuffle=False, batch_size=cfg.batch_size, num_workers=N_CORES, pin_memory=True)\n\n    return test_dl, batch_to_device\n\ndef get_state_dict(sd_fp):\n    sd = torch.load(sd_fp, map_location=\"cpu\")['model']\n    sd = {k.replace(\"module.\", \"\"):v for k,v in sd.items()}\n    return sd\n\ndef get_nets(cfg,state_dicts):\n\n    nets = []\n\n    for i,state_dict in enumerate(state_dicts):\n        net = Net(cfg).eval().to(DEVICE)\n        print(\"loading dict\")\n        sd = get_state_dict(state_dict)\n        net.load_state_dict(sd, strict=True)\n        nets += [net]\n        del sd\n        gc.collect()\n    return nets","metadata":{"papermill":{"duration":0.180428,"end_time":"2023-02-22T02:03:09.027462","exception":false,"start_time":"2023-02-22T02:03:08.847034","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.619084Z","iopub.execute_input":"2023-02-23T07:44:08.619467Z","iopub.status.idle":"2023-02-23T07:44:08.632215Z","shell.execute_reply.started":"2023-02-23T07:44:08.619432Z","shell.execute_reply":"2023-02-23T07:44:08.631185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_dl, batch_to_device = get_dl(test_df, cfg)","metadata":{"papermill":{"duration":0.120658,"end_time":"2023-02-22T02:03:09.213472","exception":false,"start_time":"2023-02-22T02:03:09.092814","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.634062Z","iopub.execute_input":"2023-02-23T07:44:08.634464Z","iopub.status.idle":"2023-02-23T07:44:08.694922Z","shell.execute_reply.started":"2023-02-23T07:44:08.634429Z","shell.execute_reply":"2023-02-23T07:44:08.693975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"state_dicts = sorted(glob.glob('/kaggle/input/rsna-seresnext50-5fold/check*.pth'))\nprint(state_dicts)\n\nnets = get_nets(cfg,state_dicts)","metadata":{"papermill":{"duration":8.555511,"end_time":"2023-02-22T02:03:17.83493","exception":false,"start_time":"2023-02-22T02:03:09.279419","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:08.696667Z","iopub.execute_input":"2023-02-23T07:44:08.697099Z","iopub.status.idle":"2023-02-23T07:44:16.807443Z","shell.execute_reply.started":"2023-02-23T07:44:08.697061Z","shell.execute_reply":"2023-02-23T07:44:16.806488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Dataloader length : {len(sub_dl.dataset)}')","metadata":{"papermill":{"duration":0.074589,"end_time":"2023-02-22T02:03:17.975922","exception":false,"start_time":"2023-02-22T02:03:17.901333","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:16.808987Z","iopub.execute_input":"2023-02-23T07:44:16.809347Z","iopub.status.idle":"2023-02-23T07:44:16.81675Z","shell.execute_reply.started":"2023-02-23T07:44:16.809314Z","shell.execute_reply":"2023-02-23T07:44:16.815712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.inference_mode():\n\n    preds = [[] for i in range(len(nets))]\n    for batch in tqdm(sub_dl):\n        batch = batch_to_device(batch, cfg.device)\n        for i, net in enumerate(nets):\n            logits = net(batch)['logits'].sigmoid().float().detach().cpu().numpy()\n            preds[i] += [logits]\n            \npreds = np.array([np.concatenate(p, axis=0) for p in preds])\npreds = preds.mean(0) #average fold predictions\npreds = preds[:,0]\npreds.shape","metadata":{"papermill":{"duration":475.136992,"end_time":"2023-02-22T02:11:13.178025","exception":false,"start_time":"2023-02-22T02:03:18.041033","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:44:16.818082Z","iopub.execute_input":"2023-02-23T07:44:16.819012Z","iopub.status.idle":"2023-02-23T07:52:11.415918Z","shell.execute_reply.started":"2023-02-23T07:44:16.818977Z","shell.execute_reply":"2023-02-23T07:52:11.414867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape","metadata":{"papermill":{"duration":0.087148,"end_time":"2023-02-22T02:11:13.342569","exception":false,"start_time":"2023-02-22T02:11:13.255421","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:11.423795Z","iopub.execute_input":"2023-02-23T07:52:11.424122Z","iopub.status.idle":"2023-02-23T07:52:11.431738Z","shell.execute_reply.started":"2023-02-23T07:52:11.424092Z","shell.execute_reply":"2023-02-23T07:52:11.430722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del batch, logits, i, net, nets, batch_to_device, state_dicts, cpu_processed_filenames, ttt, chunk, jpgfiles, pipe, gpu_processed_files, to_process","metadata":{"papermill":{"duration":0.154248,"end_time":"2023-02-22T02:11:13.582128","exception":false,"start_time":"2023-02-22T02:11:13.42788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:11.433282Z","iopub.execute_input":"2023-02-23T07:52:11.433943Z","iopub.status.idle":"2023-02-23T07:52:11.521354Z","shell.execute_reply.started":"2023-02-23T07:52:11.433909Z","shell.execute_reply":"2023-02-23T07:52:11.519565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.365722,"end_time":"2023-02-22T02:11:14.022913","exception":false,"start_time":"2023-02-22T02:11:13.657191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:11.522733Z","iopub.execute_input":"2023-02-23T07:52:11.523197Z","iopub.status.idle":"2023-02-23T07:52:11.818672Z","shell.execute_reply.started":"2023-02-23T07:52:11.523151Z","shell.execute_reply":"2023-02-23T07:52:11.817541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom keras_cv_attention_models import convnext","metadata":{"papermill":{"duration":5.307072,"end_time":"2023-02-22T02:11:19.413654","exception":false,"start_time":"2023-02-22T02:11:14.106582","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:11.820735Z","iopub.execute_input":"2023-02-23T07:52:11.821136Z","iopub.status.idle":"2023-02-23T07:52:17.298123Z","shell.execute_reply.started":"2023-02-23T07:52:11.821101Z","shell.execute_reply":"2023-02-23T07:52:17.297139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TARGET_HEIGHT = 1344\nTARGET_WIDTH = 768\nN_CHANNELS = 1\nINPUT_SHAPE = (TARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS)\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH","metadata":{"papermill":{"duration":0.083144,"end_time":"2023-02-22T02:11:19.572174","exception":false,"start_time":"2023-02-22T02:11:19.48903","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:17.299448Z","iopub.execute_input":"2023-02-23T07:52:17.29981Z","iopub.status.idle":"2023-02-23T07:52:17.309951Z","shell.execute_reply.started":"2023-02-23T07:52:17.299777Z","shell.execute_reply":"2023-02-23T07:52:17.30794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def normalize(image):\n    # Repeat channels to create 3 channel images required by pretrained ConvNextV2 models\n    image = tf.repeat(image, repeats=3, axis=3)\n    # Cast to float 32\n    image = tf.cast(image, tf.float32)\n    # Normalize with respect to ImageNet mean/std\n    image = tf.keras.applications.imagenet_utils.preprocess_input(image, mode='torch')\n\n    return image","metadata":{"papermill":{"duration":0.130092,"end_time":"2023-02-22T02:11:19.777463","exception":false,"start_time":"2023-02-22T02:11:19.647371","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:17.31163Z","iopub.execute_input":"2023-02-23T07:52:17.31278Z","iopub.status.idle":"2023-02-23T07:52:17.323409Z","shell.execute_reply.started":"2023-02-23T07:52:17.312743Z","shell.execute_reply":"2023-02-23T07:52:17.322234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model():\n    # Inputs, note the names are equal to the dictionary keys in the dataset\n    image = tf.keras.layers.Input(INPUT_SHAPE, name='image', dtype=tf.uint8)\n\n    # Normalize Input\n    image_norm = normalize(image)\n\n    # CNN Feature Maps\n    x = convnext.ConvNeXtV2Tiny(\n        input_shape=(TARGET_HEIGHT, TARGET_WIDTH, 3),\n        pretrained=None,\n        num_classes=0,\n    )(image_norm)\n\n    # Average Pooling BxHxWxC -> BxC\n    x = tf.keras.layers.GlobalAveragePooling2D()(x)\n    # Dropout to prevent Overfitting\n    x = tf.keras.layers.Dropout(0.30)(x)\n    # Output value between [0, 1] using Sigmoid function\n    outputs = tf.keras.layers.Dense(1, activation='sigmoid')(x)\n\n    # Define model with inputs and outputs\n    model = tf.keras.models.Model(inputs=image, outputs=outputs)\n\n    # Load pretrained Model Weights\n    model.load_weights('/kaggle/input/rsna-efficientnetv2-training-tensorflow-tpu-ds/model.h5')\n\n    # Set model non-trainable\n    model.trainable = False\n\n    # Compile model\n    model.compile()\n\n    return model","metadata":{"papermill":{"duration":0.088205,"end_time":"2023-02-22T02:11:19.943771","exception":false,"start_time":"2023-02-22T02:11:19.855566","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:17.32523Z","iopub.execute_input":"2023-02-23T07:52:17.325498Z","iopub.status.idle":"2023-02-23T07:52:17.337586Z","shell.execute_reply.started":"2023-02-23T07:52:17.325474Z","shell.execute_reply":"2023-02-23T07:52:17.336528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pretrained File Path: '/kaggle/input/sartorius-training-dataset/model.h5'\ntf.keras.backend.clear_session()\n# enable XLA optmizations\ntf.config.optimizer.set_jit(True)\n\nmodel = get_model()","metadata":{"papermill":{"duration":7.450759,"end_time":"2023-02-22T02:11:27.469424","exception":false,"start_time":"2023-02-22T02:11:20.018665","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:17.339303Z","iopub.execute_input":"2023-02-23T07:52:17.340744Z","iopub.status.idle":"2023-02-23T07:52:24.391927Z","shell.execute_reply.started":"2023-02-23T07:52:17.340716Z","shell.execute_reply":"2023-02-23T07:52:24.390864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = sub_dl.dataset.df\nSUBMISSION_ROWS = []\n# Iterate over all patient_id/laterality combinations groups\nfor idx, ((patient_id, laterality), g) in enumerate(tqdm(test.groupby(['patient_id', 'laterality']))):\n    # Cancer target is mean of predicted cancer values\n    cancer = 0\n    # Iterate over all scans in group\n    for row_idx, row in g.iterrows():\n        # Load Image\n        image_id = row['image_id']\n        image = cv2.imread(f'/tmp/output/{patient_id}_{image_id}.png', -1)\n        \n        if CROP_IMAGE_2nd:\n            if len(image.shape) == 2:\n                image_c = crop(image)\n                if min(*image_c.shape) > 100:\n                    image = image_c\n        \n        image = cv2.resize(image, (TARGET_WIDTH, TARGET_HEIGHT))\n        \n        # Expand to Batch HxW -> 1xHxWx1\n        image = np.expand_dims(image, [0, 3])\n        # Make Prediction\n        cancer += model.predict_on_batch(image).squeeze() / len(g)\n                \n    # Add Submission Row\n    SUBMISSION_ROWS.append({\n        'prediction_id': f'{patient_id}_{laterality}',\n        'cancer': np.float32(cancer),\n    })\n    \n    if np.random.rand() > 0.99:\n        gc.collect()","metadata":{"papermill":{"duration":103.651909,"end_time":"2023-02-22T02:13:11.197469","exception":false,"start_time":"2023-02-22T02:11:27.54556","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:52:24.393641Z","iopub.execute_input":"2023-02-23T07:52:24.393995Z","iopub.status.idle":"2023-02-23T07:56:04.92015Z","shell.execute_reply.started":"2023-02-23T07:52:24.39396Z","shell.execute_reply":"2023-02-23T07:56:04.919138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub1_std = np.std(preds)\nsub1_mean = np.std(preds)\nsub2_std = np.std([r['cancer'] for r in SUBMISSION_ROWS])\nsub2_mean = np.std([r['cancer'] for r in SUBMISSION_ROWS])","metadata":{"execution":{"iopub.status.busy":"2023-02-23T07:56:04.922181Z","iopub.execute_input":"2023-02-23T07:56:04.922924Z","iopub.status.idle":"2023-02-23T07:56:04.930161Z","shell.execute_reply.started":"2023-02-23T07:56:04.922886Z","shell.execute_reply":"2023-02-23T07:56:04.929185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_2 = {r['prediction_id']:sub1_std*((r['cancer']-sub2_mean)/sub2_std)+sub1_mean for r in SUBMISSION_ROWS}","metadata":{"papermill":{"duration":0.116262,"end_time":"2023-02-22T02:13:11.42198","exception":false,"start_time":"2023-02-22T02:13:11.305718","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:56:04.93166Z","iopub.execute_input":"2023-02-23T07:56:04.932048Z","iopub.status.idle":"2023-02-23T07:56:04.941939Z","shell.execute_reply.started":"2023-02-23T07:56:04.931999Z","shell.execute_reply":"2023-02-23T07:56:04.940784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id = sub_dl.dataset.df['patient_id'].values\nlaterality = sub_dl.dataset.df['laterality'].values\n\nab_rate = 0.7459703039378771\nprediction_id = [f'{i}_{j}' for i,j in  zip(patient_id, laterality)]\npreds_marged = [sub_1*ab_rate + submission_2[p]*(1.0-ab_rate) for sub_1,p in zip(preds,prediction_id)]\n\npred_df = pd.DataFrame({'prediction_id': prediction_id, 'cancer_raw': preds_marged})\n\n#aggregate by prediction_id , i.e. by patient_laterality\nsub = pred_df.groupby('prediction_id')[['cancer_raw']].agg('mean')\n\n# binarize predictions\nth = np.quantile(sub['cancer_raw'].values,0.97935)\nsub['cancer'] = (sub['cancer_raw'].values > th).astype(int)","metadata":{"papermill":{"duration":0.139373,"end_time":"2023-02-22T02:13:11.882309","exception":false,"start_time":"2023-02-22T02:13:11.742936","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:56:04.943229Z","iopub.execute_input":"2023-02-23T07:56:04.944346Z","iopub.status.idle":"2023-02-23T07:56:04.971165Z","shell.execute_reply.started":"2023-02-23T07:56:04.94431Z","shell.execute_reply":"2023-02-23T07:56:04.970247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub[['cancer']].to_csv('submission.csv')","metadata":{"papermill":{"duration":0.120789,"end_time":"2023-02-22T02:13:12.111236","exception":false,"start_time":"2023-02-22T02:13:11.990447","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:56:04.972523Z","iopub.execute_input":"2023-02-23T07:56:04.973746Z","iopub.status.idle":"2023-02-23T07:56:04.984582Z","shell.execute_reply.started":"2023-02-23T07:56:04.973707Z","shell.execute_reply":"2023-02-23T07:56:04.983576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for debugging purpose we can calculate the pF1 score if we infered on the train data by setting RAM_CHECK=True in the beginning","metadata":{"papermill":{"duration":0.10705,"end_time":"2023-02-22T02:13:12.325822","exception":false,"start_time":"2023-02-22T02:13:12.218772","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if RAM_CHECK:\n\n    def pfbeta(labels, predictions, beta):\n        #official implementation\n        y_true_count = 0\n        ctp = 0\n        cfp = 0\n\n        for idx in range(len(labels)):\n            prediction = min(max(predictions[idx], 0), 1)\n            if (labels[idx]):\n                y_true_count += 1\n                ctp += prediction\n    #             cfp += 1 - prediction #bugfix\n            else:\n                cfp += prediction\n\n        beta_squared = beta * beta\n        c_precision = ctp / (ctp + cfp)\n        c_recall = ctp / y_true_count\n        if (c_precision > 0 and c_recall > 0):\n            result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n            return result\n        else:\n            return 0\n\n    #aggregate by prediction_id , i.e. by patient_laterality\n    test_df['prediction_id'] = test_df.apply(lambda x: f'{x.patient_id}_{x.laterality}', 1)\n    test_df_gr = test_df.groupby('prediction_id')[['cancer']].agg('mean')\n\n    # Sort both the same\n    test_df_gr = test_df_gr.loc[sub.index]\n\n    y = test_df_gr['cancer'].values#.astype(np.float32)\n    y_pred = sub['cancer'].values\n\n#     print(y.shape, y_pred.shape)\n\n    score = pfbeta(y, y_pred, 1)\n    print(th, score)","metadata":{"papermill":{"duration":0.157151,"end_time":"2023-02-22T02:13:12.596898","exception":false,"start_time":"2023-02-22T02:13:12.439747","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:56:04.986199Z","iopub.execute_input":"2023-02-23T07:56:04.98667Z","iopub.status.idle":"2023-02-23T07:56:05.033669Z","shell.execute_reply.started":"2023-02-23T07:56:04.986635Z","shell.execute_reply":"2023-02-23T07:56:05.032663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if RAM_CHECK:\n    import optuna\n    def opt(trial):\n        ab_rate = trial.suggest_uniform('ab_rate', 0.1, 0.9)\n        th_rate = trial.suggest_uniform('th_rate', 0.0, 0.1)\n\n        preds_marged = [sub_1*ab_rate + submission_2[p]*(1.0-ab_rate) for sub_1,p in zip(preds,prediction_id)]\n\n        pred_df = pd.DataFrame({'prediction_id': prediction_id, 'cancer_raw': preds_marged})\n\n        #aggregate by prediction_id , i.e. by patient_laterality\n        sub = pred_df.groupby('prediction_id')[['cancer_raw']].agg('mean')\n\n        # binarize predictions\n        th = np.quantile(sub['cancer_raw'].values,0.9+th_rate)\n        sub['cancer'] = (sub['cancer_raw'].values > th).astype(int)\n\n        y = test_df_gr['cancer'].values#.astype(np.float32)\n        y_pred = sub['cancer'].values\n\n        score = pfbeta(y, y_pred, 1)\n\n        return score\n    study = optuna.create_study(direction='maximize')\n    study.optimize(opt, n_trials=100)\n    print(study.best_params)","metadata":{"papermill":{"duration":2.644817,"end_time":"2023-02-22T02:13:15.349034","exception":false,"start_time":"2023-02-22T02:13:12.704217","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-02-23T07:56:05.035078Z","iopub.execute_input":"2023-02-23T07:56:05.035441Z","iopub.status.idle":"2023-02-23T07:56:07.309825Z","shell.execute_reply.started":"2023-02-23T07:56:05.035404Z","shell.execute_reply":"2023-02-23T07:56:07.308757Z"},"trusted":true},"execution_count":null,"outputs":[]}]}