{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# for dali\n!pip install -q /kaggle/input/nvidia-dali-nightly-cuda110-1230dev/nvidia_dali_nightly_cuda110-1.23.0.dev20230203-7187866-py3-none-manylinux2014_x86_64.whl\n\n# for tensorrt\n!pip install tensorrt --no-index --find-links \"/kaggle/input/tensorrt-packages\"\n\n# Source: https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster?scriptVersionId=113360473\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-14T13:15:23.606003Z","iopub.execute_input":"2023-02-14T13:15:23.60727Z","iopub.status.idle":"2023-02-14T13:18:02.819719Z","shell.execute_reply.started":"2023-02-14T13:15:23.607159Z","shell.execute_reply":"2023-02-14T13:18:02.818431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pylibjpeg\nimport pydicom\nfrom pydicom.filebase import DicomBytesIO\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\n# For DALI\nimport ctypes\nimport nvidia.dali.fn as fn\nfrom nvidia.dali import pipeline_def\nfrom nvidia.dali.backend import TensorGPU, TensorListGPU\nfrom nvidia.dali.pipeline import Pipeline\nimport nvidia.dali.ops as ops\nfrom nvidia.dali import types\nfrom nvidia.dali.types import DALIDataType\nfrom nvidia.dali.plugin.base_iterator import _DaliBaseIterator\nfrom nvidia.dali.plugin.base_iterator import LastBatchPolicy\n\n# For tensorrt\nimport tensorrt as trt\nimport cupy as cp\nfrom ctypes import cdll, c_char_p\nlibcudart = cdll.LoadLibrary('libcudart.so')\nlibcudart.cudaGetErrorString.restype = c_char_p\n\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\nimport cv2\nimport glob\nimport importlib\nimport os\nimport joblib\nimport time\nimport dicomsdl\nimport gc\nimport shutil","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:02.822666Z","iopub.execute_input":"2023-02-14T13:18:02.823327Z","iopub.status.idle":"2023-02-14T13:18:08.798519Z","shell.execute_reply.started":"2023-02-14T13:18:02.823283Z","shell.execute_reply":"2023-02-14T13:18:08.797572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpus = tf.config.list_physical_devices('GPU')\nif gpus:\n    try:\n        # Currently, memory growth needs to be the same across GPUs\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        logical_gpus = tf.config.list_logical_devices('GPU')\n        print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n    except RuntimeError as e:\n        # Memory growth must be set before GPUs have been initialized\n        print(e)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-14T13:18:08.799955Z","iopub.execute_input":"2023-02-14T13:18:08.800603Z","iopub.status.idle":"2023-02-14T13:18:12.190753Z","shell.execute_reply.started":"2023-02-14T13:18:08.800564Z","shell.execute_reply":"2023-02-14T13:18:12.189612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\nTARGET_HEIGHT = 1680\nTARGET_WIDTH = 960\nBATCH_SIZE = 8\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH\n\n# for DALI\nDATA_FOLDER = '/kaggle/input/rsna-breast-cancer-detection/test_images'\nCHUNK_SIZE = 64\nJPG_FOLDER = \"/tmp/jpg\"\n\nCLAHE = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(32, 32))\n\nCROP_IMAGE = True\nAPPLY_CLAHE = False\nAPPLY_EQ_HIST = False\n\nIMAGE_FORMAT = 'jpg'\n\n# save processed images to disk\nOUTPUT_DIR = '/tmp/images'\n\nos.makedirs(OUTPUT_DIR, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:12.192409Z","iopub.execute_input":"2023-02-14T13:18:12.195852Z","iopub.status.idle":"2023-02-14T13:18:12.206837Z","shell.execute_reply.started":"2023-02-14T13:18:12.195791Z","shell.execute_reply":"2023-02-14T13:18:12.206024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Image preprocessing","metadata":{}},{"cell_type":"code","source":"# for dali\n\ndef convert_dicom_to_jpg(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")  #<---- the jpeg2000 header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(f'{save_folder}/{patient}_{image}.jpg', \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n            \n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.70':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\xff\\xd8\\xff\\xe0\")  #<---- the jpeg lossless header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(f'{save_folder}/{patient}_{image}.jpg', \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n@pipeline_def\ndef jpg_decode_pipeline(jpgfiles):\n    jpegs, _ = fn.readers.file(\n        files=jpgfiles, \n        random_shuffle=False)\n    images = fn.experimental.decoders.image(\n        jpegs, \n        device='mixed', \n        output_type=types.ANY_DATA, \n        dtype=DALIDataType.UINT16)\n    \n    return images","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:12.21209Z","iopub.execute_input":"2023-02-14T13:18:12.212407Z","iopub.status.idle":"2023-02-14T13:18:12.22587Z","shell.execute_reply.started":"2023-02-14T13:18:12.21238Z","shell.execute_reply":"2023-02-14T13:18:12.224729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut\ndef voi_lut(image, dicom):\n    # Load only the variables we need\n    try:\n        center = dicom['00281050'].value\n    except:\n        print('No center detected!')\n        return image\n        \n    try:\n        width = dicom['00281051'].value\n    except:\n        print('No width detected!')\n        return image\n\n    try:\n        bits_stored = dicom['00280101'].value\n        bits_stored = int(bits_stored)\n    except:\n        print('No bits stored detected!')\n        bits_stored = 16 # assume highest\n        \n    try:\n        voi_lut_function = dicom['00281056'].value\n    except:\n        voi_lut_function = None\n    \n    # For sigmoid it's a list, otherwise a single value\n    if isinstance(center, pydicom.multival.MultiValue):\n        center = float(center[0])\n    else:\n        center = float(center)\n        \n    if isinstance(width, pydicom.multival.MultiValue):\n        width = float(width[0])\n    else:\n        width = float(width)\n\n    # Set y_min, max & range\n    y_min = 0\n    y_max = float(2**bits_stored - 1)\n    y_range = y_max\n\n    # Function with default LINEAR (so for Nan, it will use linear)\n    if voi_lut_function == 'SIGMOID':\n        image = y_range / (1 + np.exp(-4 * (image - center) / width)) + y_min\n    else:\n        # Checks width for < 1 (in our case not necessary, always >= 750)\n        center -= 0.5\n        width -= 1\n\n        below = image <= (center - width / 2)\n        above = image > (center + width / 2)\n        between = np.logical_and(~below, ~above)\n\n        image[below] = y_min\n        image[above] = y_max\n        if between.any():\n            image[between] = (\n                ((image[between] - center) / width + 0.5) * y_range + y_min\n            )\n\n    return image\n\n# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode='same')\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[:int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n    \n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n        \n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n        \n    return offset\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = int(image.shape[0] * 0.25) + vv[int(image.shape[0] * 0.25):int(image.shape[0] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n            \n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n            \n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:,:x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n    \n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n        \n    # Crop Image\n    image = image[offset_bottom:offset_top:,:x_offset]\n        \n    return image\n\ndef process(file_path, \n            image=None, \n            size=(TARGET_WIDTH, TARGET_HEIGHT), \n            crop_image=CROP_IMAGE, \n            apply_clahe=APPLY_CLAHE, \n            apply_eq_hist=APPLY_EQ_HIST, \n            debug=False, \n            save=True):\n    # Read Dicom File\n    dicom = pydicom.dcmread(file_path)\n    \n    if image is None:\n        print('Using CPU decoding!')\n        sdl = dicomsdl.open(file_path)\n        image = sdl.pixelData()\n    \n    # Save original image for debug purposes\n    if debug:\n        fig, axes = plt.subplots(1, 5, figsize=(20,10))\n        image0 = np.copy(image)\n        axes[0].imshow(image0)\n        axes[0].set_title('Original Image')\n        axes[0].axis('off')\n    else:\n        axes = False\n    \n    # voi_lut\n    try:\n        image = voi_lut(image, dicom)\n    except:\n        pass\n    \n    # Some images have 0 values as highest intensity and need to be inverted\n    if dicom['00280004'].value == 'MONOCHROME1':\n        image = np.max(image) - image\n\n    # Normalize [0,1] range\n    image = (image - image.min()) / (image.max() - image.min())\n\n    # Convert to uint8 image in range [0, 255]\n    image = (image * 255).astype(np.uint8)\n    \n    # Flip T0 Left/Right Orientation\n    h0, w0 = image.shape\n    if image[:,int(-w0 * 0.10):].sum() > image[:,:int(w0 * 0.10)].sum():\n        image = np.flip(image, axis=1)\n    \n    # Crop Image\n    if crop_image:\n        image = crop(image, size=size, debug=axes)\n        \n    # Resize\n    if size is not None:\n        # Pad black pixels to make square image\n        h, w = image.shape\n        if (h / w) > TARGET_HEIGHT_WIDTH_RATIO:\n            pad = int(h / TARGET_HEIGHT_WIDTH_RATIO - w)\n            image = np.pad(image, [[0,0], [0, pad]])\n            h, w = image.shape\n        else:\n            pad = int(0.50 * (w * TARGET_HEIGHT_WIDTH_RATIO - h))\n            image = np.pad(image, [[pad, pad], [0,0]])\n            h, w = image.shape\n        # Resize\n        image = cv2.resize(image, size, interpolation=cv2.INTER_AREA)\n        \n    # Apply CLAHE contrast enhancement\n    if apply_clahe:\n        image = CLAHE.apply(image)\n        \n     # Apply Histogram Equalization\n    if apply_eq_hist:\n        image = cv2.equalizeHist(image)\n        \n    # Show Processed Image    \n    if debug:\n        axes[4].imshow(image)\n        axes[4].set_title('Processed Image')\n        axes[4].axis('off')\n        plt.show()\n        \n    # Save Only\n    if save:\n        image_id = file_path.split('/')[-1].split('.')[0]\n        if IMAGE_FORMAT == 'png':\n            cv2.imwrite(f'{OUTPUT_DIR}/{image_id}.png', image)\n        else:\n            cv2.imwrite(f'{OUTPUT_DIR}/{image_id}.jpg', image, [cv2.IMWRITE_JPEG_QUALITY, 95])","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:12.227813Z","iopub.execute_input":"2023-02-14T13:18:12.228236Z","iopub.status.idle":"2023-02-14T13:18:12.263939Z","shell.execute_reply.started":"2023-02-14T13:18:12.228195Z","shell.execute_reply":"2023-02-14T13:18:12.263005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# dataset","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\ndef get_file_path(args):\n    patient_id, image_id = args\n    return f'{DATA_FOLDER}/{patient_id}/{image_id}.dcm'\n    \ntest['file_path'] = test[['patient_id', 'image_id']].apply(get_file_path, axis=1)\n\n# remove views with too few samples\nvalid_views = {'MLO', 'CC'}\nis_valid_view = test['view'].isin(valid_views)\ntest = test.loc[is_valid_view, :]\n\ntest = test.reset_index()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-14T13:18:12.267467Z","iopub.execute_input":"2023-02-14T13:18:12.267832Z","iopub.status.idle":"2023-02-14T13:18:12.308434Z","shell.execute_reply.started":"2023-02-14T13:18:12.267798Z","shell.execute_reply":"2023-02-14T13:18:12.307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DALI preprocess","metadata":{}},{"cell_type":"code","source":"# preprocess images in chunks for disk space\ndef decode(gpu_id, jpgfiles):    \n    # build dali decoding pipeline\n    pipe = jpg_decode_pipeline(jpgfiles, batch_size=1, num_threads=2, device_id=gpu_id)\n    pipe.build()\n    \n    for i, f in enumerate(jpgfiles):\n        patient, dicom_id = f.split('/')[-1][:-4].split('_')\n        dicom_path = f'{DATA_FOLDER}/{patient}/{dicom_id}.dcm'\n        try:\n            out = pipe.run()\n            img = out[0].as_cpu()\n            img = img.as_array()\n            img = img.squeeze()\n            img = img.astype(np.float32)\n            process(dicom_path, image=img)\n        except Exception as e:\n            print(i, e)\n            pipe = jpg_decode_pipeline(jpgfiles[i+1:], batch_size=1, num_threads=2, device_id=gpu_id)\n            pipe.build()\n            process(dicom_path)\n            continue\n            \ndef process_chunk(chunk):\n    filepaths = chunk[['patient_id', 'image_id']].apply(\n        lambda x: f'{DATA_FOLDER}/{x[0]}/{x[1]}.dcm', \n        axis=1\n    )\n    \n    # write dicom to jpg files\n    os.makedirs(JPG_FOLDER, exist_ok=True)\n    \n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_jpg)(fp, save_folder=JPG_FOLDER)\n        for fp in filepaths\n    )\n    print('Converted dicom to jpg')\n    \n    # decode jpg files    \n    jpgfiles = glob.glob(JPG_FOLDER + \"/*.jpg\")\n    print('Number of jpgfiles found: ', len(jpgfiles))\n    \n    split_point = len(jpgfiles) // 2\n    jpgfiles_1 = jpgfiles[:split_point]\n    jpgfiles_2 = jpgfiles[split_point:]\n    \n    _ = Parallel(n_jobs=2, verbose=True, prefer='threads')(\n        delayed(decode)(gpu_id, files)\n        for gpu_id, files in [(0, jpgfiles_1), (1, jpgfiles_2)]\n    )\n    \n    shutil.rmtree(JPG_FOLDER)\n    print('Decoded jpgs!')","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:12.309516Z","iopub.execute_input":"2023-02-14T13:18:12.310281Z","iopub.status.idle":"2023-02-14T13:18:12.322282Z","shell.execute_reply.started":"2023-02-14T13:18:12.310256Z","shell.execute_reply":"2023-02-14T13:18:12.321037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor i in range(0, len(test), CHUNK_SIZE):\n    process_chunk(test.iloc[i:i+CHUNK_SIZE])","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-14T13:18:12.324042Z","iopub.execute_input":"2023-02-14T13:18:12.324709Z","iopub.status.idle":"2023-02-14T13:18:15.357674Z","shell.execute_reply.started":"2023-02-14T13:18:12.324668Z","shell.execute_reply":"2023-02-14T13:18:15.356291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inference pipeline","metadata":{}},{"cell_type":"code","source":"def get_context(device_idx, eng_path):\n    # set the cuda device to create the context on\n    ret = libcudart.cudaSetDevice(device_idx)\n    if ret != 0:\n        error_string = libcudart.cudaGetErrorString(ret)\n        raise RuntimeError(\"cudaSetDevice: \" + error_string)\n    \n    # load engine and create the tensorrt context from the engine\n    logger = trt.Logger(min_severity=trt.ILogger.WARNING)\n    runtime = trt.Runtime(logger)\n    serialized_engine = open(eng_path, 'rb').read()\n    engine = runtime.deserialize_cuda_engine(serialized_engine)\n    context = engine.create_execution_context()\n    \n    return context\n            \n# create tensorflow pipeline for loading images\ndef load_img(image_path):\n    image = tf.io.read_file(image_path)\n    image = tf.io.decode_jpeg(image)\n    return image\n\ndef pipeline(image_path):\n    image = load_img(image_path)\n    return image\n\ndef get_tf_pipeline(img_paths):\n    test_ds = tf.data.Dataset.from_tensor_slices(img_paths)\n    test_ds = test_ds.map(\n        pipeline, \n        num_parallel_calls=tf.data.AUTOTUNE).batch(BATCH_SIZE)\n    return test_ds\n\ndef get_preds(context, gpu_id, test_ds):\n    # set gpu to allocate data to for cupy\n    with cp.cuda.Device(gpu_id):\n        all_preds = []\n        for batch in test_ds:\n            batch = batch.numpy()\n            bs = len(batch)\n            if bs != BATCH_SIZE:\n                batch = np.pad(batch, ((0,BATCH_SIZE-len(batch)),(0,0),(0,0),(0,0)), 'constant', constant_values=0)\n            batch = batch.astype(np.float32)\n            \n            # Allocate memory on the GPU\n            input_data = cp.array(batch, dtype=np.float32)\n            output_data = cp.empty((BATCH_SIZE, 1), dtype=np.float32)\n            bindings = [input_data.data.ptr, output_data.data.ptr]\n\n            # Start inference\n            context.execute_v2(bindings)\n            preds = output_data.get()\n            all_preds.extend(preds.squeeze()[:bs])\n            \n    return all_preds\n\ndef run_inference_on_gpu(gpu_id, img_paths):\n    # create tensorflow image loading pipeline for images\n    test_ds = get_tf_pipeline(img_paths)\n    \n    all_preds = []\n    threshold_b3 = [0.30, 0.24, 0.29, 0.44, 0.17]\n    threshold_b4 = [0.77, 0.42, 0.56, 0.25, 0.13]\n    \n\n    for i in range(5):\n        context = get_context(gpu_id, f'/kaggle/input/rsna-mamo-effnetv1b3-trt/trt_model_fold_{i}.eng')\n        context_preds = get_preds(context, gpu_id, test_ds)\n        b3_preds = (np.array(context_preds) > threshold_b3[i]).astype(float)\n        \n        context = get_context(gpu_id, f'/kaggle/input/rsna-mamo-effnetv1b4-trt/trt_model_fold_{i}.eng')\n        context_preds = get_preds(context, gpu_id, test_ds)\n        b4_preds = (np.array(context_preds) > threshold_b4[i]).astype(float)\n        \n        all_preds.append(np.array(b3_preds) + np.array(b4_preds))\n    \n    # mean aggregation \n    all_preds = np.vstack(np.array(all_preds))\n    all_preds = all_preds.mean(0)\n    all_preds = (all_preds > 1.0).astype(float)\n\n    return all_preds","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:15.360049Z","iopub.execute_input":"2023-02-14T13:18:15.360753Z","iopub.status.idle":"2023-02-14T13:18:15.377433Z","shell.execute_reply.started":"2023-02-14T13:18:15.360703Z","shell.execute_reply":"2023-02-14T13:18:15.376345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Divide the image paths into halves\ntest['processed_img_path'] = test['image_id'].apply(lambda x: f'{OUTPUT_DIR}/{x}.{IMAGE_FORMAT}')\nsplit_point = len(test['processed_img_path']) // 2\nimg_paths_1 = test['processed_img_path'][:split_point].values\nimg_paths_2 = test['processed_img_path'][split_point:].values","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:15.378732Z","iopub.execute_input":"2023-02-14T13:18:15.37992Z","iopub.status.idle":"2023-02-14T13:18:15.575205Z","shell.execute_reply.started":"2023-02-14T13:18:15.379868Z","shell.execute_reply":"2023-02-14T13:18:15.573972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jobs = [joblib.delayed(run_inference_on_gpu)(gpu_id, img_paths) \n        for gpu_id, img_paths in [(0, img_paths_1), (1, img_paths_2)]]\npreds = joblib.Parallel(\n    n_jobs=2,\n    verbose=True,\n    prefer='threads',\n)(jobs)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:15.577033Z","iopub.execute_input":"2023-02-14T13:18:15.577694Z","iopub.status.idle":"2023-02-14T13:18:37.551097Z","shell.execute_reply.started":"2023-02-14T13:18:15.577657Z","shell.execute_reply":"2023-02-14T13:18:37.549959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_preds = np.concatenate(preds)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-02-14T13:18:37.552323Z","iopub.execute_input":"2023-02-14T13:18:37.553448Z","iopub.status.idle":"2023-02-14T13:18:37.558896Z","shell.execute_reply.started":"2023-02-14T13:18:37.553415Z","shell.execute_reply":"2023-02-14T13:18:37.557689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(20,5))\n# plt.plot(all_preds)\n# plt.plot(test.cancer*0.8, alpha=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:37.563019Z","iopub.execute_input":"2023-02-14T13:18:37.563305Z","iopub.status.idle":"2023-02-14T13:18:37.568305Z","shell.execute_reply.started":"2023-02-14T13:18:37.563279Z","shell.execute_reply":"2023-02-14T13:18:37.567127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['cancer'] = all_preds\ntest['cancer'] = test['cancer'].fillna(0)\ngrouped_preds = test.groupby(['prediction_id'])['cancer'].agg(max)\n\nsubmission_df = grouped_preds.reset_index()\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:37.569372Z","iopub.execute_input":"2023-02-14T13:18:37.570031Z","iopub.status.idle":"2023-02-14T13:18:37.589254Z","shell.execute_reply.started":"2023-02-14T13:18:37.569992Z","shell.execute_reply":"2023-02-14T13:18:37.588226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"execution":{"iopub.status.busy":"2023-02-14T13:18:37.591037Z","iopub.execute_input":"2023-02-14T13:18:37.591435Z","iopub.status.idle":"2023-02-14T13:18:37.607749Z","shell.execute_reply.started":"2023-02-14T13:18:37.591397Z","shell.execute_reply":"2023-02-14T13:18:37.606635Z"},"trusted":true},"execution_count":null,"outputs":[]}]}