{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n!pip install -q -r /kaggle/input/rsna-breast-cancer-detection-github/RSNA-Breast-Cancer-Detection-main/requirements_sub.txt --no-index --find-links=/kaggle/input/rsna-pip-wheels\n!pip install /kaggle/input/nvidia-dali-wheel/nvidia_dali_nightly_cuda110-1.22.0.dev20221213-6757685-py3-none-manylinux2014_x86_64.whl\n!pip uninstall -y timm","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":110.725887,"end_time":"2023-02-24T20:33:30.446041","exception":false,"start_time":"2023-02-24T20:31:39.720154","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:22:25.933614Z","iopub.execute_input":"2023-03-13T19:22:25.934023Z","iopub.status.idle":"2023-03-13T19:23:00.390679Z","shell.execute_reply.started":"2023-03-13T19:22:25.933979Z","shell.execute_reply":"2023-03-13T19:23:00.389117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install /kaggle/input/rsna-pip-wheels/timm-0.8.11.dev0-py3-none-any.whl","metadata":{"papermill":{"duration":30.317382,"end_time":"2023-02-24T20:34:00.773758","exception":false,"start_time":"2023-02-24T20:33:30.456376","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:00.393089Z","iopub.execute_input":"2023-03-13T19:23:00.39342Z","iopub.status.idle":"2023-03-13T19:23:30.578067Z","shell.execute_reply.started":"2023-03-13T19:23:00.393388Z","shell.execute_reply":"2023-03-13T19:23:30.576753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install -q dicomsdl --no-index --find-links=/kaggle/input/rsnabc-pip-wheel","metadata":{"papermill":{"duration":9.422369,"end_time":"2023-02-24T20:34:10.206547","exception":false,"start_time":"2023-02-24T20:34:00.784178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:30.57987Z","iopub.execute_input":"2023-03-13T19:23:30.580279Z","iopub.status.idle":"2023-03-13T19:23:40.085274Z","shell.execute_reply.started":"2023-03-13T19:23:30.580237Z","shell.execute_reply":"2023-03-13T19:23:40.083884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nGITHUB_PATH='/kaggle/input/rsna-breast-cancer-detection-github/RSNA-Breast-Cancer-Detection-main'\nsys.path.append(f'{GITHUB_PATH}/configs')\nsys.path.append(f'{GITHUB_PATH}/data')\nsys.path.append(f'{GITHUB_PATH}/models')\nsys.path.append(f'{GITHUB_PATH}/scripts')\nsys.path.append(f'{GITHUB_PATH}/postprocess')","metadata":{"papermill":{"duration":0.020439,"end_time":"2023-02-24T20:34:10.238058","exception":false,"start_time":"2023-02-24T20:34:10.217619","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:40.088595Z","iopub.execute_input":"2023-03-13T19:23:40.09016Z","iopub.status.idle":"2023-03-13T19:23:40.096693Z","shell.execute_reply.started":"2023-03-13T19:23:40.090107Z","shell.execute_reply":"2023-03-13T19:23:40.095656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport sys\nimport importlib\nfrom copy import copy\nimport gc\nimport shutil \nimport torch\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\n\nimport multiprocessing as mp\n# from tqdm import tqdm\nimport glob\nfrom scipy.special import expit\n\nimport albumentations as A\nimport cv2\nimport dicomsdl\n\ncv2.setNumThreads(0)\n\nfrom os.path import join\nfrom torch.cuda.amp import GradScaler, autocast\n\nimport pydicom\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\nimport nvidia.dali.fn as fn\nimport nvidia.dali.types as types\nfrom nvidia.dali import pipeline_def\nfrom nvidia.dali.types import DALIDataType\nfrom pydicom.filebase import DicomBytesIO\nfrom kernel_utils import feed_ndarray, to_torch_type, FilterImage, parse_window_element, linear_window\nfrom PIL import Image\n\n#timm.__version__, cv2.__version__","metadata":{"papermill":{"duration":4.146592,"end_time":"2023-02-24T20:34:14.394764","exception":false,"start_time":"2023-02-24T20:34:10.248172","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:40.098291Z","iopub.execute_input":"2023-03-13T19:23:40.098654Z","iopub.status.idle":"2023-03-13T19:23:45.871769Z","shell.execute_reply.started":"2023-03-13T19:23:40.098608Z","shell.execute_reply":"2023-03-13T19:23:45.870702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COMP_FOLDER = '/kaggle/input/rsna-breast-cancer-detection/'\nDATA_FOLDER = COMP_FOLDER + 'test_images/'\n\nsample_submission = pd.read_csv(COMP_FOLDER + 'sample_submission.csv')\nsample_submission","metadata":{"papermill":{"duration":0.039787,"end_time":"2023-02-24T20:34:14.444757","exception":false,"start_time":"2023-02-24T20:34:14.40497","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:45.873192Z","iopub.execute_input":"2023-03-13T19:23:45.87355Z","iopub.status.idle":"2023-03-13T19:23:45.899468Z","shell.execute_reply.started":"2023-03-13T19:23:45.873507Z","shell.execute_reply":"2023-03-13T19:23:45.898544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Params\n\nCOMP_FOLDER = '/kaggle/input/rsna-breast-cancer-detection/'\nDATA_FOLDER = COMP_FOLDER + 'test_images/'\n\nsample_submission = pd.read_csv(COMP_FOLDER + 'sample_submission.csv')\n\nPUBLIC_RUN = len(sample_submission) == 2\n\nN_CORES = mp.cpu_count()\nMIXED_PRECISION = False\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nFILTER_IMAGES = True\n\nRAM_CHECK = False\nOOF_CHECK = False\nDEBUG = False\n\ntest_df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\ntest_df['cancer'] = 0\n\n\nif PUBLIC_RUN is False:\n    RAM_CHECK = False\n    OOF_CHECK = False\n    DEBUG = False\n\nif RAM_CHECK is True:\n    #tbd\n    pass\n    \nif OOF_CHECK is True:\n    test_df = pd.read_csv('/kaggle/input/rsnabc-aws-data/train_folded_v01.csv')\n    #patient_filter = list(sorted((set(test_df.patient_id.unique()))))[:100]\n    #test_df = test_df[test_df.patient_id.isin(patient_filter)]\n    test_df = test_df[test_df['fold']==0].copy()\n    DATA_FOLDER = DATA_FOLDER.replace('test','train')\n\nif DEBUG is True:\n    # test_df = test_df.head(100)\n    #tbd\n    pass\n\ntest_df","metadata":{"papermill":{"duration":0.038383,"end_time":"2023-02-24T20:34:14.493257","exception":false,"start_time":"2023-02-24T20:34:14.454874","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:45.900956Z","iopub.execute_input":"2023-03-13T19:23:45.901577Z","iopub.status.idle":"2023-03-13T19:23:45.992984Z","shell.execute_reply.started":"2023-03-13T19:23:45.901536Z","shell.execute_reply":"2023-03-13T19:23:45.991917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Len df : {len(test_df)}')","metadata":{"papermill":{"duration":0.019215,"end_time":"2023-02-24T20:34:14.55466","exception":false,"start_time":"2023-02-24T20:34:14.535445","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:45.994276Z","iopub.execute_input":"2023-03-13T19:23:45.994631Z","iopub.status.idle":"2023-03-13T19:23:46.001011Z","shell.execute_reply.started":"2023-03-13T19:23:45.994593Z","shell.execute_reply":"2023-03-13T19:23:45.999815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"fns\"] = test_df['patient_id'].astype(str) + '/' + test_df['image_id'].astype(str) + '.dcm'","metadata":{"papermill":{"duration":0.018849,"end_time":"2023-02-24T20:34:14.5846","exception":false,"start_time":"2023-02-24T20:34:14.565751","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:46.002524Z","iopub.execute_input":"2023-03-13T19:23:46.003678Z","iopub.status.idle":"2023-03-13T19:23:46.011696Z","shell.execute_reply.started":"2023-03-13T19:23:46.00362Z","shell.execute_reply":"2023-03-13T19:23:46.010466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_dicom_to_j2k(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")  #<---- the jpeg2000 header info we're looking for\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jp2\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n            \n@pipeline_def\ndef j2k_decode_pipeline(j2kfiles):\n    jpegs, _ = fn.readers.file(files=j2kfiles)\n    images = fn.experimental.decoders.image(jpegs, device='mixed', output_type=types.ANY_DATA, dtype=DALIDataType.UINT16)\n    return images\n\ndef process_dicom(img, dicom):\n    try:\n        invert = getattr(dicom, \"PhotometricInterpretation\", None) == \"MONOCHROME1\"\n    except:\n        invert = False\n    try:\n        voi_func = getattr(dicom, \"VOILUTFunction\", \"LINEAR\")\n        if voi_func is None:\n            voi_func = 'LINEAR'\n        voi_func = voi_func.strip().upper()\n    except:\n        voi_func = 'LINEAR'\n\n    center = parse_window_element(dicom[\"WindowCenter\"])\n    width = parse_window_element(dicom[\"WindowWidth\"])\n    if (voi_func in [\"LINEAR\",\"SIGMOID\"]) & (center is not None) & (width is not None):\n        img = linear_window(img, center, width)\n\n    img = (img - img.min()) / (img.max() - img.min())\n    if invert:\n        img = 1 - img\n    return img","metadata":{"papermill":{"duration":0.02517,"end_time":"2023-02-24T20:34:14.620052","exception":false,"start_time":"2023-02-24T20:34:14.594882","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:46.017318Z","iopub.execute_input":"2023-03-13T19:23:46.01758Z","iopub.status.idle":"2023-03-13T19:23:46.029734Z","shell.execute_reply.started":"2023-03-13T19:23:46.017554Z","shell.execute_reply":"2023-03-13T19:23:46.028543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SAVE_FOLDER = \"/tmp/output/\"\nos.makedirs(SAVE_FOLDER, exist_ok=True)\nN_CHUNKS = len(test_df[\"fns\"]) // 2000 if len(test_df[\"fns\"]) > 2000 else 1\nCHUNKS = [(len(test_df[\"fns\"]) / N_CHUNKS * k, len(test_df[\"fns\"]) / N_CHUNKS * (k + 1)) for k in range(N_CHUNKS)]\nCHUNKS = np.array(CHUNKS).astype(int)\nJ2K_FOLDER = \"/tmp/j2k/\"","metadata":{"papermill":{"duration":0.020168,"end_time":"2023-02-24T20:34:14.65041","exception":false,"start_time":"2023-02-24T20:34:14.630242","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:46.031513Z","iopub.execute_input":"2023-03-13T19:23:46.032224Z","iopub.status.idle":"2023-03-13T19:23:46.044139Z","shell.execute_reply.started":"2023-03-13T19:23:46.032185Z","shell.execute_reply":"2023-03-13T19:23:46.043227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if FILTER_IMAGES:\n    filt = FilterImage(anchor_size = 1024, kernel_size = 13)\n    filt.to('cuda')\n    filt.kernel = filt.kernel.to('cuda')\n    rle_x_dict = {}\n    rle_y_dict = {}","metadata":{"papermill":{"duration":2.299014,"end_time":"2023-02-24T20:34:16.959508","exception":false,"start_time":"2023-02-24T20:34:14.660494","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:46.04568Z","iopub.execute_input":"2023-03-13T19:23:46.046159Z","iopub.status.idle":"2023-03-13T19:23:48.81493Z","shell.execute_reply.started":"2023-03-13T19:23:46.046021Z","shell.execute_reply":"2023-03-13T19:23:48.813518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class cfg:\n    image_width = 1024\n    image_height = 1024\n    \nval_aug = A.Compose([\n    A.CenterCrop(width=cfg.image_width, height=cfg.image_height, p=1.0)\n])\n\ndef pad_sides(img, padsize):\n    paddim = int(img.shape[0] > img.shape[1])\n    if paddim == 1:\n        padimg = np.zeros((img.shape[0], padsize//2), dtype = np.uint8)\n    else:\n        padimg = np.zeros((padsize//2, img.shape[1]), dtype = np.uint8)\n    img = np.concatenate((padimg, img, padimg), paddim)\n    return img\n\ndef resize_image(img, x_idx, y_idx, cfg = cfg):\n    # x_idx = filt.rldecode(*rle_x_dict[f.split('/')[-1]])\n    # y_idx = filt.rldecode(*rle_y_dict[f.split('/')[-1]])\n    \n    img = img[y_idx>0][:, x_idx>0]\n    image_width=cfg.image_width\n    image_height = cfg.image_height\n    \n    if max(img.shape) < image_width:\n        #img = img[:, :, 0]\n        hh, ww = img.shape\n        scale = image_width * 1.125 / max(hh, ww)\n        new_hh, new_ww = int(round(hh * scale)), int(round(ww * scale))\n        img = Image.fromarray(img).resize((new_ww, new_hh), Image.Resampling.LANCZOS)\n        img = np.array(img)\n        img = img[:, :, np.newaxis]\n        \n    # Pad, augment and resize large image\n    #img = img[:, :, 0]\n    padsize = abs(img.shape[0] - img.shape[1])\n    if padsize > 2:\n        img = pad_sides(img, padsize//5)\n        \n    img = Image.fromarray(img)\n    assert image_width == image_height\n    new_size = int(image_width * 1.125)\n    img.thumbnail((new_size, new_size), Image.Resampling.LANCZOS)\n    img = np.array(img)\n    \n    padsize = abs(img.shape[0] - img.shape[1])\n    if padsize > 2:\n        img = pad_sides(img, padsize)\n        \n    img = img[:,:,np.newaxis]\n    img = val_aug(image=img)[\"image\"]\n            \n    return img","metadata":{"papermill":{"duration":0.027791,"end_time":"2023-02-24T20:34:16.997985","exception":false,"start_time":"2023-02-24T20:34:16.970194","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:48.816496Z","iopub.execute_input":"2023-03-13T19:23:48.818148Z","iopub.status.idle":"2023-03-13T19:23:48.831459Z","shell.execute_reply.started":"2023-03-13T19:23:48.818098Z","shell.execute_reply":"2023-03-13T19:23:48.830336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for ttt, chunk in enumerate(tqdm(CHUNKS)):\n    print(f'chunk {ttt} of {len(CHUNKS)} chunks')\n    os.makedirs(J2K_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_j2k)(f'{DATA_FOLDER}/{img}', save_folder=J2K_FOLDER)\n        for img in test_df[\"fns\"].tolist()[chunk[0]: chunk[1]]\n    )\n    \n    j2kfiles = glob.glob(J2K_FOLDER + \"*.jp2\")\n\n    if not len(j2kfiles):\n        continue\n\n    pipe = j2k_decode_pipeline(j2kfiles, batch_size=1, num_threads=2, device_id=0, debug=True)\n    pipe.build()\n\n    for i, f in enumerate(tqdm(j2kfiles)):\n        patient, image = f.split('/')[-1][:-4].split('_')\n        dicom = pydicom.dcmread(DATA_FOLDER + f\"/{patient}/{image}.dcm\")\n\n        out = pipe.run()\n\n        # Dali -> Torch\n        img = out[0][0]\n        img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n        feed_ndarray(img, img_torch, cuda_stream=torch.cuda.current_stream(device=0))\n        img = img_torch.float()\n        img = process_dicom(img, dicom)\n\n        img = (img * 255).clip(0,255).to(torch.uint8)\n        \n        # f\"{row.patient_id}_{row.laterality}_{row['view']}_{row.image_id}.png\"\n        out_file_name = SAVE_FOLDER + f\"{patient}_{image}.png\"\n        if FILTER_IMAGES:\n            dkey = out_file_name.split('/')[-1]\n            x_idx, y_idx = filt(img[:,:,0], RLE = True)\n        x_idx = filt.rldecode(*x_idx)\n        y_idx = filt.rldecode(*y_idx)\n        # Back to CPU, filter resize + SAVE\n        img = img.cpu().numpy()\n        img = resize_image(img[:,:,0], x_idx, y_idx, cfg = cfg)\n\n        cv2.imwrite(out_file_name, img)\n\n    shutil.rmtree(J2K_FOLDER)\nprint(f'DALI Raw image load complete')","metadata":{"papermill":{"duration":2.392517,"end_time":"2023-02-24T20:34:19.421286","exception":false,"start_time":"2023-02-24T20:34:17.028769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:48.833162Z","iopub.execute_input":"2023-03-13T19:23:48.833589Z","iopub.status.idle":"2023-03-13T19:23:53.137724Z","shell.execute_reply.started":"2023-03-13T19:23:48.83355Z","shell.execute_reply":"2023-03-13T19:23:53.136657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_saved = len(glob.glob(f'{SAVE_FOLDER}/*.png'))\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.031394,"end_time":"2023-02-24T20:34:19.497479","exception":false,"start_time":"2023-02-24T20:34:19.466085","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:53.139622Z","iopub.execute_input":"2023-03-13T19:23:53.140308Z","iopub.status.idle":"2023-03-13T19:23:53.147836Z","shell.execute_reply.started":"2023-03-13T19:23:53.140269Z","shell.execute_reply":"2023-03-13T19:23:53.146846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl\n\ndef process(f, save_folder=\"\"):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n\n    dicom = pydicom.dcmread(f)\n    \n\n    if dicom.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':  # ALREADY PROCESSED\n        return\n    \n    dicom = dicomsdl.open(f)\n    img = dicom.pixelData()\n    img = process_dicom(img, dicom)\n    img = (img * 255).clip(0,255).astype(np.uint8)\n    \n    img = torch.from_numpy(img).cuda()\n    x_idx, y_idx = filt(img[:,:], RLE = True)\n    x_idx = filt.rldecode(*x_idx)\n    y_idx = filt.rldecode(*y_idx)\n    # Back to CPU, filter resize + SAVE\n    img = img.cpu().numpy()\n    img = resize_image(img, x_idx, y_idx, cfg = cfg)    \n\n    out_file_name = save_folder + f\"{patient}_{image}.png\"\n    cv2.imwrite(out_file_name, img)\n    return out_file_name","metadata":{"papermill":{"duration":0.034062,"end_time":"2023-02-24T20:34:19.550082","exception":false,"start_time":"2023-02-24T20:34:19.51602","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:53.151214Z","iopub.execute_input":"2023-03-13T19:23:53.151941Z","iopub.status.idle":"2023-03-13T19:23:53.169373Z","shell.execute_reply.started":"2023-03-13T19:23:53.151891Z","shell.execute_reply":"2023-03-13T19:23:53.168269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_processed_filenames = Parallel(n_jobs=2)(\n    delayed(process)(f'{DATA_FOLDER}/{img}', save_folder=SAVE_FOLDER)\n    for img in tqdm(test_df[\"fns\"])\n)\ncpu_processed_filenames = [f for f in cpu_processed_filenames if f]\nprint(f'CPU Raw image load complete with {len(cpu_processed_filenames)} loaded')","metadata":{"papermill":{"duration":5.80666,"end_time":"2023-02-24T20:34:25.368837","exception":false,"start_time":"2023-02-24T20:34:19.562177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:23:53.171055Z","iopub.execute_input":"2023-03-13T19:23:53.171383Z","iopub.status.idle":"2023-03-13T19:24:04.913424Z","shell.execute_reply.started":"2023-03-13T19:23:53.171349Z","shell.execute_reply":"2023-03-13T19:24:04.912171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()\ntorch.cuda.empty_cache()","metadata":{"papermill":{"duration":0.226827,"end_time":"2023-02-24T20:34:25.612104","exception":false,"start_time":"2023-02-24T20:34:25.385277","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:04.917237Z","iopub.execute_input":"2023-03-13T19:24:04.919795Z","iopub.status.idle":"2023-03-13T19:24:05.146834Z","shell.execute_reply.started":"2023-03-13T19:24:04.919745Z","shell.execute_reply":"2023-03-13T19:24:05.145464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for tt,f in enumerate(glob.glob(f'{SAVE_FOLDER}/*.png')):\n    img = cv2.imread(f, cv2.IMREAD_UNCHANGED)\n    if tt == 1: break\nImage.fromarray(img[:, :])","metadata":{"papermill":{"duration":0.135215,"end_time":"2023-02-24T20:34:25.758805","exception":false,"start_time":"2023-02-24T20:34:25.62359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.148397Z","iopub.execute_input":"2023-03-13T19:24:05.148793Z","iopub.status.idle":"2023-03-13T19:24:05.265685Z","shell.execute_reply.started":"2023-03-13T19:24:05.148725Z","shell.execute_reply":"2023-03-13T19:24:05.264848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_saved = len(glob.glob(f'{SAVE_FOLDER}/*.png'))\nprint(f'Image on disk count : {n_saved}')","metadata":{"papermill":{"duration":0.026562,"end_time":"2023-02-24T20:34:25.801054","exception":false,"start_time":"2023-02-24T20:34:25.774492","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.266709Z","iopub.execute_input":"2023-03-13T19:24:05.26813Z","iopub.status.idle":"2023-03-13T19:24:05.274035Z","shell.execute_reply.started":"2023-03-13T19:24:05.268087Z","shell.execute_reply":"2023-03-13T19:24:05.272834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for fn in glob.glob(f'{SAVE_FOLDER}/*.png')[:5]:\n    print(cv2.imread(fn).shape)","metadata":{"papermill":{"duration":0.063343,"end_time":"2023-02-24T20:34:25.881173","exception":false,"start_time":"2023-02-24T20:34:25.81783","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.275713Z","iopub.execute_input":"2023-03-13T19:24:05.276937Z","iopub.status.idle":"2023-03-13T19:24:05.319178Z","shell.execute_reply.started":"2023-03-13T19:24:05.27689Z","shell.execute_reply":"2023-03-13T19:24:05.318077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert n_saved == len(test_df)","metadata":{"papermill":{"duration":0.024133,"end_time":"2023-02-24T20:34:25.922199","exception":false,"start_time":"2023-02-24T20:34:25.898066","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.320655Z","iopub.execute_input":"2023-03-13T19:24:05.321547Z","iopub.status.idle":"2023-03-13T19:24:05.327306Z","shell.execute_reply.started":"2023-03-13T19:24:05.321517Z","shell.execute_reply":"2023-03-13T19:24:05.326158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cfg(CFG):\n    cfg = importlib.import_module('default_config')\n    importlib.reload(cfg)\n    cfg = importlib.import_module(CFG)\n    importlib.reload(cfg)\n    cfg = copy(cfg.cfg)\n    cfg.post_process_pipeline = importlib.import_module(cfg.post_process_pipeline).post_process_pipeline\n\n    cfg.data_dir = COMP_FOLDER\n    cfg.test_data_folder = DATA_FOLDER\n    cfg.mixed_precision = MIXED_PRECISION\n    cfg.pretrained = False\n    cfg.pretrained_weights = False\n    # cfg.batch_size = cfg.batch_size//3\n    cfg.offline_inference = True\n    cfg.device = DEVICE\n\n    print(CFG, cfg.model, cfg.dataset, cfg.backbone, cfg.pretrained_weights, cfg.post_process_pipeline)\n    \n    return cfg\n\ndef get_state_dict(sd_fp):\n    sd = torch.load(sd_fp, map_location=\"cpu\")['model']\n    sd = {k.replace(\"module.\", \"\"):v for k,v in sd.items()}\n    return sd\n\ndef get_nets(cfg,state_dicts):\n    model = importlib.import_module(cfg.model)\n    importlib.reload(model)\n    Net = model.Net\n\n    nets = []\n\n    for i,state_dict in enumerate(state_dicts):\n        net = Net(cfg).eval().to(DEVICE)\n        print(\"loading dict\")\n        sd = get_state_dict(state_dict)\n        net.load_state_dict(sd, strict=True)\n        nets += [net]\n        del sd\n        gc.collect()\n    return nets\n\ndef get_dl(test_df, cfg):\n    ds = importlib.import_module(cfg.dataset)\n    importlib.reload(ds)\n\n    CustomDataset = ds.CustomDataset\n    batch_to_device = ds.batch_to_device\n\n    test_ds = CustomDataset(test_df, cfg, cfg.val_aug, mode=\"test\")\n    test_dl = DataLoader(test_ds, shuffle=False, batch_size=cfg.batch_size, collate_fn=ds.val_collate_fn, num_workers=N_CORES, pin_memory=True)\n\n    return test_dl, batch_to_device","metadata":{"papermill":{"duration":0.030251,"end_time":"2023-02-24T20:34:25.968027","exception":false,"start_time":"2023-02-24T20:34:25.937776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.328896Z","iopub.execute_input":"2023-03-13T19:24:05.329706Z","iopub.status.idle":"2023-03-13T19:24:05.34263Z","shell.execute_reply.started":"2023-03-13T19:24:05.329562Z","shell.execute_reply":"2023-03-13T19:24:05.341697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader","metadata":{"papermill":{"duration":0.023708,"end_time":"2023-02-24T20:34:26.006904","exception":false,"start_time":"2023-02-24T20:34:25.983196","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.345708Z","iopub.execute_input":"2023-03-13T19:24:05.346072Z","iopub.status.idle":"2023-03-13T19:24:05.355825Z","shell.execute_reply.started":"2023-03-13T19:24:05.346042Z","shell.execute_reply":"2023-03-13T19:24:05.354646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG = 'cfg_dh_4c_aux14B_b3_agg1'\nstate_dicts = sorted(glob.glob(f'/kaggle/input/breast-cancer-weights/cfg_dh_4c_aux14B_b3_agg1/fold-1/check*.pth'))\nprint(state_dicts)\ncfg = get_cfg(CFG)\ncfg.pretrained = False\ncfg.batch_size = 2\ncfg.val_batch_size = 2\ncfg.data_folder = cfg.test_data_folder = SAVE_FOLDER\nnets = get_nets(cfg,state_dicts)","metadata":{"papermill":{"duration":2.652901,"end_time":"2023-02-24T20:34:28.706947","exception":false,"start_time":"2023-02-24T20:34:26.054046","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:05.357573Z","iopub.execute_input":"2023-03-13T19:24:05.358052Z","iopub.status.idle":"2023-03-13T19:24:08.173356Z","shell.execute_reply.started":"2023-03-13T19:24:05.358013Z","shell.execute_reply":"2023-03-13T19:24:08.172248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for CFG1 in 'cfg_dh_4c_aux14B_v2_s_agg1 cfg_dh_4c_aux14B_b4_agg1 cfg_dh_4c_aux14B_b5_agg1'.split(): # cfg_ip_4c_aux14B_v2_m_agg1 \n    state_dicts = sorted(glob.glob(f'/kaggle/input/breast-cancer-weights/{CFG1}/fold-1/check*.pth'))\n    print(state_dicts)\n    cfg1 = get_cfg(CFG1)\n    nets += get_nets(cfg1,state_dicts)\n    del CFG1, cfg1, state_dicts\nprint(f'Models loaded : {len(nets)}')","metadata":{"papermill":{"duration":3.14196,"end_time":"2023-02-24T20:34:31.865028","exception":false,"start_time":"2023-02-24T20:34:28.723068","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:08.174891Z","iopub.execute_input":"2023-03-13T19:24:08.175271Z","iopub.status.idle":"2023-03-13T19:24:18.116381Z","shell.execute_reply.started":"2023-03-13T19:24:08.175232Z","shell.execute_reply":"2023-03-13T19:24:18.1144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_ds = CustomDataset(test_df, cfg, cfg.val_aug, mode=\"test\")\n# sub_dl = DataLoader(sub_ds, shuffle=False, batch_size=cfg.batch_size, collate_fn=val_collate_fn, num_workers=N_CORES, pin_memory=True)\ntest_df['biopsy']=0\ntest_df['density']='C'\ntest_df['BIRADS']=0.\ntest_df['invasive']=0\ntest_df['difficult_negative_case']=0\nsub_dl, batch_to_device = get_dl(test_df, cfg)\n#sub_dl.dataset.fns = [i.split('_')[0] + '_' + i.split('_')[-1]  for i in sub_dl.dataset.fns]\nsub_dl.dataset.df.fns = sub_dl.dataset.df.fns.apply(lambda x: [i.split('_')[0] + '_' + i.split('_')[-1]  for i in x])\n","metadata":{"papermill":{"duration":0.061452,"end_time":"2023-02-24T20:34:40.081925","exception":false,"start_time":"2023-02-24T20:34:40.020473","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:18.117763Z","iopub.execute_input":"2023-03-13T19:24:18.118171Z","iopub.status.idle":"2023-03-13T19:24:18.157831Z","shell.execute_reply.started":"2023-03-13T19:24:18.11813Z","shell.execute_reply":"2023-03-13T19:24:18.156845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Dataloader length : {len(sub_dl.dataset)}')","metadata":{"papermill":{"duration":0.023778,"end_time":"2023-02-24T20:34:40.122367","exception":false,"start_time":"2023-02-24T20:34:40.098589","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:18.1639Z","iopub.execute_input":"2023-03-13T19:24:18.164186Z","iopub.status.idle":"2023-03-13T19:24:18.173047Z","shell.execute_reply.started":"2023-03-13T19:24:18.164159Z","shell.execute_reply":"2023-03-13T19:24:18.171686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = []\nwith torch.inference_mode():\n    preds = [[] for i in range(len(nets))]\n    for tt, batch in tqdm(enumerate(sub_dl), total = len(sub_dl)):\n        batch = batch_to_device(batch, cfg.device)\n        for i, net in enumerate(nets):\n            logits = net(batch)['logits'].sigmoid().float().detach().cpu().numpy()\n            preds[i] += [logits]\n        if tt%50==0: \n            print(f'processed {tt} batches from {len(sub_dl)}')\n            gc.collect()\n            torch.cuda.empty_cache()\n\n    results += [preds]\n    print('Prediction complete')","metadata":{"papermill":{"duration":2.081316,"end_time":"2023-02-24T20:34:42.260727","exception":false,"start_time":"2023-02-24T20:34:40.179411","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:18.174504Z","iopub.execute_input":"2023-03-13T19:24:18.175026Z","iopub.status.idle":"2023-03-13T19:24:21.975382Z","shell.execute_reply.started":"2023-03-13T19:24:18.174981Z","shell.execute_reply":"2023-03-13T19:24:21.974199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r2 = [np.array([np.concatenate(p, axis=0) for p in preds]) for preds in results]\npreds = np.concatenate(r2, axis=1)\npreds.shape #models images classes","metadata":{"papermill":{"duration":0.023377,"end_time":"2023-02-24T20:34:42.422916","exception":false,"start_time":"2023-02-24T20:34:42.399539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:21.977254Z","iopub.execute_input":"2023-03-13T19:24:21.978436Z","iopub.status.idle":"2023-03-13T19:24:21.987671Z","shell.execute_reply.started":"2023-03-13T19:24:21.978393Z","shell.execute_reply":"2023-03-13T19:24:21.986618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds = np.array([np.concatenate(p, axis=0) for p in preds])\npreds = preds.mean(0) #aggregate models\npreds = preds[:,0]\npreds.shape","metadata":{"papermill":{"duration":0.025781,"end_time":"2023-02-24T20:34:42.506625","exception":false,"start_time":"2023-02-24T20:34:42.480844","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:21.98922Z","iopub.execute_input":"2023-03-13T19:24:21.989681Z","iopub.status.idle":"2023-03-13T19:24:22.026306Z","shell.execute_reply.started":"2023-03-13T19:24:21.989641Z","shell.execute_reply":"2023-03-13T19:24:22.025049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds[:20]","metadata":{"papermill":{"duration":0.025912,"end_time":"2023-02-24T20:34:42.549621","exception":false,"start_time":"2023-02-24T20:34:42.523709","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.028485Z","iopub.execute_input":"2023-03-13T19:24:22.029031Z","iopub.status.idle":"2023-03-13T19:24:22.038939Z","shell.execute_reply.started":"2023-03-13T19:24:22.028994Z","shell.execute_reply":"2023-03-13T19:24:22.03792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id = sub_dl.dataset.df['patient_id'].values\nlaterality = sub_dl.dataset.df['laterality'].values\n\nprediction_id = [f'{i}_{j}' for i,j in  zip(patient_id, laterality)]\n\npred_df = pd.DataFrame({'prediction_id': prediction_id, 'cancer': preds})\npred_df.shape","metadata":{"papermill":{"duration":0.028507,"end_time":"2023-02-24T20:34:42.595167","exception":false,"start_time":"2023-02-24T20:34:42.56666","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.040431Z","iopub.execute_input":"2023-03-13T19:24:22.041029Z","iopub.status.idle":"2023-03-13T19:24:22.052311Z","shell.execute_reply.started":"2023-03-13T19:24:22.04099Z","shell.execute_reply":"2023-03-13T19:24:22.050801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_df","metadata":{"papermill":{"duration":0.029056,"end_time":"2023-02-24T20:34:42.640889","exception":false,"start_time":"2023-02-24T20:34:42.611833","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.054528Z","iopub.execute_input":"2023-03-13T19:24:22.055069Z","iopub.status.idle":"2023-03-13T19:24:22.065701Z","shell.execute_reply.started":"2023-03-13T19:24:22.055032Z","shell.execute_reply":"2023-03-13T19:24:22.0645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#aggregate by prediction_id , i.e. by patient_laterality\nsub = pred_df.groupby('prediction_id')[['cancer']].agg('mean')\n\n#actual pp part\nsub['cancer'] = (sub['cancer'].values > 0.32).astype(int)\nsub","metadata":{"papermill":{"duration":0.037347,"end_time":"2023-02-24T20:34:42.695216","exception":false,"start_time":"2023-02-24T20:34:42.657869","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.067395Z","iopub.execute_input":"2023-03-13T19:24:22.068282Z","iopub.status.idle":"2023-03-13T19:24:22.089352Z","shell.execute_reply.started":"2023-03-13T19:24:22.068231Z","shell.execute_reply":"2023-03-13T19:24:22.087925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv')","metadata":{"papermill":{"duration":0.028243,"end_time":"2023-02-24T20:34:42.740506","exception":false,"start_time":"2023-02-24T20:34:42.712263","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.090582Z","iopub.execute_input":"2023-03-13T19:24:22.090849Z","iopub.status.idle":"2023-03-13T19:24:22.098802Z","shell.execute_reply.started":"2023-03-13T19:24:22.090823Z","shell.execute_reply":"2023-03-13T19:24:22.097528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if OOF_CHECK:\n    \n    def pfbeta(labels, predictions, beta):\n        #official implementation\n        y_true_count = 0\n        ctp = 0\n        cfp = 0\n\n        for idx in range(len(labels)):\n            prediction = min(max(predictions[idx], 0), 1)\n            if (labels[idx]):\n                y_true_count += 1\n                ctp += prediction\n    #             cfp += 1 - prediction #bugfix\n            else:\n                cfp += prediction\n\n        beta_squared = beta * beta\n        c_precision = ctp / (ctp + cfp)\n        c_recall = ctp / y_true_count\n        if (c_precision > 0 and c_recall > 0):\n            result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n            return result\n        else:\n            return 0\n    \n    #aggregate by prediction_id , i.e. by patient_laterality\n    test_df['prediction_id'] = test_df.apply(lambda x: f'{x.patient_id}_{x.laterality}', 1)\n    test_df = test_df.groupby('prediction_id')[['cancer']].agg('mean')\n    \n    # Sort both the same\n    test_df = test_df.loc[sub.index]\n\n    y = test_df['cancer'].values#.astype(np.float32)\n    y_pred = sub['cancer'].values\n    \n#     print(y.shape, y_pred.shape)\n    \n    score = pfbeta(y, y_pred, 1)\n    print(score)","metadata":{"papermill":{"duration":0.029179,"end_time":"2023-02-24T20:34:42.786767","exception":false,"start_time":"2023-02-24T20:34:42.757588","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-03-13T19:24:22.100623Z","iopub.execute_input":"2023-03-13T19:24:22.101061Z","iopub.status.idle":"2023-03-13T19:24:22.111176Z","shell.execute_reply.started":"2023-03-13T19:24:22.101024Z","shell.execute_reply":"2023-03-13T19:24:22.11011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.02445,"end_time":"2023-02-24T20:34:42.82841","exception":false,"start_time":"2023-02-24T20:34:42.80396","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]}]}