{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!mkdir -p /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/\n!cp ../input/rsna-image-preprocessing/RSNA_image_preprocessing.py ./preprocessing.py\n!unzip -o -q ../input/timm-with-dependencies/timm_all -d timm-with-dependencies","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:34:35.723071Z","iopub.execute_input":"2023-02-25T12:34:35.723584Z","iopub.status.idle":"2023-02-25T12:35:05.870598Z","shell.execute_reply.started":"2023-02-25T12:34:35.723536Z","shell.execute_reply":"2023-02-25T12:35:05.869024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links timm-with-dependencies timm","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:35:05.876996Z","iopub.execute_input":"2023-02-25T12:35:05.879356Z","iopub.status.idle":"2023-02-25T12:35:21.474247Z","shell.execute_reply.started":"2023-02-25T12:35:05.879309Z","shell.execute_reply":"2023-02-25T12:35:21.472929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --no-index --find-links timm-with-dependencies timm\n!mkdir -p /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/\n!pip install /kaggle/input/nvidia-dali-wheel/nvidia_dali_nightly_cuda110-1.22.0.dev20221213-6757685-py3-none-manylinux2014_x86_64.whl\n!pip install /kaggle/input/nvidia-dali-wheel/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!cp /kaggle/input/modified-pytorchpy/pytorch.py /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/pytorch.py\n!pip install ../input/rsna-python-libraries/pydicom-2.3.1-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/python_gdcm-3.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_libjpeg-1.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_openjpeg-1.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:35:21.479715Z","iopub.execute_input":"2023-02-25T12:35:21.480424Z","iopub.status.idle":"2023-02-25T12:40:57.600437Z","shell.execute_reply.started":"2023-02-25T12:35:21.480382Z","shell.execute_reply":"2023-02-25T12:40:57.598912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport importlib\nimport pydicom\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport gdcm\nimport dicomsdl\nimport torch\nimport sys\nimport cv2\nimport glob\nimport gdcm\nimport json\nimport shutil\nimport pydicom\nimport seaborn as sns\n\nimportlib.reload(__import__(\"gdcm\"))\n\nfrom gdcm import DataElement\nfrom pathlib import Path\nfrom collections import defaultdict\nfrom fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom PIL import Image\nfrom pdb import set_trace\nfrom pydicom import dcmread\nfrom pydicom.filebase import DicomBytesIO\nfrom nvidia.dali.plugin.pytorch import feed_ndarray, to_torch_type\nfrom nvidia.dali.types import DALIDataType\nfrom nvidia.dali import pipeline_def\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\nimport nvidia.dali.types as types\nimport nvidia.dali.fn as fn\nimport torch.nn.functional as F","metadata":{"_uuid":"335b7335-b926-45d0-8de9-d81f870d6a9f","_cell_guid":"1ec149b9-f104-4a70-a25a-ce825992541a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:40:57.607541Z","iopub.execute_input":"2023-02-25T12:40:57.610021Z","iopub.status.idle":"2023-02-25T12:40:57.632392Z","shell.execute_reply.started":"2023-02-25T12:40:57.60997Z","shell.execute_reply":"2023-02-25T12:40:57.630932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_name = \"rsna-breast-cancer-detection\"\njson = False\nlines = False\nsubset_rows = None\nfile_path = f\"/kaggle/input/{competition_name}\"\n\npath = Path(file_path)\ntrain_csv = pd.read_csv(f\"{path}/train.csv\", nrows=subset_rows)\ntest_csv = pd.read_csv(f\"{path}/test.csv\", nrows=subset_rows)\n\n# \"\"\" START https://www.kaggle.com/code/radek1/fast-ai-starter-pack-train-inference/data \"\"\"\n# NUM_EPOCHS = 4\n# NUM_SPLITS = 4\n# RESIZE_TO = (1024, 1024)\n\n# patient_id_any_cancer = train_csv.groupby('patient_id').cancer.max().reset_index()\n# skf = StratifiedKFold(NUM_SPLITS, shuffle=True, random_state=42)\n# splits = list(skf.split(patient_id_any_cancer.patient_id, patient_id_any_cancer.cancer))\n# \"\"\"END\"\"\"\n\n\n# train_csv.head()","metadata":{"_uuid":"97aad611-eecb-4117-b5cf-853cb59c099d","_cell_guid":"dedef351-d1a9-41b4-a977-bf3ccaabdca4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:40:57.638786Z","iopub.execute_input":"2023-02-25T12:40:57.641615Z","iopub.status.idle":"2023-02-25T12:40:57.74097Z","shell.execute_reply.started":"2023-02-25T12:40:57.641569Z","shell.execute_reply":"2023-02-25T12:40:57.73971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_col = test_csv.columns.get_loc('patient_id')\nimage_id_col = test_csv.columns.get_loc('image_id')\nprint(image_id_col)","metadata":{"_uuid":"9ab8fcb1-8237-48b4-969d-f83d911dd868","_cell_guid":"0bcddf67-42bb-43b2-a785-8fb6ee9e4798","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:40:57.74607Z","iopub.execute_input":"2023-02-25T12:40:57.74854Z","iopub.status.idle":"2023-02-25T12:40:57.75928Z","shell.execute_reply.started":"2023-02-25T12:40:57.74848Z","shell.execute_reply":"2023-02-25T12:40:57.758152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369267  \ndef pfbeta_torch(preds, labels, beta=1):\n    if preds.dim() != 2 or (preds.dim() == 2 and preds.shape[1] !=2): raise ValueError('Houston, we got a problem')\n    preds = preds[:, 1]\n    preds = preds.clip(0, 1)\n    y_true_count = labels.sum()\n    ctp = preds[labels==1].sum()\n    cfp = preds[labels==0].sum()\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0.0\n\n# https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/369886    \ndef pfbeta_torch_thresh(preds, labels):\n    optimized_preds = optimize_preds(preds, labels)\n    return pfbeta_torch(optimized_preds, labels)\n\ndef optimize_preds(preds, labels=None, thresh=None, return_thresh=False, print_results=False):\n    preds = preds.clone()\n    if labels is not None: without_thresh = pfbeta_torch(preds, labels)\n    \n    if not thresh and labels is not None:\n        threshs = np.linspace(0, 1, 101)\n        f1s = [pfbeta_torch((preds > thr).float(), labels) for thr in threshs]\n        idx = np.argmax(f1s)\n        thresh, best_pfbeta = threshs[idx], f1s[idx]\n\n    preds = (preds > thresh).float()\n\n    if print_results:\n        print(f'without optimization: {without_thresh}')\n        pfbeta = pfbeta_torch(preds, labels)\n        print(f'with optimization: {pfbeta}')\n        print(f'best_thresh = {thresh}')\n    if return_thresh:\n        return thresh\n    return preds\n\nfn2label = {fn: cancer_or_not for fn, cancer_or_not in zip(train_csv['image_id'].astype('str'), train_csv['cancer'])}\n\ndef splitting_func(paths):\n    train = []\n    valid = []\n    for idx, path in enumerate(paths):\n        if int(path.parent.name) in patient_id_any_cancer.iloc[splits[SPLIT][0]].patient_id.values:\n            train.append(idx)\n        else:\n            valid.append(idx)\n    return train, valid\n\ndef label_func(path):\n    return fn2label[path.stem]\n\ndef get_items(image_dir_path):\n    items = []\n    for p in get_image_files(image_dir_path):\n        items.append(p)\n        if p.stem in fn2label and int(p.parent.name) in patient_id_any_cancer.iloc[splits[SPLIT][0]].patient_id.values:\n            if label_func(p) == 1:\n                for _ in range(5):\n                    items.append(p)\n    return items","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:40:57.764923Z","iopub.execute_input":"2023-02-25T12:40:57.767217Z","iopub.status.idle":"2023-02-25T12:40:57.881212Z","shell.execute_reply.started":"2023-02-25T12:40:57.767179Z","shell.execute_reply":"2023-02-25T12:40:57.880014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_column = train_csv.columns.get_loc('patient_id')\nimage_id_column = train_csv.columns.get_loc('image_id')\ncancer_column = train_csv.columns.get_loc('cancer')\n\n\ndef get_x(x):\n    return f\"../input/rsna-breast-cancer-detection/test_images/{x[patient_id_column]}/{x[image_id_column]}.dcm\"\n#     return f\"./processed_images/{x[patient_id_column]}/{x[image_id_column]}.png\"\n\ndef get_y(y):\n    return y[cancer_column]\n\n\ncancer = DataBlock(\n        blocks=(\n            ImageBlock(cls=PILDicom),\n            CategoryBlock\n        ),\n        get_x=get_x,\n        get_y=get_y,\n        splitter=RandomSplitter(),\n        item_tfms=[Resize(224, resamples= (Image.Resampling.NEAREST,0))],\n        batch_tfms=[\n            IntToFloatTensor(div=2**16-1),\n            \n            *aug_transforms(size=224),\n            Normalize.from_stats(*imagenet_stats)\n        ]\n    )\n\n\ndls = cancer.dataloaders(test_csv.values, num_workers=0) \n# dls.device = \"cpu\"\n# dls.show_batch(max_n=32, nrows=8, unique=True)","metadata":{"_uuid":"50377e80-7a03-4441-82f0-dacae3ff3c46","_cell_guid":"5820b86c-70da-4182-a4c0-254d0a45cf42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:40:57.886208Z","iopub.execute_input":"2023-02-25T12:40:57.888636Z","iopub.status.idle":"2023-02-25T12:40:59.83508Z","shell.execute_reply.started":"2023-02-25T12:40:57.888594Z","shell.execute_reply":"2023-02-25T12:40:59.830566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dls = cancer.dataloaders(train_csv.values, num_workers=0) ","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:40:59.847219Z","iopub.execute_input":"2023-02-25T12:40:59.849601Z","iopub.status.idle":"2023-02-25T12:40:59.871106Z","shell.execute_reply.started":"2023-02-25T12:40:59.849562Z","shell.execute_reply":"2023-02-25T12:40:59.86965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dls.show_batch(max_n=32, nrows=2, unique=True) ","metadata":{"execution":{"iopub.status.busy":"2023-02-25T12:41:31.221344Z","iopub.execute_input":"2023-02-25T12:41:31.22182Z","iopub.status.idle":"2023-02-25T12:41:31.22996Z","shell.execute_reply.started":"2023-02-25T12:41:31.22178Z","shell.execute_reply":"2023-02-25T12:41:31.229011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download https://download.pytorch.org/models/resnet34-b627a593.pth & upload as data source / https://www.kaggle.com/datasets/pytorch/resnet34\nimport os\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '../input/resnet34/resnet34.pth' '/root/.cache/torch/hub/checkpoints/resnet34-b627a593.pth'\n!cp '../input/pytorch-pretrained/resnet18-f37072fd.pth' '/root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth'","metadata":{"_uuid":"0beb0796-a356-41ce-9ba2-5a24a603876d","_cell_guid":"1c45a99b-42e4-4694-9321-f008d1910356","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:41:33.514206Z","iopub.execute_input":"2023-02-25T12:41:33.514678Z","iopub.status.idle":"2023-02-25T12:41:37.259209Z","shell.execute_reply.started":"2023-02-25T12:41:33.514639Z","shell.execute_reply":"2023-02-25T12:41:37.257697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from timm.models.layers.adaptive_avgmax_pool import SelectAdaptivePool2d\nfrom torch.nn import Flatten\nfrom fastai.metrics import ActivationType\n\nlearn = load_learner('../input/fscore-rsna-v11/learner.pkl')\nlearn.model.cuda()\ntest_dl = learn.dls.test_dl(test_csv.values)\npredictions, _, decoded = learn.get_preds(dl=test_dl, with_decoded=True)\nprint('predictions:')\nprint(predictions)\nprint('decoded')\nprint(decoded)","metadata":{"_uuid":"515dcbc0-6717-483e-93e2-fb7876e522db","_cell_guid":"ee24f49b-852c-45c7-8d12-048e383cd8fc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:41:37.265506Z","iopub.execute_input":"2023-02-25T12:41:37.267943Z","iopub.status.idle":"2023-02-25T12:41:41.324318Z","shell.execute_reply.started":"2023-02-25T12:41:37.267899Z","shell.execute_reply":"2023-02-25T12:41:41.322793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame([])\nsubmission['prediction_id'] = test_csv['patient_id'].astype(str) + \"_\" + test_csv['laterality']\nsubmission['cancer'] = pd.DataFrame(predictions.numpy())[0]\nsubmission = submission.groupby('prediction_id').mean().reset_index()\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n! head submission.csv","metadata":{"_uuid":"e48cf73b-1b48-42d2-934d-d3d1838cffe4","_cell_guid":"39aa0070-a76b-4517-b6a8-0acd01cbc0aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-25T12:41:41.326026Z","iopub.execute_input":"2023-02-25T12:41:41.331733Z","iopub.status.idle":"2023-02-25T12:41:42.761939Z","shell.execute_reply.started":"2023-02-25T12:41:41.331694Z","shell.execute_reply":"2023-02-25T12:41:42.760576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}