{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip download pylibjpeg pylibjpeg-libjpeg pydicom python-gdcm","metadata":{"_uuid":"c9425651-2442-44c7-8f2f-3ecea7038358","_cell_guid":"54404b16-4371-46f7-af76-ec3dd4b4e8c0","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:48:24.46924Z","iopub.execute_input":"2023-02-08T19:48:24.469717Z","iopub.status.idle":"2023-02-08T19:48:24.477516Z","shell.execute_reply.started":"2023-02-08T19:48:24.46961Z","shell.execute_reply":"2023-02-08T19:48:24.476331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:48:24.534553Z","iopub.execute_input":"2023-02-08T19:48:24.534942Z","iopub.status.idle":"2023-02-08T19:48:25.506363Z","shell.execute_reply.started":"2023-02-08T19:48:24.534906Z","shell.execute_reply":"2023-02-08T19:48:25.505002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install /kaggle/input/rsna-2022-whl/{pydicom-2.3.0-py3-none-any.whl,pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}\n!pip install /kaggle/input/nvidia-dali-wheel/nvidia_dali_nightly_cuda110-1.22.0.dev20221213-6757685-py3-none-manylinux2014_x86_64.whl\n!pip install /kaggle/input/nvidia-dali-wheel/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!cp /kaggle/input/modified-pytorchpy/pytorch.py /opt/conda/lib/python3.7/site-packages/nvidia/dali/plugin/pytorch.py","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:48:25.510445Z","iopub.execute_input":"2023-02-08T19:48:25.510916Z","iopub.status.idle":"2023-02-08T19:49:27.441176Z","shell.execute_reply.started":"2023-02-08T19:48:25.510869Z","shell.execute_reply":"2023-02-08T19:49:27.439782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install ../input/rsna-python-libraries/pydicom-2.3.1-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install ../input/rsna-python-libraries/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install ../input/rsna-python-libraries/python_gdcm-3.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_libjpeg-1.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install ../input/rsna-python-libraries/pylibjpeg_openjpeg-1.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"_uuid":"7852867e-9e47-4f66-b253-92b64e785015","_cell_guid":"2b0e7534-24b6-41a7-8c24-c8391a9881ee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:49:27.446376Z","iopub.execute_input":"2023-02-08T19:49:27.448085Z","iopub.status.idle":"2023-02-08T19:52:58.547081Z","shell.execute_reply.started":"2023-02-08T19:49:27.448037Z","shell.execute_reply":"2023-02-08T19:52:58.545762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ln -s ../input/rsna-breast-cancer-256-pngs/ ./processed_images","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:52:58.552447Z","iopub.execute_input":"2023-02-08T19:52:58.552923Z","iopub.status.idle":"2023-02-08T19:52:58.562491Z","shell.execute_reply.started":"2023-02-08T19:52:58.552868Z","shell.execute_reply":"2023-02-08T19:52:58.561377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir processed_images\n# !mv ../input/rsna-breast-cancer-256-pngs/* ./processed_images/\n!find ../input/rsna-breast-cancer-256-pngs/ -name \"*\" -exec cp -ruf \"{}\" ./processed_images/ \\;","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:52:58.568683Z","iopub.execute_input":"2023-02-08T19:52:58.570223Z","iopub.status.idle":"2023-02-08T19:56:15.944516Z","shell.execute_reply.started":"2023-02-08T19:52:58.570174Z","shell.execute_reply":"2023-02-08T19:56:15.943143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gdcm\n\nimport importlib\nimportlib.reload(__import__(\"gdcm\"))\n\nfrom gdcm import DataElement\nimport pandas as pd\nimport os\nfrom pathlib import Path","metadata":{"_uuid":"335b7335-b926-45d0-8de9-d81f870d6a9f","_cell_guid":"1ec149b9-f104-4a70-a25a-ce825992541a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:15.949403Z","iopub.execute_input":"2023-02-08T19:56:15.949879Z","iopub.status.idle":"2023-02-08T19:56:16.028215Z","shell.execute_reply.started":"2023-02-08T19:56:15.949828Z","shell.execute_reply":"2023-02-08T19:56:16.027401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"competition_name = \"rsna-breast-cancer-detection\"\njson = False\nlines = False\nsubset_rows = None\nfile_path = f\"/kaggle/input/{competition_name}\"\n\niskaggle = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nif iskaggle:\n    path = Path(file_path)\nelse:\n    path = Path('titanic')\n    if not path.exists():\n        import zipfile\n        import kaggle\n        kaggle.api.competition_download_cli(str(path))\n        zipfile.ZipFile(f'{path}.zip').extractall(path)\n\n\n# load test and train data\n# [train/test]_images/[patient_id]/[image_id].dcm \nif json:\n    train = pd.read_json(f\"{path}/train.json\", lines=lines, nrows=subset_rows)\n    test = pd.read_json(f\"{path}/test.json\", lines=lines, nrows=subset_rows)\nelse:\n    train_csv = pd.read_csv(f\"{path}/train.csv\", nrows=subset_rows)\n    test_csv = pd.read_csv(f\"{path}/test.csv\", nrows=subset_rows)","metadata":{"_uuid":"97aad611-eecb-4117-b5cf-853cb59c099d","_cell_guid":"dedef351-d1a9-41b4-a977-bf3ccaabdca4","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:16.029877Z","iopub.execute_input":"2023-02-08T19:56:16.030452Z","iopub.status.idle":"2023-02-08T19:56:16.139179Z","shell.execute_reply.started":"2023-02-08T19:56:16.030411Z","shell.execute_reply":"2023-02-08T19:56:16.138131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv['test'] = False\ntest_csv['test'] = True","metadata":{"_uuid":"2fbbcf2e-d222-4cb4-8ea8-131912885d94","_cell_guid":"722b751b-ec50-409a-ba0e-0e364c29a89d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:16.142981Z","iopub.execute_input":"2023-02-08T19:56:16.14375Z","iopub.status.idle":"2023-02-08T19:56:16.150813Z","shell.execute_reply.started":"2023-02-08T19:56:16.143703Z","shell.execute_reply":"2023-02-08T19:56:16.149737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\n\nimport pydicom\n\nimport pandas as pd\n\nfrom pydicom import dcmread\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport gdcm","metadata":{"_uuid":"d682ef44-9fdd-4610-b60c-07e07f8683ed","_cell_guid":"7d28427a-3916-47b2-a9ed-1f87ed7ab77b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:16.152543Z","iopub.execute_input":"2023-02-08T19:56:16.15341Z","iopub.status.idle":"2023-02-08T19:56:18.000131Z","shell.execute_reply.started":"2023-02-08T19:56:16.153363Z","shell.execute_reply":"2023-02-08T19:56:17.999128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 1\ndcm_path = f'{path}/test_images/{test_csv.loc[row,\"patient_id\"]}/{test_csv.loc[row, \"image_id\"]}.dcm'\ndcm = dcmread(dcm_path)#, force=True)\n# transfer syntaxes https://pydicom.github.io/pydicom/stable/old/image_data_handlers.html\n\nimage = Image.fromarray(dcm.pixel_array.astype(float))\nplt.imshow(dcm.pixel_array, cmap=plt.cm.bone)","metadata":{"_uuid":"46dd73a1-9137-4ee5-8880-661a349b347c","_cell_guid":"375c6757-d769-429c-b478-ea58e6303886","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:18.00175Z","iopub.execute_input":"2023-02-08T19:56:18.002396Z","iopub.status.idle":"2023-02-08T19:56:19.54142Z","shell.execute_reply.started":"2023-02-08T19:56:18.002361Z","shell.execute_reply":"2023-02-08T19:56:19.53887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{"_uuid":"98ebd1e8-b592-44bd-a57d-b43a18b2a6a3","_cell_guid":"f94fbc18-6fef-4ace-abb9-2f7ae65a514b","trusted":true}},{"cell_type":"code","source":"patient_id_col = test_csv.columns.get_loc('patient_id')\nimage_id_col = test_csv.columns.get_loc('image_id')\nprint(image_id_col)","metadata":{"_uuid":"9ab8fcb1-8237-48b4-969d-f83d911dd868","_cell_guid":"0bcddf67-42bb-43b2-a785-8fb6ee9e4798","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:19.543718Z","iopub.execute_input":"2023-02-08T19:56:19.5445Z","iopub.status.idle":"2023-02-08T19:56:19.552329Z","shell.execute_reply.started":"2023-02-08T19:56:19.544432Z","shell.execute_reply":"2023-02-08T19:56:19.551302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp ../input/rsna-image-preprocessing/RSNA_image_preprocessing.py ./preprocessing.py","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:19.554452Z","iopub.execute_input":"2023-02-08T19:56:19.555384Z","iopub.status.idle":"2023-02-08T19:56:20.894924Z","shell.execute_reply.started":"2023-02-08T19:56:19.555322Z","shell.execute_reply":"2023-02-08T19:56:20.892398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir test-folder\n# !chmod 777 test-folder","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:20.905381Z","iopub.execute_input":"2023-02-08T19:56:20.90822Z","iopub.status.idle":"2023-02-08T19:56:20.917805Z","shell.execute_reply.started":"2023-02-08T19:56:20.908163Z","shell.execute_reply":"2023-02-08T19:56:20.916713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import dicomsdl\nfrom nvidia.dali.plugin.pytorch import feed_ndarray, to_torch_type\nfrom pydicom.filebase import DicomBytesIO\nfrom nvidia.dali.types import DALIDataType\nfrom nvidia.dali import pipeline_def\nimport nvidia.dali.types as types\nimport nvidia.dali.fn as fn\nimport torch.nn.functional as F\nimport torch\nimport os\nimport sys\nimport cv2\nimport glob\nimport gdcm\nimport json\nimport shutil\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\n\nDEBUG = False\n\nIMG_PATH = \"/kaggle/input/rsna-breast-cancer-detection/test_images/\"\ntest_images = glob.glob(f\"{IMG_PATH}*/*.dcm\")\n\nif DEBUG:\n    IMG_PATH = \"/kaggle/input/rsna-breast-cancer-detection/train_images/\"\n#     test_images = glob.glob(f\"{IMG_PATH}*/*.dcm\")[:1000]\n    test_images = glob.glob(f\"{IMG_PATH}10042/*.dcm\")\n\nprint(\"Number of images :\", len(test_images))\n\nSAVE_FOLDER = \"./processed_images/\"\nSIZE = 1024\n\nos.makedirs(SAVE_FOLDER, exist_ok=True)\n\nif len(test_images) > 100:\n    N_CHUNKS = 4\nelse:\n    N_CHUNKS = 1\n\nCHUNKS = [(len(test_images) / N_CHUNKS * k, len(test_images) /\n           N_CHUNKS * (k + 1)) for k in range(N_CHUNKS)]\nCHUNKS = np.array(CHUNKS).astype(int)\n\nJ2K_FOLDER = \"/tmp/j2k/\"\n\n\ndef convert_dicom_to_j2k(file, save_folder=\"\"):\n    patient = file.split('/')[-2]\n    image = file.split('/')[-1][:-4]\n    dcmfile = pydicom.dcmread(file)\n\n    if dcmfile.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':\n        with open(file, 'rb') as fp:\n            raw = DicomBytesIO(fp.read())\n            ds = pydicom.dcmread(raw)\n        # <---- the jpeg2000 header info we're looking for\n        offset = ds.PixelData.find(b\"\\x00\\x00\\x00\\x0C\")\n        hackedbitstream = bytearray()\n        hackedbitstream.extend(ds.PixelData[offset:])\n        with open(save_folder + f\"{patient}_{image}.jp2\", \"wb\") as binary_file:\n            binary_file.write(hackedbitstream)\n\n\n@pipeline_def\ndef j2k_decode_pipeline(j2kfiles):\n    jpegs, _ = fn.readers.file(files=j2kfiles)\n    images = fn.experimental.decoders.image(\n        jpegs, device='mixed', output_type=types.ANY_DATA, dtype=DALIDataType.UINT16)\n    return images\n\n\nfor chunk in tqdm(CHUNKS):\n    os.makedirs(J2K_FOLDER, exist_ok=True)\n\n    _ = Parallel(n_jobs=2)(\n        delayed(convert_dicom_to_j2k)(img, save_folder=J2K_FOLDER)\n        for img in test_images[chunk[0]: chunk[1]]\n    )\n\n    j2kfiles = glob.glob(J2K_FOLDER + \"*.jp2\")\n\n    if not len(j2kfiles):\n        continue\n\n    pipe = j2k_decode_pipeline(\n        j2kfiles, batch_size=1, num_threads=2, device_id=0, debug=True)\n    pipe.build()\n\n    for i, f in enumerate(j2kfiles):\n        patient, image = f.split('/')[-1][:-4].split('_')\n        dicom = pydicom.dcmread(IMG_PATH + f\"{patient}/{image}.dcm\")\n\n        out = pipe.run()\n\n        # Dali -> Torch\n        img = out[0][0]\n        img_torch = torch.empty(img.shape(), dtype=torch.int16, device=\"cuda\")\n        feed_ndarray(img, img_torch,\n                     cuda_stream=torch.cuda.current_stream(device=0))\n        img = img_torch.float()\n\n        # Scale, resize, invert on GPU !\n        min_, max_ = img.min(), img.max()\n        img = (img - min_) / (max_ - min_)\n\n        if SIZE:\n            img = F.interpolate(img.view(1, 1, img.size(0), img.size(\n                1)), (SIZE, SIZE), mode=\"bilinear\")[0, 0]\n\n        if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n            img = 1 - img\n\n        # Back to CPU + SAVE\n        img = (img * 255).cpu().numpy().astype(np.uint8)\n\n        cv2.imwrite(SAVE_FOLDER + f\"{patient}_{image}.png\", img)\n\n    shutil.rmtree(J2K_FOLDER)\n\n\ndef dicomsdl_to_numpy_image(dicom, index=0):\n    info = dicom.getPixelDataInfo()\n    dtype = info['dtype']\n    if info['SamplesPerPixel'] != 1:\n        raise RuntimeError('SamplesPerPixel != 1')\n    else:\n        shape = [info['Rows'], info['Cols']]\n    outarr = np.empty(shape, dtype=dtype)\n    dicom.copyFrameData(index, outarr)\n    return outarr\n\n\ndef load_img_dicomsdl(f):\n    return dicomsdl_to_numpy_image(dicomsdl.open(f))\n\n\ndef process(f, size=256, save_folder=\"./processed_images/\"):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n    \n\n    dicom = pydicom.dcmread(f)\n\n#     if dicom.file_meta.TransferSyntaxUID == '1.2.840.10008.1.2.4.90':  # ALREADY PROCESSED\n#         return\n\n    try:\n        img = load_img_dicomsdl(f)\n    except:\n        img = dicom.pixel_array\n\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n\n    img = cv2.resize(img, (size, size))\n\n    file_name = f\"{save_folder}{patient}_{image}.png\"\n    print(file_name)\n    print(type((img * 255).astype(np.uint8)))\n#     cv2.imwrite(f\"{file_name}\",(img * 255).astype(np.uint8))\n    res = cv2.imwrite(f\"{file_name}\", img)\n    print(res)\n\n_ = Parallel(n_jobs=2)(\n    delayed(process)(img, size=SIZE, save_folder=SAVE_FOLDER)\n    for img in tqdm(test_images)\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:20.938597Z","iopub.execute_input":"2023-02-08T19:56:20.944013Z","iopub.status.idle":"2023-02-08T19:56:27.679521Z","shell.execute_reply.started":"2023-02-08T19:56:20.943964Z","shell.execute_reply":"2023-02-08T19:56:27.677346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al ./processed_images/10008*","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:27.682551Z","iopub.execute_input":"2023-02-08T19:56:27.68399Z","iopub.status.idle":"2023-02-08T19:56:28.721483Z","shell.execute_reply.started":"2023-02-08T19:56:27.68394Z","shell.execute_reply":"2023-02-08T19:56:28.72006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id_column = train_csv.columns.get_loc('patient_id')\nimage_id_column = train_csv.columns.get_loc('image_id')\ncancer_column = train_csv.columns.get_loc('cancer')","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:28.723897Z","iopub.execute_input":"2023-02-08T19:56:28.724725Z","iopub.status.idle":"2023-02-08T19:56:28.732838Z","shell.execute_reply.started":"2023-02-08T19:56:28.724671Z","shell.execute_reply":"2023-02-08T19:56:28.731575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\n\npre_processed = True\n\ndef get_x(x):\n    if pre_processed:\n        return f\"./processed_images/{x[patient_id_column]}_{x[image_id_column]}.png\"\n    sub_path = 'test_images'  if x[-1] else 'train_images'\n    return f\"{path}/{sub_path}/{x[patient_id_col]}/{x[image_id_col]}.dcm\"\n\ndef get_y(y):\n    return y[cancer_column]\n    \n# cancer = DataBlock(\n#         blocks=(\n#             ImageBlock(cls=PILDicom),\n#             CategoryBlock\n#         ),\n#         get_x=get_x,\n#         get_y=get_y,\n#         item_tfms=Resize(224),\n#         batch_tfms=[\n#             *aug_transforms(size=224),\n#             Normalize.from_stats(*imagenet_stats)\n#         ]\n#     )]\n\n# class PILDicomCustom(PILBase):\n#     _open_args,_tensor_cls,_show_args = {},TensorDicom,TensorDicom._show_args\n#     @classmethod\n#     def create(cls, fn:Path|str|bytes, mode=None)->None:\n#         \"Open a `DICOM file` from path `fn` or bytes `fn` and load it as a `PIL Image`\"\n#         if isinstance(fn,bytes): im = Image.fromarray(pydicom.dcmread(pydicom.filebase.DicomBytesIO(fn)).pixel_array)\n#         if isinstance(fn,(Path,str)): im = Image.fromarray(pydicom.dcmread(fn).pixel_array)\n#         im.load()\n#         im = im._new(im.im)\n#         return cls(im.convert(mode) if mode else im)\n\n\ncancer = DataBlock(\n        blocks=(\n            ImageBlock(cls=PILImage),\n            CategoryBlock\n        ),\n        get_x=get_x,\n        get_y=get_y,\n        item_tfms=[Resize(224, resamples= (Image.Resampling.NEAREST,0))],\n        batch_tfms=[\n            IntToFloatTensor(div=2**16-1),\n            *aug_transforms(size=224),\n            Normalize.from_stats(*imagenet_stats)\n        ]\n    )","metadata":{"_uuid":"50377e80-7a03-4441-82f0-dacae3ff3c46","_cell_guid":"5820b86c-70da-4182-a4c0-254d0a45cf42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:28.735855Z","iopub.execute_input":"2023-02-08T19:56:28.736325Z","iopub.status.idle":"2023-02-08T19:56:28.762999Z","shell.execute_reply.started":"2023-02-08T19:56:28.736227Z","shell.execute_reply":"2023-02-08T19:56:28.761524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('dataloaders')\nprint('_________________________')","metadata":{"_uuid":"1e9b5073-5880-4156-a5e3-ca9839098402","_cell_guid":"6214dc00-ccc5-47f5-9b97-6aaff1b1f7f1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:28.764908Z","iopub.execute_input":"2023-02-08T19:56:28.765648Z","iopub.status.idle":"2023-02-08T19:56:28.773173Z","shell.execute_reply.started":"2023-02-08T19:56:28.765608Z","shell.execute_reply":"2023-02-08T19:56:28.771846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = cancer.dataloaders(train_csv.values, num_workers=0) ","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:28.775012Z","iopub.execute_input":"2023-02-08T19:56:28.775862Z","iopub.status.idle":"2023-02-08T19:56:32.923544Z","shell.execute_reply.started":"2023-02-08T19:56:28.775742Z","shell.execute_reply":"2023-02-08T19:56:32.922482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls.show_batch(max_n=32, nrows=2, unique=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-08T19:56:32.9251Z","iopub.execute_input":"2023-02-08T19:56:32.92547Z","iopub.status.idle":"2023-02-08T19:56:34.988403Z","shell.execute_reply.started":"2023-02-08T19:56:32.925432Z","shell.execute_reply":"2023-02-08T19:56:34.978303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# download https://download.pytorch.org/models/resnet34-b627a593.pth & upload as data source / https://www.kaggle.com/datasets/pytorch/resnet34\nimport os\nif not os.path.exists('/root/.cache/torch/hub/checkpoints/'):\n        os.makedirs('/root/.cache/torch/hub/checkpoints/')\n!cp '../input/resnet34/resnet34.pth' '/root/.cache/torch/hub/checkpoints/resnet34-b627a593.pth'","metadata":{"_uuid":"0beb0796-a356-41ce-9ba2-5a24a603876d","_cell_guid":"1c45a99b-42e4-4694-9321-f008d1910356","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:34.990754Z","iopub.execute_input":"2023-02-08T19:56:34.99127Z","iopub.status.idle":"2023-02-08T19:56:36.697319Z","shell.execute_reply.started":"2023-02-08T19:56:34.991228Z","shell.execute_reply":"2023-02-08T19:56:36.695887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('lr_find')\nprint('_________________________')","metadata":{"_uuid":"28805024-f498-47fc-8d1f-6b16469853a1","_cell_guid":"0cc361e2-61a7-410f-9ad3-b094fa870c9e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:36.702144Z","iopub.execute_input":"2023-02-08T19:56:36.703399Z","iopub.status.idle":"2023-02-08T19:56:36.71756Z","shell.execute_reply.started":"2023-02-08T19:56:36.703339Z","shell.execute_reply":"2023-02-08T19:56:36.70984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_model = False\nif new_model:\n    learn = vision_learner(dls, resnet34, metrics=accuracy)\n    # learn.fine_tune(1) ???\n    learn.lr_find()\n    learn.fit_one_cycle(1)\n    learn.export('learner.pkl')\nelse:\n    print('hi')\n    learn = load_learner('../input/version-27-rsna/learner.pkl')\n\n# version27-rsna","metadata":{"_uuid":"21bfb103-8d1e-4de5-a8d2-822b0c50d47b","_cell_guid":"42d453de-859e-4cc7-827a-4fdeb9cb2b37","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T19:56:36.718999Z","iopub.execute_input":"2023-02-08T19:56:36.719322Z","iopub.status.idle":"2023-02-08T20:01:22.092394Z","shell.execute_reply.started":"2023-02-08T19:56:36.719293Z","shell.execute_reply":"2023-02-08T20:01:22.090327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nprint('_________________________')\nprint (datetime.datetime.now())\nprint('interp')\nprint('_________________________')","metadata":{"_uuid":"b2063a76-5fc7-4c22-a08c-fb7f9539c7c5","_cell_guid":"1c44c0a9-6bab-49e4-92bd-056f63cd3976","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T20:01:22.094847Z","iopub.execute_input":"2023-02-08T20:01:22.096165Z","iopub.status.idle":"2023-02-08T20:01:22.104479Z","shell.execute_reply.started":"2023-02-08T20:01:22.096122Z","shell.execute_reply":"2023-02-08T20:01:22.103384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_csv","metadata":{"_uuid":"ad7ddbcb-8cfd-4dff-a221-8c21b3a0831b","_cell_guid":"4452e1ae-8655-4999-90f3-044015da453b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T20:01:22.106925Z","iopub.execute_input":"2023-02-08T20:01:22.108346Z","iopub.status.idle":"2023-02-08T20:01:22.135792Z","shell.execute_reply.started":"2023-02-08T20:01:22.108306Z","shell.execute_reply":"2023-02-08T20:01:22.134677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 10008_736471439\n# !ls ./processed_images/10008*\n!ls ../input/rsna-breast-cancer-detection/","metadata":{"execution":{"iopub.status.busy":"2023-02-08T20:01:22.13823Z","iopub.execute_input":"2023-02-08T20:01:22.139584Z","iopub.status.idle":"2023-02-08T20:01:23.222013Z","shell.execute_reply.started":"2023-02-08T20:01:22.139541Z","shell.execute_reply":"2023-02-08T20:01:23.220659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = learn.dls.test_dl(test_csv.values)\npredictions, _, decoded = learn.get_preds(dl=test_dl, with_decoded=True)\nprint('predictions:')\nprint(predictions)\nprint('decoded')\nprint(decoded)","metadata":{"_uuid":"515dcbc0-6717-483e-93e2-fb7876e522db","_cell_guid":"ee24f49b-852c-45c7-8d12-048e383cd8fc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T20:01:23.224754Z","iopub.execute_input":"2023-02-08T20:01:23.225569Z","iopub.status.idle":"2023-02-08T20:01:23.391362Z","shell.execute_reply.started":"2023-02-08T20:01:23.225521Z","shell.execute_reply":"2023-02-08T20:01:23.390093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# submit_df = predictions[['prediction_id', 'cancer']]\n# submission = pd.DataFrame([])\n# submission['cancer'] = pd.DataFrame(predictions.numpy())[0]\n# submission['prediction_id']  = test_csv['patient_id'].astype(str) + \"-\" + test_csv['laterality']\n# # submission.columns = ['predictions_id', 'cancer']\n# submission.sort_index()\n# # subsmission.groupby('prediction_id')","metadata":{"_uuid":"6e47da43-15ad-4f98-b75d-85c4bdf5becb","_cell_guid":"2e886b22-4c02-4b6c-810d-5eb29dc0ca05","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T20:01:23.392972Z","iopub.execute_input":"2023-02-08T20:01:23.39417Z","iopub.status.idle":"2023-02-08T20:01:23.39977Z","shell.execute_reply.started":"2023-02-08T20:01:23.394119Z","shell.execute_reply":"2023-02-08T20:01:23.398635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame([])\nsubmission['prediction_id'] = test_csv['patient_id'].astype(str) + \"_\" + test_csv['laterality']\nsubmission['cancer'] = pd.DataFrame(predictions.numpy())[0]\nsubmission = submission.groupby('prediction_id').mean().reset_index()\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()\n! head submission.csv","metadata":{"_uuid":"e48cf73b-1b48-42d2-934d-d3d1838cffe4","_cell_guid":"39aa0070-a76b-4517-b6a8-0acd01cbc0aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-08T20:01:23.401743Z","iopub.execute_input":"2023-02-08T20:01:23.402522Z","iopub.status.idle":"2023-02-08T20:01:24.601693Z","shell.execute_reply.started":"2023-02-08T20:01:23.402478Z","shell.execute_reply":"2023-02-08T20:01:24.600366Z"},"trusted":true},"execution_count":null,"outputs":[]}]}