{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Install two dedicated libraries for handling jpeg2000 files, and a package for dicomsdl\n!pip install --no-index --no-deps /kaggle/input/extrapackages-dicomsdl-gdcm-pylibjpeg/wheelhouse/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install --no-index --no-deps /kaggle/input/extrapackages-dicomsdl-gdcm-pylibjpeg/wheelhouse/numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl\n!pip install --no-index --no-deps /kaggle/input/extrapackages-dicomsdl-gdcm-pylibjpeg/wheelhouse/pylibjpeg-1.4.0-py3-none-any.whl\n!pip install --no-index --no-deps /kaggle/input/extrapackages-dicomsdl-gdcm-pylibjpeg/wheelhouse/python_gdcm-3.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-26T22:57:03.191983Z","iopub.execute_input":"2023-02-26T22:57:03.192509Z","iopub.status.idle":"2023-02-26T22:57:15.2175Z","shell.execute_reply.started":"2023-02-26T22:57:03.192396Z","shell.execute_reply":"2023-02-26T22:57:15.216335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # for making plots\nfrom tqdm.notebook import tqdm\nimport sys\nfrom joblib import Parallel, delayed\nfrom multiprocessing import cpu_count\nimport dicomsdl\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\nimport glob\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-26T22:57:15.22086Z","iopub.execute_input":"2023-02-26T22:57:15.221292Z","iopub.status.idle":"2023-02-26T22:57:15.396165Z","shell.execute_reply.started":"2023-02-26T22:57:15.221254Z","shell.execute_reply":"2023-02-26T22:57:15.39462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import csv data as a data frame.\ncsv_train = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/train.csv\")\ncsv_test = pd.read_csv(\"/kaggle/input/rsna-breast-cancer-detection/test.csv\")\n\ntrain_images_folder = \"/kaggle/input/rsna-breast-cancer-detection/train_images\"\ntest_images_folder = \"/kaggle/input/rsna-breast-cancer-detection/test_images\"","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:15.40002Z","iopub.execute_input":"2023-02-26T22:57:15.400582Z","iopub.status.idle":"2023-02-26T22:57:15.552282Z","shell.execute_reply.started":"2023-02-26T22:57:15.40053Z","shell.execute_reply":"2023-02-26T22:57:15.550881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If any entries in the test set have missing entries in age, just set it to an avg.\ndata_test = csv_test.copy()\ndata_test['age'] = data_test['age'].fillna(data_test['age'].mean())","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:15.55572Z","iopub.execute_input":"2023-02-26T22:57:15.556197Z","iopub.status.idle":"2023-02-26T22:57:15.568644Z","shell.execute_reply.started":"2023-02-26T22:57:15.556157Z","shell.execute_reply":"2023-02-26T22:57:15.567634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put paths to all the images in a list to iterate through later\n\ntrain_im_dir = '/kaggle/input/rsna-breast-cancer-detection/train_images/'\ntest_im_dir = '/kaggle/input/rsna-breast-cancer-detection/test_images/'\n\npath = [test_im_dir + str(patient) + '/' + str(image) + '.dcm'\n        for patient, image in zip(data_test['patient_id'], data_test['image_id'])]","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:15.569991Z","iopub.execute_input":"2023-02-26T22:57:15.570961Z","iopub.status.idle":"2023-02-26T22:57:15.585298Z","shell.execute_reply.started":"2023-02-26T22:57:15.570917Z","shell.execute_reply":"2023-02-26T22:57:15.583881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nimport time\nimport ray\n\ndef process_im_ray(im, image_size = 256, show = False):\n    if show:\n        try:\n            print('begin process')\n            plt.imshow(im.pixelData(), cmap='gray')\n            plt.show(); plt.close()\n        except: \n            print('begin process')\n            plt.imshow(im.pixel_array, cmap='gray')\n            plt.show(); plt.close()\n    try:\n        out = im.pixelData()\n        phototype = im.getPixelDataInfo()['PhotometricInterpretation']\n    except:\n        out = im.pixel_array\n        phototype = im.PhotometricInterpretation\n        \n    # MONOCHROME1 pictures use a backward brightness-darkness scheme from MONOCHROME2.\n    # We reverse the MONOCHROME1 pictures to make this consistent.\n    if phototype == 'MONOCHROME1':\n        out = out.max() - out\n        \n    minval = out.min()\n    maxval = out.max()\n    \n    # Crop image to interesting part.\n    Threshold = maxval/5\n    out = cut_empty_space_ray(out, T = Threshold, show = show)\n    \n    # scale up pixel vals so that pixel values lie between 0 and 1.\n    out = (out-minval) / (maxval-minval)\n    \n    # Resize the photo.\n    out = cv2.resize(out, (image_size, image_size))\n    \n    if show:\n        plt.imshow(out, cmap='gray')\n        plt.show(); plt.close()\n        \n    return out\n\n\ndef cut_empty_space_ray(im, T = 100, cutedge = 10, show = False):\n    # ignore the border of images since some stray pixels there interfere with cropping.\n    impx_cv2_raw = im[cutedge:-cutedge, cutedge:-cutedge] \n    \n    # Convert image foreground % background to black and white.\n    _, impx_cv2 = cv2.threshold(impx_cv2_raw, T, 255, cv2.THRESH_BINARY) \n    \n    # Get edge detection contours\n    contours, _ = cv2.findContours(impx_cv2.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) \n    boundary = max(contours, key=cv2.contourArea) # Get the contour enclosing the most space.\n    \n    # Make a mask so background is set to \"color=0.\"\n    # This eliminates image defects/non tissue such as labels, other random devices, etc\n    impx_mask = np.zeros(impx_cv2.shape, dtype=\"uint8\") \n    cv2.drawContours(impx_mask, [boundary], -1, 255, cv2.FILLED) \n    # Apply the mask on top of the original image.\n    impx_cv2_masked = cv2.bitwise_and(impx_cv2_raw, impx_cv2_raw, mask = impx_mask) \n    \n    # Get coordinates bounding the rectangle of the interesting parts of the image.\n    s = time.time()\n    x, y, w, h = cv2.boundingRect(boundary)\n\n    \n    ## Testing out how well the boundingRect function selects the correct region.\n    #print(x, y, w, h)\n    #blank = np.zeros(impx_cv2.shape, dtype=\"uint8\")\n    #m = np.amax(impx_cv2_masked)\n    #cv2.rectangle((impx_cv2_masked),(x,y),(x+w,y+h), color = (int(m), 0, 0), thickness = 100, lineType=cv2.LINE_AA)\n    #plt.imshow(impx_cv2_masked)\n    #plt.show(); plt.close()\n    \n    out = impx_cv2_masked[y:y+h, x:x+w]\n    if show:        \n        plt.imshow(out)\n        plt.show(); plt.close()\n        print('end crop')\n    return out","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:15.587715Z","iopub.execute_input":"2023-02-26T22:57:15.589251Z","iopub.status.idle":"2023-02-26T22:57:16.518831Z","shell.execute_reply.started":"2023-02-26T22:57:15.589189Z","shell.execute_reply":"2023-02-26T22:57:16.517143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new ray setup for parallel processing\nray.shutdown()\nray.init(log_to_driver=False, num_gpus=1, num_cpus=cpu_count())\n\n# Progress bar solution for ray taken from: \n# https://github.com/ray-project/ray/issues/5554#issuecomment-615477207\n\n# Parallel processing iterator. Runs \"ray.get()\" on each ray future.\n# Then iterating through images to save them as PNGs with a more polished version\n# of the above algorithm.\n# PNG saving solution taken & adapted from \n# https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster\n\n# I use the ray package to make processing these images in parallel efficient.\n# Ray parallel processing solution was taken & adapted from \n# https://www.kaggle.com/code/remekkinas/ray-parallel-processing-dicom-files-and\n\ndef to_iterator(obj_ids):\n    while obj_ids:\n        done,  obj_ids = ray.wait(obj_ids)\n        yield ray.get(done[0])\n\n@ray.remote\ndef get_images_new(imagepath, image_size = 256):\n    \n    im = dicomsdl.open(imagepath)\n    image = process_im_ray(im, image_size)\n    image = np.ndarray.flatten(image)\n    del im\n    return image\n    \n@ray.remote\ndef save_images(imagepath, image_size = 256, savedir = None):\n    \n    im = dicomsdl.open(imagepath)\n    image = process_im_ray(im, image_size)\n    \n    directory_split = imagepath.split('/')\n    image_id = directory_split[-1].split('.')[0]\n    patient_id = directory_split[-2]\n    if savedir is not None:\n        newfolder = savedir + '/' + patient_id\n        fname = newfolder + '/' + image_id + '.png'\n        os.makedirs(newfolder, exist_ok=True)\n        image_save = (image * 255).astype(np.uint8)\n        cv2.imwrite(fname, image_save)\n    \nstart = time.time()\nimage_size = 512\n\nrun_dcm_to_png = True\nshow_prog_bar = False\nendpoint = 100\n\nif run_dcm_to_png:\n\n    workdir = '/kaggle/working/test/'\n    save_dir = workdir + 'processed_' + str(image_size)\n    os.makedirs(save_dir, exist_ok=True)\n\n    Save_Im_futures = [save_images.remote(p, image_size = image_size, savedir = save_dir) for p in path]\n    if show_prog_bar:\n        Save_Im = [x for x in tqdm(to_iterator(Save_Im_futures), total=len(Save_Im_futures))]\n    else:\n        Save_Im = [x for x in (to_iterator(Save_Im_futures))]\n    \n    #del Save_Im_futures\n#    ML_patientID_inputs_futures = [\n#        save_images_patient.remote(p, image_size = image_size, savedir = save_dir) for p in patientpathlist[:100]\n#    ]\n#    ML_inputs = [x for x in tqdm(to_iterator(ML_patientID_inputs_futures), total = len(ML_patientID_inputs_futures))] ## Progress bar\n    #ML_inputs = [x for x in (to_iterator(ML_patientID_inputs_futures))]\n\nray.shutdown()\n#Parallel(n_jobs = 4, require='sharedmem')(delayed(get_images_patient)(p, ML_inputs) for p in tqdm(patientpathlist[:125]))\nprint(time.time() - start)\n#print(len(ML_inputs))\n#print(len(ML_inputs[0]))","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:16.520775Z","iopub.execute_input":"2023-02-26T22:57:16.521184Z","iopub.status.idle":"2023-02-26T22:57:27.605834Z","shell.execute_reply.started":"2023-02-26T22:57:16.521152Z","shell.execute_reply":"2023-02-26T22:57:27.604199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def onehot(df, encode):\n    temp = pd.get_dummies(df[encode])\n    df = df.drop([encode], axis=1)\n    df = pd.concat([df, temp], axis=1)\n    return df\n\n\nonehotcols = ['laterality', 'view']\nfor col in onehotcols:\n    data_test = onehot(data_test, col)\n    data_train_relevant = onehot(csv_train, col)\n\nfor col in data_train_relevant.columns:\n    if col not in data_test.columns:\n        data_test[col] = 0\n    \n# Implant column is a int64 for some reason; let's convert it to uint8 so it's lighter.\ndata_test['implant'] = data_test['implant'].astype('uint8')\ndata_test.head()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-02-26T22:57:27.608081Z","iopub.execute_input":"2023-02-26T22:57:27.609123Z","iopub.status.idle":"2023-02-26T22:57:27.704659Z","shell.execute_reply.started":"2023-02-26T22:57:27.609049Z","shell.execute_reply":"2023-02-26T22:57:27.70326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:27.706316Z","iopub.execute_input":"2023-02-26T22:57:27.70704Z","iopub.status.idle":"2023-02-26T22:57:27.732735Z","shell.execute_reply.started":"2023-02-26T22:57:27.706998Z","shell.execute_reply":"2023-02-26T22:57:27.731236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Sequential, Model, load_model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\n\npngfolder = '/kaggle/working/test/processed_512'\n\n# We'll add a column to the dataframe with all of the filenames to use with flow_from_dataframe.\ndata_test['file'] = data_test.apply(\n    lambda x: pngfolder + '/' + str(int(x['patient_id'])) + '/' + str(int(x['image_id'])) \n    + '.png', axis=1)\n\n#print(data_train_relevant.head)\n\n# To create the correct shape of inputs for the network, we need to get both the\n# image data and the csv data -- have all the columns be the \"target y\" for flow_from_dataframe.\ndef test_from_dataframe(directory, generator, subset='training', batch_size = 64,\n                        data = data_test, columns = ['cancer'], seed = None):\n    \n    gendat = generator.flow_from_dataframe(data, directory=directory, shuffle = True, \n                                           target_size = (ImageSize, ImageSize),\n                                           subset = subset, batch_size = batch_size,\n                                           x_col = 'file', y_col = columns, \n                                           class_mode = 'multi_output', \n                                           color_mode = 'grayscale', \n                                           validate_filenames=False)\n    N = gendat.n\n    i = 0\n    while i < N:\n        data = gendat.next()\n        x_im = np.array(data[0]).astype(np.float16) # reference to image\n        x_info = np.array(data[1][:]).T.astype(np.float16) # data columns\n        i += batch_size\n        yield [[x_im, x_info]]\n    return\nImageSize = 512 # i don't know if we should be resizing this\nval_split = 0.0 \nbatch_size = 8 # No idea what a good number for this is, i'm using something close to what i saw on Google\n\ncols = ['age', 'implant', 'L', 'R', 'AT', 'CC', 'LM', 'LMO', 'ML', 'MLO']\n# Make a Keras ImageDataGenerator\nimagegen = ImageDataGenerator(rescale = 1./255., validation_split = 0)\n\ntest_gen = test_from_dataframe(None, imagegen, batch_size = batch_size,\n                                data = data_test, columns = cols, subset='training')","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:27.73747Z","iopub.execute_input":"2023-02-26T22:57:27.738023Z","iopub.status.idle":"2023-02-26T22:57:35.40175Z","shell.execute_reply.started":"2023-02-26T22:57:27.737945Z","shell.execute_reply":"2023-02-26T22:57:35.400094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.403861Z","iopub.execute_input":"2023-02-26T22:57:35.404898Z","iopub.status.idle":"2023-02-26T22:57:35.43394Z","shell.execute_reply.started":"2023-02-26T22:57:35.404852Z","shell.execute_reply":"2023-02-26T22:57:35.432482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_import_path = '/kaggle/input/breastcancerpredictor-npd-b/breastcancerpredictor_NPD_B/'\nML = keras.models.load_model(model_import_path, compile=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.435644Z","iopub.execute_input":"2023-02-26T22:57:35.436023Z","iopub.status.idle":"2023-02-26T22:57:35.946223Z","shell.execute_reply.started":"2023-02-26T22:57:35.435991Z","shell.execute_reply":"2023-02-26T22:57:35.94441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = tf.keras.optimizers.Adam(\n        learning_rate=1e-5, weight_decay = 1e-6\n    )\n    \nfrom_logits = False\nML.compile(optimizer=optimizer,\n           loss='binary_crossentropy',\n           metrics=['accuracy']\n          )","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.947209Z","iopub.status.idle":"2023-02-26T22:57:35.947717Z","shell.execute_reply.started":"2023-02-26T22:57:35.947476Z","shell.execute_reply":"2023-02-26T22:57:35.947506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred = ML.predict(test_gen, steps=None, batch_size = batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.949126Z","iopub.status.idle":"2023-02-26T22:57:35.949607Z","shell.execute_reply.started":"2023-02-26T22:57:35.949383Z","shell.execute_reply":"2023-02-26T22:57:35.949403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred_series = pd.Series(test_pred.T[0])\npredictions = pd.DataFrame()\npredictions['image_id'] = pd.Series(data_test['image_id']).tolist()\npredictions['cancer'] = (test_pred_series).tolist()\npredict_test = csv_test.copy()\npredict_test['cancer'] = (test_pred_series).tolist()\nprint(predict_test)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.951486Z","iopub.status.idle":"2023-02-26T22:57:35.952156Z","shell.execute_reply.started":"2023-02-26T22:57:35.951924Z","shell.execute_reply":"2023-02-26T22:57:35.951947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_merge = predict_test.groupby(['patient_id', 'laterality'])['cancer'].max()\nprint(predict_merge)\npredict_merge = predict_merge\npredict_merge = predict_merge.reset_index()\npredict_merge['prediction_id'] = predict_merge.apply(\n    lambda x: str(x['patient_id']) + '_' + x['laterality'], axis = 1\n)\nprint(predict_merge)\noutput_cols = ['predict_merge', 'cancer']\npredict_merge = predict_merge.reindex(columns=['prediction_id', 'cancer'])\nprint(predict_merge)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.953514Z","iopub.status.idle":"2023-02-26T22:57:35.954494Z","shell.execute_reply.started":"2023-02-26T22:57:35.95417Z","shell.execute_reply":"2023-02-26T22:57:35.954202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_merge.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-26T22:57:35.956082Z","iopub.status.idle":"2023-02-26T22:57:35.95669Z","shell.execute_reply.started":"2023-02-26T22:57:35.956395Z","shell.execute_reply":"2023-02-26T22:57:35.956422Z"},"trusted":true},"execution_count":null,"outputs":[]}]}