{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/for-pydicom/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:56:55.866713Z","iopub.execute_input":"2023-04-07T01:56:55.867164Z","iopub.status.idle":"2023-04-07T01:57:27.591509Z","shell.execute_reply.started":"2023-04-07T01:56:55.867107Z","shell.execute_reply":"2023-04-07T01:57:27.590134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n#import tensorflow_io as tfio\n#import pydicom\nimport dicomsdl\nimport cv2\nimport tqdm\nfrom joblib import Parallel, delayed\nimport matplotlib.pyplot as plt\n\nprint('TF version - ', tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:27.594458Z","iopub.execute_input":"2023-04-07T01:57:27.594909Z","iopub.status.idle":"2023-04-07T01:57:35.642776Z","shell.execute_reply.started":"2023-04-07T01:57:27.594837Z","shell.execute_reply":"2023-04-07T01:57:35.641509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tf.debugging.set_log_device_placement(False)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:35.644802Z","iopub.execute_input":"2023-04-07T01:57:35.64631Z","iopub.status.idle":"2023-04-07T01:57:35.652099Z","shell.execute_reply.started":"2023-04-07T01:57:35.646259Z","shell.execute_reply":"2023-04-07T01:57:35.650756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    strategy = tf.distribute.MirroredStrategy() # for CPU/GPU or multi-GPU machines\n\nprint(\"Number of replicas: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:35.656184Z","iopub.execute_input":"2023-04-07T01:57:35.656683Z","iopub.status.idle":"2023-04-07T01:57:39.813334Z","shell.execute_reply.started":"2023-04-07T01:57:35.656641Z","shell.execute_reply":"2023-04-07T01:57:39.811983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = [768, 512]\n\nMAIN_PATH = '/kaggle/input/rsna-breast-cancer-detection/'\nTEST_PATH = MAIN_PATH + 'test_images/'\nBATCH_SIZE = 8 * strategy.num_replicas_in_sync\nDEBUG = False","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:39.815205Z","iopub.execute_input":"2023-04-07T01:57:39.815946Z","iopub.status.idle":"2023-04-07T01:57:39.822458Z","shell.execute_reply.started":"2023-04-07T01:57:39.815901Z","shell.execute_reply":"2023-04-07T01:57:39.821144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert dicom images to jpeg with ROI and resizing","metadata":{}},{"cell_type":"code","source":"if DEBUG:\n    # experiment on train dataset\n    test_df = pd.read_csv(MAIN_PATH + 'train.csv', nrows=2048)\n    TEST_PATH = MAIN_PATH + 'train_images/'\n    NUM_TEST_IMAGES = len(test_df)\nelse:\n    test_df = pd.read_csv(MAIN_PATH + 'test.csv')\n    TEST_PATH = MAIN_PATH + 'test_images/'\n    NUM_TEST_IMAGES = len(test_df)\n\nprint('{} rows in test.csv'.format(NUM_TEST_IMAGES))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:39.824289Z","iopub.execute_input":"2023-04-07T01:57:39.825031Z","iopub.status.idle":"2023-04-07T01:57:39.856723Z","shell.execute_reply.started":"2023-04-07T01:57:39.824984Z","shell.execute_reply":"2023-04-07T01:57:39.855538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def roi_resize(image, percent=95):\n    \n    # standard deviation for excluding background\n    row_mask = np.std(image, axis=1) > 0.05\n    clmn_mask =  np.std(image, axis=0) > 0.05\n    image = image[row_mask, :][:, clmn_mask]\n\n    # 95-th percentile, for excluding of on-image labels \n    row_mask = np.percentile(image, percent, axis=1) > 0.05\n    clmn_mask =  np.percentile(image, percent, axis=0) > 0.05\n    image = image[row_mask, :][:, clmn_mask]\n    \n    image = cv2.resize(image, IMAGE_SIZE[::-1], interpolation=cv2.INTER_NEAREST)\n    \n    return image\n\ndef dicom_to_jpg(dcm_file):\n    patient_id = dcm_file.split('/')[-2]\n    image_id   = dcm_file.split('/')[-1][:-4]\n\n    dicom = dicomsdl.open(dcm_file)\n    image = dicom.pixelData()\n    image = (image - image.min()) / (image.max() - image.min())\n\n    if dicom.PhotometricInterpretation == 'MONOCHROME1':\n        image = 1 - image\n        \n    image = roi_resize(image)\n\n    cv2.imwrite(str(patient_id)+'_'+str(image_id)+'.jpg', image*255) ","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:39.858172Z","iopub.execute_input":"2023-04-07T01:57:39.858591Z","iopub.status.idle":"2023-04-07T01:57:39.870932Z","shell.execute_reply.started":"2023-04-07T01:57:39.858547Z","shell.execute_reply":"2023-04-07T01:57:39.869585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# making jpeg\nfiles = [TEST_PATH + os.path.join(str(row.patient_id), str(row.image_id) + '.dcm')\n              for _, row in test_df.iterrows()\n             ]\n\n_ = Parallel(n_jobs=-1)(delayed(dicom_to_jpg)(f) for f in tqdm.tqdm(files, ncols=80))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T01:57:39.872621Z","iopub.execute_input":"2023-04-07T01:57:39.874578Z","iopub.status.idle":"2023-04-07T02:22:39.021055Z","shell.execute_reply.started":"2023-04-07T01:57:39.874532Z","shell.execute_reply":"2023-04-07T02:22:39.019585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction on a pre-calculated model","metadata":{}},{"cell_type":"code","source":"def read_file(path):\n\n    image = tf.io.read_file(path)\n    image = tf.io.decode_jpeg(image)\n\n    return image\n\ndef get_test_dataset(test_df):\n    \n    pathes = [str(row.patient_id) + '_' + str(row.image_id) + '.jpg' for _, row in test_df.iterrows()]\n\n    dataset = tf.data.Dataset.from_tensor_slices(pathes)\n    dataset = dataset.map(read_file, num_parallel_calls=tf.data.AUTOTUNE)\n\n    dataset = dataset.batch(BATCH_SIZE).prefetch(8)\n    \n    return dataset\n\ndef pf1_score(labels, predictions):\n\n    pTP = tf.math.reduce_sum(labels * predictions)\n    pFP = tf.math.reduce_sum((1-labels) * predictions)\n\n    pPrecision = pTP/(pTP+pFP)\n    pRecall = pTP/tf.math.reduce_sum(labels)\n    \n\n    if (pPrecision > 0 and pRecall > 0):\n        pF1 = 2 * pPrecision * pRecall/(pPrecision + pRecall)\n        return pF1\n    else:\n        return 0.0","metadata":{"execution":{"iopub.status.busy":"2023-04-07T02:22:39.022805Z","iopub.execute_input":"2023-04-07T02:22:39.023188Z","iopub.status.idle":"2023-04-07T02:22:39.035336Z","shell.execute_reply.started":"2023-04-07T02:22:39.02315Z","shell.execute_reply":"2023-04-07T02:22:39.034022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = get_test_dataset(test_df)","metadata":{"execution":{"iopub.status.busy":"2023-04-07T02:22:39.039886Z","iopub.execute_input":"2023-04-07T02:22:39.041124Z","iopub.status.idle":"2023-04-07T02:22:39.328676Z","shell.execute_reply.started":"2023-04-07T02:22:39.041046Z","shell.execute_reply":"2023-04-07T02:22:39.327499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = tf.keras.models.load_model('/kaggle/input/rsna-bsd-tensorflow/dn121-768x512.h5', custom_objects={'pf1_score': pf1_score})","metadata":{"execution":{"iopub.status.busy":"2023-04-07T02:22:39.330123Z","iopub.execute_input":"2023-04-07T02:22:39.330564Z","iopub.status.idle":"2023-04-07T02:22:53.272585Z","shell.execute_reply.started":"2023-04-07T02:22:39.330522Z","shell.execute_reply":"2023-04-07T02:22:53.271193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=\"./logs\")\n# profiler = tf.profiler.experimental.Profile('logs')\n\nprint('Computing predictions...')\nwith strategy.scope():\n    probabilities = model.predict(test_dataset, use_multiprocessing=True)\nprint('number of test sample - ', len(probabilities))\n\nprint('Generating submission.csv file...')\ntest_ids = [str(row.patient_id)+'_'+row.laterality for _, row in test_df.iterrows()]\nsubmit_df = pd.DataFrame(zip(test_ids, np.squeeze(probabilities)), columns=['prediction_id', 'cancer'])\nsubmit_df.groupby('prediction_id', sort=False, as_index=False).max().to_csv('submission.csv', index=False)\nprint('Done')","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-04-07T02:22:53.278039Z","iopub.execute_input":"2023-04-07T02:22:53.278459Z","iopub.status.idle":"2023-04-07T02:24:13.754514Z","shell.execute_reply.started":"2023-04-07T02:22:53.278421Z","shell.execute_reply":"2023-04-07T02:24:13.753128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm /kaggle/working/*.jpg","metadata":{"execution":{"iopub.status.busy":"2023-04-07T02:24:13.756503Z","iopub.execute_input":"2023-04-07T02:24:13.757Z","iopub.status.idle":"2023-04-07T02:24:14.982755Z","shell.execute_reply.started":"2023-04-07T02:24:13.756949Z","shell.execute_reply":"2023-04-07T02:24:14.981323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEBUG:\n    labels = test_df.groupby(['patient_id', 'laterality'], sort=False, as_index=False)[['patient_id', 'laterality', 'cancer']].max('cancer')\n    print('pF1-score is: {:.4f}'.format(pf1_score(labels.cancer.astype(np.float32), \n                                              submit_df.groupby('prediction_id', sort=False, as_index=False).max().cancer)))","metadata":{"execution":{"iopub.status.busy":"2023-04-07T02:24:14.984695Z","iopub.execute_input":"2023-04-07T02:24:14.985111Z","iopub.status.idle":"2023-04-07T02:24:15.035421Z","shell.execute_reply.started":"2023-04-07T02:24:14.985051Z","shell.execute_reply":"2023-04-07T02:24:15.034015Z"},"trusted":true},"execution_count":null,"outputs":[]}]}