{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\n\nimport gdcm\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nfrom pathlib import Path\nimport glob\nfrom PIL import Image\nfrom joblib import Parallel, delayed\nfrom tqdm import tqdm\n# import pylibjpeg\n# import libjpeg\n\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-01T20:14:38.052691Z","iopub.execute_input":"2022-12-01T20:14:38.054475Z","iopub.status.idle":"2022-12-01T20:15:06.732239Z","shell.execute_reply.started":"2022-12-01T20:14:38.054321Z","shell.execute_reply":"2022-12-01T20:15:06.730879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:06.734515Z","iopub.execute_input":"2022-12-01T20:15:06.735464Z","iopub.status.idle":"2022-12-01T20:15:12.690241Z","shell.execute_reply.started":"2022-12-01T20:15:06.735415Z","shell.execute_reply":"2022-12-01T20:15:12.689033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rescale_img_to_hu(dcm_ds):\n    \"\"\"Rescales the image to Hounsfield unit.\"\"\"\n    return dcm_ds.pixel_array# * dcm_ds.RescaleSlope + dcm_ds.RescaleIntercept\n\ndef read_dicom_image(image_path):\n    try:\n        x = rescale_img_to_hu(pydicom.dcmread(image_path + \".dcm\"))\n        x = x - np.min(x)\n        x = x / (np.max(x) + 1e-4)\n        x = (x * 255).astype(np.uint8)\n    except:\n        x = cv2.imread(image_path + \".jpeg\")\n        x = cv2.cvtColor(x, cv2.COLOR_RGB2BGR)\n    return x\n\ndef crop_out_image(img):\n    av = np.mean(img, axis=0)\n    mi = np.min(img, axis=0)\n    ma = np.max(img, axis=0)\n    img = img[:, (((av - mi) > 2) + ((av - ma) > 2))]\n    img = np.array(Image.fromarray(img).resize((1024, 1024)))\n    return img","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:12.691605Z","iopub.execute_input":"2022-12-01T20:15:12.692421Z","iopub.status.idle":"2022-12-01T20:15:12.70538Z","shell.execute_reply.started":"2022-12-01T20:15:12.692378Z","shell.execute_reply":"2022-12-01T20:15:12.703702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n  \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n  if isinstance(value, type(tf.constant(0))):\n    value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n  \"\"\"Returns a float_list from a float / double.\"\"\"\n  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n  \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:12.708562Z","iopub.execute_input":"2022-12-01T20:15:12.70908Z","iopub.status.idle":"2022-12-01T20:15:12.720998Z","shell.execute_reply.started":"2022-12-01T20:15:12.709024Z","shell.execute_reply":"2022-12-01T20:15:12.719236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(feature0, feature1, feature2):\n  feature = {\n      'image': _bytes_feature(feature0), #image\n      'age': _float_feature(feature1), #age\n      'cancer': _int64_feature(feature2), #cancer\n  }\n  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n  return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:12.722552Z","iopub.execute_input":"2022-12-01T20:15:12.723453Z","iopub.status.idle":"2022-12-01T20:15:12.73636Z","shell.execute_reply.started":"2022-12-01T20:15:12.723398Z","shell.execute_reply":"2022-12-01T20:15:12.73523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/rsna-breast-cancer-detection/train.csv\")\ntrain['file_path'] = '../input/rsna-breast-cancer-detection/train_images/' + train['patient_id'].astype('str') + \"/\" + train['image_id'].astype('str')\n\nskf = StratifiedKFold(n_splits=40, random_state=2022, shuffle=True)\nfor fold, ( _, val_) in enumerate(skf.split(X=train, y=train.cancer)):\n    train.loc[val_ , \"kfold\"] = fold\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:12.738067Z","iopub.execute_input":"2022-12-01T20:15:12.738474Z","iopub.status.idle":"2022-12-01T20:15:13.107624Z","shell.execute_reply.started":"2022-12-01T20:15:12.738435Z","shell.execute_reply":"2022-12-01T20:15:13.106133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_fold_tf(fold):\n    tmp_df = train[train['kfold'] == fold].reset_index(drop=True)\n    with tf.io.TFRecordWriter('train_%i.tfrec'%(fold)) as writer:\n        for i, row in tmp_df.iterrows():\n            img = read_dicom_image(row['file_path'])\n            img = crop_out_image(img)\n            img = cv2.imencode('.jpg', img, (cv2.IMWRITE_JPEG_QUALITY, 98))[1].tobytes()\n            example = serialize_example(img, row['age'], row['cancer'])\n            writer.write(example)","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:13.109467Z","iopub.execute_input":"2022-12-01T20:15:13.110383Z","iopub.status.idle":"2022-12-01T20:15:13.117608Z","shell.execute_reply.started":"2022-12-01T20:15:13.110339Z","shell.execute_reply":"2022-12-01T20:15:13.116434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = Parallel(n_jobs=4)(delayed(create_fold_tf)(i) for i in tqdm(range(0, 40), total=40))","metadata":{"execution":{"iopub.status.busy":"2022-12-01T20:15:43.153725Z","iopub.execute_input":"2022-12-01T20:15:43.154223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}