{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Hello fellow Kagglers,\n\nThis notebook demonstrates the processing of the huge 300GB+ dataset of this competition into TFRecords for fast dataloading during training.\n\nTFRecords have the benefit of loading large chunks of data containing many samples instead of loading every image and label seperately.\n\nAll images are resized to 1024x1024 and saved in 100 TFRecords, making each TFRecord contain roughly 550 images.\n\n[RSNA EfficientNetV2 Training Tensorflow TPU](https://www.kaggle.com/code/markwijkhuizen/rsna-efficientnetv2-training-tensorflow-tpu)\n\n**V2**\n\n* 640x512 -> 1024x1024 resolution\n* Cropping images\n* Single image approach, not both CC and MLO image\n\nInference notebook in the making\n\n**V3**\n\n* 1024x1024 -> 768x1344 based on cropped image ratio\n* using PNG encoded images instead of raw tensors to reduce disk space needed\n\n**V5**\n\nThis will be the final update of my notebooks for this competitions, which should achieve a LB score in the low 0.50s. I will continue participating in this competition, however, I will not share my progress anymore to keep this competition competitive.\n\nUpdates in this version:\n\n* Correct linear/sigmoid normalization of images, many thanks to [\nBob de Graaf\n](https://www.kaggle.com/bobdegraaf) which shared this amazing notebook: [DicomSDL & VOI-LUT](https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut)\n* Switching from PNG to JPEG with 95 compression level to stay within 20GB diskspace\n* Adapted cropping algorithm to searches from maxima to threshold instead of edge to threshold (see Example Processing)\n* Padding cropping offsets to image dimensions to keep image information instead of zero padding (see Example Processing)\n* Actually using discomsdl instead of just installing it...\n\nGood luck to all of you in the last month of this excisting competition!","metadata":{}},{"cell_type":"code","source":"%%capture\n# Source: https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster?scriptVersionId=113360473\n!pip install /kaggle/input/rsnamodules/dicomsdl-0.109.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl \n\ntry:\n    import pylibjpeg\nexcept:\n   !pip install /kaggle/input/rsna-2022-whl/{pylibjpeg-1.4.0-py3-none-any.whl,python_gdcm-3.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl}","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:21:01.214149Z","iopub.execute_input":"2023-02-02T06:21:01.214653Z","iopub.status.idle":"2023-02-02T06:21:26.286339Z","shell.execute_reply.started":"2023-02-02T06:21:01.214555Z","shell.execute_reply":"2023-02-02T06:21:26.284905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pylibjpeg\nimport pydicom\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nfrom joblib import Parallel, delayed\nfrom tqdm.notebook import tqdm\nfrom multiprocessing import cpu_count\n\nimport cv2\nimport glob\nimport importlib\nimport os\nimport joblib\nimport sys\nimport dicomsdl\n\nprint(f'Tensorflow Version: {tf.__version__}')\nprint(f'Python Version: {sys.version}')\n\n# Tensorflow and CV2 set number of threads to 1 for speedup in parallell function mapping\ntf.config.threading.set_inter_op_parallelism_threads(num_threads=1)\ncv2.setNumThreads(1)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:21:26.288844Z","iopub.execute_input":"2023-02-02T06:21:26.289195Z","iopub.status.idle":"2023-02-02T06:21:32.049019Z","shell.execute_reply.started":"2023-02-02T06:21:26.289162Z","shell.execute_reply":"2023-02-02T06:21:32.04802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Interactive flag for debugging purposes\nIS_INTERACTIVE = os.environ['KAGGLE_KERNEL_RUN_TYPE'] == 'Interactive'\n\n# Dimensions of processed images\nTARGET_HEIGHT = 1344\nTARGET_WIDTH = 768\nN_CHANNELS = 1\nTARGET_HEIGHT_WIDTH_RATIO = TARGET_HEIGHT / TARGET_WIDTH\n\n# Image Normalization Tools, did not improve LB score\n# For a tutorial see: https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\nCLAHE = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(32, 32))\nAPPLY_CLAHE = False\nAPPLY_EQ_HIST = False\n\n# Image Format and Config\nIMAGE_FORMAT = 'JPG'\nIMAGE_QUALITY = 95\n\n# Random Generators Seed\nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:21:32.050382Z","iopub.execute_input":"2023-02-02T06:21:32.050946Z","iopub.status.idle":"2023-02-02T06:21:32.060829Z","shell.execute_reply.started":"2023-02-02T06:21:32.050913Z","shell.execute_reply":"2023-02-02T06:21:32.059464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Matplotlib Config","metadata":{}},{"cell_type":"code","source":"# MatplotLib Global Settings\nmpl.rcParams.update(mpl.rcParamsDefault)\nmpl.rcParams['xtick.labelsize'] = 16\nmpl.rcParams['ytick.labelsize'] = 16\nmpl.rcParams['axes.labelsize'] = 18\nmpl.rcParams['axes.titlesize'] = 24","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:21:32.0635Z","iopub.execute_input":"2023-02-02T06:21:32.063947Z","iopub.status.idle":"2023-02-02T06:21:32.071671Z","shell.execute_reply.started":"2023-02-02T06:21:32.063913Z","shell.execute_reply":"2023-02-02T06:21:32.070315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# if IS_INTERACTIVE:\n#     train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv').head(1024)\n# else:\ntrain = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n    \n# artifact症例リスト\n# https://www.kaggle.com/competitions/rsna-breast-cancer-detection/discussion/370639\nartifact_list = ['1511/1760744211',  '25323/1743461841', '26530/166085523',  '26530/877218507',\n                 '38739/1110010839', '40317/840036202',  '41647/2115055383', '41647/1330615132',\n                 '43377/6736533549', '50454/1133144412', '7412/264916633',\n                 '1511/64545189',    '1511/1031853445',  '1511/1273092690',  '822/1942326353',\n                 '55724/694341224',  '822/1942326353',   '51115/1648448368', '51115/136057818',\n                 '51115/1997037350', '51115/1906555893', '33439/1214323095', '33439/31473718',\n                 '33084/1990776518', '33439/1518655897',\n                 '1147/597771506'] # pacemaker\n\ndef get_file_path(args):\n    patient_id, image_id = args\n    return f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n    \ntrain['file_path'] = train[['patient_id', 'image_id']].apply(get_file_path, axis=1)\n\ndelete_indexes = []\nfor k in range(len(train)):\n    row = train.iloc[k, :]\n    ids = str(row.patient_id) + \"/\" + str(row.image_id)\n    if ids in artifact_list:\n        delete_indexes.append(k)\n# artifactをtrainからも削除\ntrain = train.drop(train.index[delete_indexes]).reset_index(drop=True)\n\n# display(train.info())\n# display(train.head())\n# print(train.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:21:32.074507Z","iopub.execute_input":"2023-02-02T06:21:32.074939Z","iopub.status.idle":"2023-02-02T06:21:42.294329Z","shell.execute_reply.started":"2023-02-02T06:21:32.074904Z","shell.execute_reply":"2023-02-02T06:21:42.293538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extracting file_path from external positive dataset  \n付与すべき情報は？  \n・patient_id→ある  \n・image_id→dcmファイル名であてはめ  \n","metadata":{}},{"cell_type":"code","source":"add_train = pd.read_csv(\"/kaggle/input/ddsm-mammography-positive-case/additional_train.csv\")\nadd_train = add_train.sort_values('patient_id').reset_index(drop=True)\n# Get image path\nbase_path = f'/kaggle/input/ddsm-mammography-positive-case/DICOM/DICOM/'","metadata":{"execution":{"iopub.status.busy":"2023-01-31T11:30:07.074772Z","iopub.execute_input":"2023-01-31T11:30:07.07525Z","iopub.status.idle":"2023-01-31T11:30:07.100165Z","shell.execute_reply.started":"2023-01-31T11:30:07.075201Z","shell.execute_reply":"2023-01-31T11:30:07.099187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"add_train = pd.read_csv(\"/kaggle/input/ddsm-mammography-positive-case/additional_train.csv\")\nadd_train = add_train.sort_values('patient_id').reset_index(drop=True)\n\n# Get image path\nbase_path = f'/kaggle/input/ddsm-mammography-positive-case/DICOM/DICOM/'\nadditional_paths = []\nimage_ids = []\nfor k in tqdm(range(len(add_train))):\n    # image_id(dcmファイル名)を取得\n    read_path = base_path + add_train.iloc[k]['patient_id'] + '*'\n    path_list = sorted(glob.glob(read_path))\n    \n    # image_id・file_pathを付与\n    for path in path_list:        \n        image_id = os.listdir(path)[0]\n        add_path = path + f'/{image_id}'\n        if (add_path in additional_paths):\n            continue\n        else:\n            image_ids.append(image_id)\n            additional_paths.append(add_path)\n\nprint(add_train.shape, len(image_ids), len(additional_paths))\nadd_train['image_id'] = image_ids\nadd_train[\"file_path\"] = additional_paths","metadata":{"execution":{"iopub.status.busy":"2023-01-31T11:30:07.102089Z","iopub.execute_input":"2023-01-31T11:30:07.102561Z","iopub.status.idle":"2023-01-31T11:30:21.962926Z","shell.execute_reply.started":"2023-01-31T11:30:07.102518Z","shell.execute_reply":"2023-01-31T11:30:21.961091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_feat = ['patient_id','laterality','view','cancer','image_id','file_path']\ntrain = pd.concat([train[concat_feat],add_train]).reset_index(drop = True)\ntrain.to_csv('train_add_cancer_cases.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T11:30:21.964319Z","iopub.execute_input":"2023-01-31T11:30:21.964681Z","iopub.status.idle":"2023-01-31T11:30:22.19186Z","shell.execute_reply.started":"2023-01-31T11:30:21.964649Z","shell.execute_reply":"2023-01-31T11:30:22.190683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### External dataset ver2  \nRef. https://www.kaggle.com/code/javigallego/rsna-complete-eda-external-data#3-External-Data  \n・中国のマンモグラフィ データベース  \n★WW/WCあり  \n・正常、良性、または悪性の生検で確認された腫瘍を持つ 1,775 人の患者からの 5,202 のマンモグラム  \n・classificationでbenighとmalignantがある  ","metadata":{}},{"cell_type":"code","source":"def data_info(csv, name=\"Train\"):\n    '''Prints basic information about the datasets we are given.'''\n    '''Inspired by: https://www.kaggle.com/code/andradaolteanu/rsna-fracture-detection-dicom-images-explore'''\n    \n    #print(colored('Columns: ', 'blue', attrs=['bold']), list(csv.columns))\n    \n    display_html(csv.head())\n    if name != 'Sample Submission': print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:24:14.182889Z","iopub.execute_input":"2023-02-02T06:24:14.183941Z","iopub.status.idle":"2023-02-02T06:24:14.189445Z","shell.execute_reply.started":"2023-02-02T06:24:14.1839Z","shell.execute_reply":"2023-02-02T06:24:14.18822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom IPython.display import clear_output, display_html\ndef load_data():\n    '''Load each of the datasets we are given.'''\n    \n    data_dir = Path(\"../input/cmmd2022/TheChineseMammographyDatabase\")\n    metadata = pd.read_csv(data_dir / \"metadata.csv\")\n    sheet = pd.read_excel(data_dir / \"CMMD_clinicaldata_revision.xlsx\")\n    return metadata, sheet\n\nmetadata, sheet = load_data()\nclear_output()\n\nnames = [\"Metadata\", \"Sheet\"]\nfor i, df in enumerate([metadata, sheet]): \n    data_info(df, names[i])","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:24:14.296229Z","iopub.execute_input":"2023-02-02T06:24:14.297315Z","iopub.status.idle":"2023-02-02T06:24:14.572297Z","shell.execute_reply.started":"2023-02-02T06:24:14.297278Z","shell.execute_reply":"2023-02-02T06:24:14.571274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_mix_cases = []\nfor ID in sheet.ID1.unique().tolist():\n    data = sheet[sheet.ID1 == ID]\n    if len(data) == 1:\n        continue\n    else:\n        if len(data[data.classification == 'Benign']) > 0 and len(data[data.classification == 'Malignant']) > 0:\n            label_mix_cases.append(ID)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:56:39.076495Z","iopub.execute_input":"2023-02-02T07:56:39.077679Z","iopub.status.idle":"2023-02-02T07:56:40.159993Z","shell.execute_reply.started":"2023-02-02T07:56:39.077624Z","shell.execute_reply":"2023-02-02T07:56:40.158478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dicom_org = pydicom.read_file('/kaggle/input/rsna-breast-cancer-detection/train_images/10006/1459541791.dcm')\n# dicom_org","metadata":{"execution":{"iopub.status.busy":"2023-02-02T06:26:59.426256Z","iopub.execute_input":"2023-02-02T06:26:59.426727Z","iopub.status.idle":"2023-02-02T06:26:59.576739Z","shell.execute_reply.started":"2023-02-02T06:26:59.426674Z","shell.execute_reply":"2023-02-02T06:26:59.575444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dicom=pydicom.read_file('/kaggle/input/cmmd2022/TheChineseMammographyDatabase/CMMD/D1-0087/07-18-2010-NA-NA-34194/1.000000-NA-99089/1-4.dcm')\n# plt.imshow(dicom.pixel_array)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:46:51.607684Z","iopub.execute_input":"2023-02-02T07:46:51.608109Z","iopub.status.idle":"2023-02-02T07:46:52.333717Z","shell.execute_reply.started":"2023-02-02T07:46:51.608076Z","shell.execute_reply":"2023-02-02T07:46:52.332291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\n\nbenign_ids = sheet[sheet.classification == 'Benign'].ID1.values\nmalignant_ids = sheet[sheet.classification == 'Malignant'].ID1.values\n\n# =============== Code by stpete_ishii https://www.kaggle.com/stpeteishii/siim-acr-dicom-slide-show/notebook?select=stage_2_images\n\nfiles=os.listdir('../input/cmmd2022/TheChineseMammographyDatabase/CMMD')\nprint(len(files))\nn=len(files)\nN=[]\nfor i in range(n//10):\n    N+=[i*10]\n\n# ============== Code by: https://www.kaggle.com/code/mpwolke/mammography-dcm-slide-show\n\ndef load_dicom(path):\n    dicom=pydicom.read_file(path,force=True)\n    data=dicom.pixel_array\n    data=data-np.min(data)\n    if np.max(data) != 0:\n        data=data/np.max(data)\n    data=(data*255).astype(np.uint8)\n    return data\n\ndef load_dicom_line(path):\n    t_paths = sorted(\n        glob.glob(os.path.join(path,\"*/**/***/*.dcm\")), #Original (os.path.join(path,\"*\")),\n        key=lambda x: x[:-4].split(\"_\")[-1],\n    )\n    \n    images = []\n    for filename in tqdm(np.array(t_paths)[N]):\n        data = load_dicom(filename)\n        if data.max() == 0:\n            continue\n        images.append(data)\n        \n    return t_paths, images\n\n# ==============\n\nscans = !ls ../input/cmmd2022/TheChineseMammographyDatabase/CMMD/ -d\nall_paths, all_images = load_dicom_line(scans[0])\n\n# benign_imgs_path = []\n# malignant_imgs_path = []\n# benigh_patient_ids = []\n# benigh_image_ids = []\n# malignant_patient_ids = []\n# malignant_image_ids = []\n\n# for img_path in all_paths: \n#     for idx in benign_ids:\n#         if img_path.find(idx) != -1: \n#             benign_imgs_path.append(img_path)\n#             benigh_patient_ids.append(img_path.rsplit('/',1)[0])\n#             benigh_image_ids.append(img_path.rsplit('/',1)[1])\n#             break\n            \n# for img_path in all_paths: \n#     for idx in malignant_ids:\n#         if img_path.find(idx) != -1: \n#             malignant_imgs_path.append(img_path)\n#             malignant_patient_ids.append(img_path.rsplit('/',1)[0])\n#             malignant_image_ids.append(img_path.rsplit('/',1)[1])\n#             break\nfile_path = []\nimage_ids = []\npatient_ids = []\ncancers = []\nfor img_path in tqdm(all_paths):\n    \n    # sheetからclassificaition→cancerの0or1にして格納\n    Dname = img_path.split('CMMD/')[1].split('/')[0]\n    \n    # D1-やD2-の中でmalignantとbenignが混合するものは使用しない\n    # sheetとmetadataでLRの紐づけが出来ないため\n    if Dname in label_mix_cases:\n        continue\n    if len(sheet[sheet.ID1 == Dname]['classification']) > 1:\n        classification = sheet[sheet.ID1 == Dname]['classification'].head(1).item()\n    else:\n        classification = sheet[sheet.ID1 == Dname]['classification'].item()\n        \n    # Malignant:1, Benign:0\n    if classification == 'Malignant':\n        cancers.append(1)\n    else:\n        cancers.append(0)\n        \n    file_path.append(img_path)\n    patient_ids.append(img_path.rsplit('/',1)[0])\n    image_ids.append(img_path.rsplit('/',1)[1])\n\n# print(add_train.shape, len(image_ids), len(additional_paths))\n# add_train['image_id'] = image_ids\n# add_train[\"file_path\"] = additional_paths","metadata":{"execution":{"iopub.status.busy":"2023-02-02T07:58:25.915798Z","iopub.execute_input":"2023-02-02T07:58:25.916196Z","iopub.status.idle":"2023-02-02T07:58:44.018091Z","shell.execute_reply.started":"2023-02-02T07:58:25.916155Z","shell.execute_reply":"2023-02-02T07:58:44.012014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_CMMD = pd.DataFrame(\n    {'file_path': file_path,\n     'patient_id': patient_ids,\n     'image_id': image_ids,\n     'cancer':cancers\n    })","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:03:00.111507Z","iopub.execute_input":"2023-02-02T08:03:00.111912Z","iopub.status.idle":"2023-02-02T08:03:00.121051Z","shell.execute_reply.started":"2023-02-02T08:03:00.111876Z","shell.execute_reply":"2023-02-02T08:03:00.119981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"concat_feat = ['patient_id','cancer','image_id','file_path']\ntrain = pd.concat([train[concat_feat],df_CMMD]).reset_index(drop = True)\ntrain.to_csv('train_add_CMMD_cancer_cases.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:04:03.799485Z","iopub.execute_input":"2023-02-02T08:04:03.79998Z","iopub.status.idle":"2023-02-02T08:04:04.030011Z","shell.execute_reply.started":"2023-02-02T08:04:03.799938Z","shell.execute_reply":"2023-02-02T08:04:04.028803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# VOI_LUT","metadata":{}},{"cell_type":"code","source":"# Source: https://www.kaggle.com/code/bobdegraaf/dicomsdl-voi-lut\ndef voi_lut(image, dicom):\n    # Load only the variables we need\n    center = dicom['WindowCenter']\n    width = dicom['WindowWidth']\n    bits_stored = dicom['BitsStored']\n    voi_lut_function = dicom['VOILUTFunction']\n    \n    # dicom情報が無かった場合の処理(DSakさん案適応)\n    if center == None:\n        center = (np.min(image) + np.max(image))/2\n    if width == None:\n        width = (np.max(image) - np.min(image))\n\n    # For sigmoid it's a list, otherwise a single value\n    if isinstance(center, list):\n        center = center[0]\n    if isinstance(width, list):\n        width = width[0]\n\n    # Set y_min, max & range\n    y_min = 0\n    y_max = float(2**bits_stored - 1)\n    y_range = y_max\n\n    # Function with default LINEAR (so for Nan, it will use linear)\n    if voi_lut_function == 'SIGMOID':\n        image = y_range / (1 + np.exp(-4 * (image - center) / width)) + y_min\n    else:\n        # Checks width for < 1 (in our case not necessary, always >= 750)\n        center -= 0.5\n        width -= 1        \n#         y_min = center - width//2 \n#         y_max = center + width//2 \n#         image[image<y_min] = y_min \n#         image[image>y_max] = y_max \n\n        below = image <= (center - width / 2)\n        above = image > (center + width / 2)\n        between = np.logical_and(~below, ~above)\n\n        image[below] = y_min\n        image[above] = y_max\n        if between.any():\n            image[between] = (\n                ((image[between] - center) / width + 0.5) * y_range + y_min\n            )\n\n    # Normalize to have 0 as background, some images are reversed where 0 is max intensity\n    if dicom['PhotometricInterpretation'] == 'MONOCHROME1':\n        image = np.max(image) - image\n\n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:36:55.412977Z","iopub.execute_input":"2023-02-02T08:36:55.413388Z","iopub.status.idle":"2023-02-02T08:36:55.427655Z","shell.execute_reply.started":"2023-02-02T08:36:55.413355Z","shell.execute_reply":"2023-02-02T08:36:55.426412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Crop Image","metadata":{}},{"cell_type":"code","source":"# Smooth vector used to smoothen sums/stds of axes\ndef smooth(l):\n    # kernel size is 1% of vector\n    kernel_size = int(len(l) * 0.01)\n    kernel = np.ones(kernel_size) / kernel_size\n    return np.convolve(l, kernel, mode='same')\n\n# X Crop offset based on first column with sum below 5% of maximum column sums*std\ndef get_x_offset(image, max_col_sum_ratio_threshold=0.05, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Percentual margin added to offset\n    margin = int(image.shape[1] * 0.00)\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=0).squeeze()) * smooth(image.std(axis=0).squeeze())\n    # Find maximum sum in first 75% of columns\n    vv_argmax = vv[:int(image.shape[1] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_col_sum_ratio_threshold\n    \n    # Find first column after maximum column below threshold value\n    for offset, v in enumerate(vv):\n        # Start searching from vv_argmax\n        if offset < vv_argmax:\n            continue\n        \n        # Column below threshold value found\n        if v < vv_threshold:\n            offset = min(W, offset + margin)\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[1].imshow(image)\n        debug[1].set_title('X Offset')\n        vv_scale = H / vv.max() * 0.90\n        # Values\n        debug[1].plot(H - vv * vv_scale , c='red', label='vv')\n        # Threshold\n        debug[1].hlines(H - vv_threshold * vv_scale, 0, W -1, colors='orange', label='threshold')\n        # Max Value\n        debug[1].scatter(vv_argmax, H - vv[vv_argmax] * vv_scale, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[1].scatter(offset, H - vv[offset] * vv_scale, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[1].set_ylim(H, 0)\n        debug[1].legend()\n        debug[1].axis('off')\n        \n    return offset\n\n# Y Crop offset based on first bottom and top rows with sum below 10% of maximum row sum*std\ndef get_y_offsets(image, max_row_sum_ratio_threshold=0.10, debug=None):\n    # Image Dimensions\n    H, W = image.shape\n    # Margin to add to offsets\n    margin = 0\n    # Threshold values based on smoothed sum x std to capture varying intensity columns\n    vv = smooth(image.sum(axis=1).squeeze()) * smooth(image.std(axis=1).squeeze())\n    # Find maximum sum * std row in inter quartile rows\n    vv_argmax = int(image.shape[0] * 0.25) + vv[int(image.shape[0] * 0.25):int(image.shape[0] * 0.75)].argmax()\n    # Threshold value\n    vv_threshold = vv.max() * max_row_sum_ratio_threshold\n    # Default crop offsets\n    offset_bottom = 0\n    offset_top = H\n\n    # Bottom offset, search from argmax to bottom\n    for offset in reversed(range(0, vv_argmax)):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_bottom = offset\n            break\n    \n    if isinstance(debug, np.ndarray):\n        debug[2].imshow(image)\n        debug[2].set_title('Y Bottom Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[2].plot(vv * vv_scale, np.arange(H), c='red', label='vv')\n        # Threshold\n        debug[2].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[2].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[2].scatter(vv[offset_bottom] * vv_scale, offset_bottom, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[2].legend()\n        debug[2].axis('off')\n            \n    # Top offset, search from argmax to top\n    for offset in range(vv_argmax, H):\n        v = vv[offset]\n        if v < vv_threshold:\n            offset_top = offset\n            break\n            \n    if isinstance(debug, np.ndarray):\n        debug[3].imshow(image)\n        debug[3].set_title('Y Top Offset')\n        vv_scale = W / vv.max() * 0.90\n        # Values\n        debug[3].plot(vv * vv_scale, np.arange(H) , c='red', label='vv')\n        # Threshold\n        debug[3].vlines(vv_threshold * vv_scale, 0, H -1, colors='orange', label='threshold')\n        # Max Value\n        debug[3].scatter(vv[vv_argmax] * vv_scale, vv_argmax, c='blue', s=100, label='Max', zorder=np.PINF)\n        # First Column Below Threshold\n        debug[3].scatter(vv[offset_top] * vv_scale, offset_top, c='purple', s=100, label='Offset', zorder=np.PINF)\n        debug[2].set_ylim(H, 0)\n        debug[3].legend()\n        debug[3].axis('off')\n            \n    return max(0, offset_bottom - margin), min(image.shape[0], offset_top + margin)\n\n# Crop image and pad offsets to target image height/width ratio to preserve information\ndef crop(image, size=None, debug=False):\n    # Image dimensions\n    H, W = image.shape\n    # Compute x/bottom/top offsets\n    x_offset = get_x_offset(image, debug=debug)\n    offset_bottom, offset_top = get_y_offsets(image[:,:x_offset], debug=debug)\n    # Crop Height and Width\n    h_crop = offset_top - offset_bottom\n    w_crop = x_offset\n    \n    # Pad crop offsets to target aspect ratio\n    if size is not None:\n        # Height too large, pad x offset\n        if (h_crop / w_crop) > TARGET_HEIGHT_WIDTH_RATIO:\n            x_offset += int(h_crop / TARGET_HEIGHT_WIDTH_RATIO - w_crop)\n        else:\n            # Height too small, pad bottom/top offsets\n            offset_bottom -= int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_bottom_correction = max(0, -offset_bottom)\n            offset_bottom += offset_bottom_correction\n\n            offset_top += int(0.50 * (w_crop * TARGET_HEIGHT_WIDTH_RATIO - h_crop))\n            offset_top += offset_bottom_correction\n        \n    # Crop Image\n    image = image[offset_bottom:offset_top:,:x_offset]\n        \n    return image","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:36:56.706605Z","iopub.execute_input":"2023-02-02T08:36:56.707015Z","iopub.status.idle":"2023-02-02T08:36:56.7362Z","shell.execute_reply.started":"2023-02-02T08:36:56.706983Z","shell.execute_reply":"2023-02-02T08:36:56.735103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility","metadata":{}},{"cell_type":"code","source":"# based on: https://www.kaggle.com/code/remekkinas/fast-dicom-processing-1-6-2x-faster?scriptVersionId=113360473\ndef process(file_path, size=None, dicom_process=True, ret_target=False, crop_image=False, apply_clahe=APPLY_CLAHE, apply_eq_hist=APPLY_EQ_HIST, debug=False):\n    # Read Dicom File\n    dicom = dicomsdl.open(file_path)\n    image = dicom.pixelData()\n    \n    # Save original image for debug purposes\n    if debug:\n        fig, axes = plt.subplots(1, 5, figsize=(20,10))\n        image0 = np.copy(image)\n        axes[0].imshow(image0)\n        axes[0].set_title('Original Image')\n        axes[0].axis('off')\n    else:\n        axes = False\n        \n    # voi_lut\n    image = voi_lut(image, dicom)\n\n    # Normalize [0,1] range\n    image = (image - image.min()) / (image.max() - image.min())\n\n    # Convert to uint8 image in range [0, 255]\n    image = (image * 255).astype(np.uint8)\n    \n    # Normalize to left/right orientation by flipping right/left oriented images\n    h0, w0 = image.shape\n    if image[:,int(-w0 * 0.10):].sum() > image[:,:int(w0 * 0.10)].sum():\n        image = np.flip(image, axis=1)\n    \n    if crop_image:\n        image = crop(image, size=size, debug=axes)\n    \n    # Resize\n    if size is not None:\n        # Pad black pixels to get correct image ratios\n        h, w = image.shape\n        if (h / w) > TARGET_HEIGHT_WIDTH_RATIO:\n            pad = int(h / TARGET_HEIGHT_WIDTH_RATIO - w)\n            image = np.pad(image, [[0,0], [0, pad]])\n            h, w = image.shape\n        else:\n            pad = int(0.50 * (w * TARGET_HEIGHT_WIDTH_RATIO - h))\n            image = np.pad(image, [[pad, pad], [0,0]])\n            h, w = image.shape\n        # Resize\n        image = cv2.resize(image, size, interpolation=cv2.INTER_AREA)\n        \n    # Apply CLAHE contrast enhancement\n    # https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\n    if apply_clahe:\n        image = CLAHE.apply(image)\n        \n    # Apply Histogram Equalization\n    # https://docs.opencv.org/4.x/d5/daf/tutorial_py_histogram_equalization.html\n    if apply_eq_hist:\n        image = cv2.equalizeHist(image)\n        \n    if debug:\n        axes[4].imshow(image)\n        axes[4].set_title('Processed Image')\n        axes[4].axis('off')\n        plt.show()\n\n    # Return Cancer Target\n    if ret_target:\n        if 'DICOM' in file_path:\n            patient_id = file_path.split('/')[-2].rsplit('_',1)[0]\n            image_id = file_path.split('/')[-1]\n        elif 'CMMD' in file_path:\n            patient_id = file_path.rsplit('/',1)[0]\n            image_id = file_path.rsplit('/',1)[1]\n        else:\n            patient_id = int(file_path.split('/')[-2])\n            image_id = int(file_path.split('/')[-1].split('.')[0])\n\n        target = PATIENT_ID_IMAGE_ID2CANCER[(patient_id, image_id)]\n        \n        return image, target\n    # Return image Only\n    else:\n        if debug:\n            return image0, image\n        else:\n            return image","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:36:57.921262Z","iopub.execute_input":"2023-02-02T08:36:57.921672Z","iopub.status.idle":"2023-02-02T08:36:57.938257Z","shell.execute_reply.started":"2023-02-02T08:36:57.92163Z","shell.execute_reply":"2023-02-02T08:36:57.937294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Processing","metadata":{}},{"cell_type":"code","source":"IS_INTERACTIVE = False","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:36:59.251537Z","iopub.execute_input":"2023-02-02T08:36:59.252299Z","iopub.status.idle":"2023-02-02T08:36:59.256828Z","shell.execute_reply.started":"2023-02-02T08:36:59.25226Z","shell.execute_reply":"2023-02-02T08:36:59.255581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # trainの中での最頻値を探す\n# centers = {}\n# widths = {}\n# voi_lut_functions = {}\n# for i in tqdm(range(len(train))):\n#     fp = train.iloc[i]['file_path']\n#     dicom = dicomsdl.open(fp)\n#     bits_stored = dicom['BitsStored']    \n#     if bits_stored != 16:\n#         continue\n#     else:        \n#         if dicom['VOILUTFunction'] in voi_lut_functions:\n#             current_num = voi_lut_functions.get(dicom['VOILUTFunction'])\n#             voi_lut_functions[dicom['VOILUTFunction']] = current_num + 1\n#         else:\n#             voi_lut_functions[dicom['VOILUTFunction']] = 1\n        \n#         if type(dicom['WindowCenter']) == list:\n#             if dicom['WindowCenter'][0] in centers:\n#                 current_num = centers.get(dicom['WindowCenter'][0])\n#                 centers[dicom['WindowCenter'][0]] = current_num + 1\n#             else:\n#                 centers[dicom['WindowCenter'][0]] = 1\n#         else:\n#             if dicom['WindowCenter'] in centers:\n#                 current_num = centers.get(dicom['WindowCenter'])\n#                 centers[dicom['WindowCenter']] = current_num + 1\n#             else:\n#                 centers[dicom['WindowCenter']] = 1\n        \n#         if type(dicom['WindowWidth']) == list:\n#             if dicom['WindowWidth'][0] in widths:\n#                 current_num = widths.get(dicom['WindowWidth'][0])\n#                 widths[dicom['WindowWidth'][0]] = current_num + 1\n#             else:\n#                 widths[dicom['WindowWidth'][0]] = 1\n#         else:\n#             if dicom['WindowWidth'] in widths:\n#                 current_num = widths.get(dicom['WindowWidth'])\n#                 widths[dicom['WindowWidth']] = current_num + 1\n#             else:\n#                 widths[dicom['WindowWidth']] = 1      ","metadata":{"execution":{"iopub.status.busy":"2023-01-29T07:40:07.517037Z","iopub.execute_input":"2023-01-29T07:40:07.51817Z","iopub.status.idle":"2023-01-29T08:44:14.210699Z","shell.execute_reply.started":"2023-01-29T07:40:07.518127Z","shell.execute_reply":"2023-01-29T08:44:14.206006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(centers)\n# print(widths)\n# print(voi_lut_functions)","metadata":{"execution":{"iopub.status.busy":"2023-01-29T08:44:17.392427Z","iopub.execute_input":"2023-01-29T08:44:17.393757Z","iopub.status.idle":"2023-01-29T08:44:17.414899Z","shell.execute_reply.started":"2023-01-29T08:44:17.393675Z","shell.execute_reply":"2023-01-29T08:44:17.413243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example Processed Images","metadata":{}},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"# Maps a patient_id and image_id to cancer target\nPATIENT_ID_IMAGE_ID2CANCER = train.set_index(['patient_id', 'image_id'])['cancer'].to_dict()","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:37:01.825452Z","iopub.execute_input":"2023-02-02T08:37:01.825884Z","iopub.status.idle":"2023-02-02T08:37:02.025541Z","shell.execute_reply.started":"2023-02-02T08:37:01.825849Z","shell.execute_reply":"2023-02-02T08:37:02.024361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Chunk Generation","metadata":{}},{"cell_type":"code","source":"# Make Pairs of Views as input to the model\nFILE_PATHS_PAIRS = []\nfor row_idx, row in tqdm(train.iterrows(), total=len(train)):\n        FILE_PATHS_PAIRS.append(row[['patient_id', 'image_id']].values)\n        \nFILE_PATHS_PAIRS = np.array(FILE_PATHS_PAIRS, dtype=object)\nprint(f'FILE_PATHS_PAIRS shape: {FILE_PATHS_PAIRS.shape}')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:37:06.79533Z","iopub.execute_input":"2023-02-02T08:37:06.795753Z","iopub.status.idle":"2023-02-02T08:37:35.281336Z","shell.execute_reply.started":"2023-02-02T08:37:06.795721Z","shell.execute_reply":"2023-02-02T08:37:35.279951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put every image in a seperate TFRecord file\nN_CHUNKS = 100\nCHUNKS = np.array_split(FILE_PATHS_PAIRS, N_CHUNKS)\n\nprint(f'N_CHUNKS: {N_CHUNKS}, CHUNK len: {len(CHUNKS[0])}, shape: {CHUNKS[0].shape}')","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:37:40.663645Z","iopub.execute_input":"2023-02-02T08:37:40.664087Z","iopub.status.idle":"2023-02-02T08:37:40.670487Z","shell.execute_reply.started":"2023-02-02T08:37:40.664045Z","shell.execute_reply":"2023-02-02T08:37:40.669323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Single sample processing\ndef process_chunk(args):\n    patient_id, image_id = args\n    # Define file path\n    if  type(patient_id) != int:\n        if 'CMMD' in patient_id:\n            fp = patient_id + '/' + image_id\n        else:\n            add_id = image_id.split('.dcm')[0]\n            add_id = add_id.rsplit('_',1)[-1]\n            fp = '/kaggle/input/ddsm-mammography-positive-case/DICOM/DICOM/' + patient_id + '_' + add_id + '/' + image_id\n    else:\n        fp = f'/kaggle/input/rsna-breast-cancer-detection/train_images/{patient_id}/{image_id}.dcm'\n    \n    # Get processed image and target\n    image, target = process(fp, size=(TARGET_WIDTH, TARGET_HEIGHT), ret_target=True, crop_image=True)\n\n    # Make grayscale channel\n    image = np.expand_dims(image, 2)\n    \n    # Encode PNG\n    if IMAGE_FORMAT == 'PNG':\n        image_serialized = tf.io.encode_png(image, compression=9).numpy()\n    # Encode JPEG\n    else:\n        image_serialized = tf.io.encode_jpeg(image, quality=IMAGE_QUALITY, optimize_size=True).numpy()\n    \n    return image_serialized, target, patient_id, image_id","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:38:41.546264Z","iopub.execute_input":"2023-02-02T08:38:41.546744Z","iopub.status.idle":"2023-02-02T08:38:41.556669Z","shell.execute_reply.started":"2023-02-02T08:38:41.546683Z","shell.execute_reply":"2023-02-02T08:38:41.555488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_tf_records(chunks):\n    for chunk_idx, chunk in enumerate(tqdm(chunks)):\n        print(f'===== GENERATING TFRECORDS {chunk_idx} =====')\n        if chunk_idx < 91:\n            continue\n        \n        tfrecord_name = f'batch_{chunk_idx}.tfrecords'\n        \n        # Create the actual TFRecords\n        options = tf.io.TFRecordOptions(compression_type='GZIP', compression_level=9)\n        with tf.io.TFRecordWriter(tfrecord_name, options=options) as file_writer:\n            # Process Samples in Chunk in Parallell\n            jobs = [joblib.delayed(process_chunk)(args) for args in chunk]\n            chunk_processed = joblib.Parallel(\n                n_jobs=cpu_count(),\n                verbose=0,\n                backend='multiprocessing',\n                prefer='threads',\n            )(jobs)\n            \n            # Add Processed Samples to TFRecord\n            for image, target, patient_id, image_id in chunk_processed:\n                record_bytes = tf.train.Example(features=tf.train.Features(feature={\n                    # Image\n                    'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image])),\n\n                    # target\n                    'target': tf.train.Feature(int64_list=tf.train.Int64List(value=[target])),\n                    \n                    # 外部データでpatient_idとimage_idを取得する処理が煩わしいこと\n                    # またtrainの中でpatient_idとimage_idを使用していないことから、保存していない\n#                     # patient_id\n#                     'patient_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[patient_id])),\n                    \n#                     # image_id\n#                     'image_id': tf.train.Feature(int64_list=tf.train.Int64List(value=[image_id])),\n                })).SerializeToString()\n                file_writer.write(record_bytes)\n            \n# Create TFRecords\nif IS_INTERACTIVE:\n    to_tf_records(CHUNKS[:10])\nelse:\n    to_tf_records(CHUNKS)","metadata":{"execution":{"iopub.status.busy":"2023-02-02T08:38:44.952845Z","iopub.execute_input":"2023-02-02T08:38:44.954057Z","iopub.status.idle":"2023-02-02T08:42:38.746162Z","shell.execute_reply.started":"2023-02-02T08:38:44.953999Z","shell.execute_reply":"2023-02-02T08:42:38.737332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check TFRecords","metadata":{}},{"cell_type":"code","source":"N = 16 if IS_INTERACTIVE else 50","metadata":{"execution":{"iopub.status.busy":"2023-01-31T12:46:13.046092Z","iopub.execute_input":"2023-01-31T12:46:13.047475Z","iopub.status.idle":"2023-01-31T12:46:13.055143Z","shell.execute_reply.started":"2023-01-31T12:46:13.047405Z","shell.execute_reply":"2023-01-31T12:46:13.052955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to decode the TFRecords\ndef decode_tfrecord(record_bytes):\n    features = tf.io.parse_single_example(record_bytes, {\n        'image': tf.io.FixedLenFeature([], tf.string),\n        'target': tf.io.FixedLenFeature([], tf.int64),\n    })\n        \n    if IMAGE_FORMAT == 'PNG':\n        image = tf.io.decode_png(features['image'], channels=N_CHANNELS)\n    else:\n        image = tf.io.decode_jpeg(features['image'], channels=N_CHANNELS)\n        \n    image = tf.reshape(image, [TARGET_HEIGHT, TARGET_WIDTH, N_CHANNELS])\n\n    target = features['target']\n    \n    return image, target","metadata":{"execution":{"iopub.status.busy":"2023-01-31T12:46:13.057318Z","iopub.execute_input":"2023-01-31T12:46:13.058194Z","iopub.status.idle":"2023-01-31T12:46:13.078166Z","shell.execute_reply.started":"2023-01-31T12:46:13.05814Z","shell.execute_reply":"2023-01-31T12:46:13.075498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"More on Tensorflow TFRecord Datasets: [TFRecordDataset](https://www.tensorflow.org/api_docs/python/tf/data/TFRecordDataset)","metadata":{}},{"cell_type":"code","source":"# Sample TFRecord Dataset\ndef get_train_dataset():\n    # Read all TFRecord file paths\n    FNAMES_TRAIN_TFRECORDS = tf.io.gfile.glob('./*.tfrecords')\n    # initialize TFRecord dataset\n    train_dataset = tf.data.TFRecordDataset(FNAMES_TRAIN_TFRECORDS, num_parallel_reads=1, compression_type='GZIP')\n    # Decode samples by mapping with decode function\n    train_dataset = train_dataset.map(decode_tfrecord)\n    # Batch samples\n    train_dataset = train_dataset.batch(N)\n    \n    return train_dataset","metadata":{"execution":{"iopub.status.busy":"2023-01-31T12:46:13.08279Z","iopub.execute_input":"2023-01-31T12:46:13.083427Z","iopub.status.idle":"2023-01-31T12:46:13.092276Z","shell.execute_reply.started":"2023-01-31T12:46:13.083384Z","shell.execute_reply":"2023-01-31T12:46:13.091196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shows a batch of images\ndef show_batch(dataset, rows=N, cols=1):\n    images, targets = next(iter(dataset))\n    images = np.moveaxis(images, 3, 1)\n    fig, axes = plt.subplots(nrows=rows, ncols=cols, figsize=(cols*6, rows*10))\n    for r in range(rows):\n        for c in range(cols):\n            img = images[r,c]\n            axes[r].imshow(img)\n            if c == 0:\n                target = targets[r]\n                axes[r].set_title(f'target: {target}', fontsize=12, pad=16)\n        \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-31T12:46:13.093898Z","iopub.execute_input":"2023-01-31T12:46:13.095063Z","iopub.status.idle":"2023-01-31T12:46:13.108775Z","shell.execute_reply.started":"2023-01-31T12:46:13.095004Z","shell.execute_reply":"2023-01-31T12:46:13.106503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show Example Batch\ntrain_dataset = get_train_dataset()\nshow_batch(train_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-01-31T12:46:13.111133Z","iopub.execute_input":"2023-01-31T12:46:13.112024Z","iopub.status.idle":"2023-01-31T12:46:32.082557Z","shell.execute_reply.started":"2023-01-31T12:46:13.111971Z","shell.execute_reply":"2023-01-31T12:46:32.080131Z"},"trusted":true},"execution_count":null,"outputs":[]}]}