{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q python-gdcm\n!pip install -q pylibjpeg\n# !pip install -q dicomsdl","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:22:34.691108Z","iopub.execute_input":"2023-07-28T18:22:34.691567Z","iopub.status.idle":"2023-07-28T18:22:56.426579Z","shell.execute_reply.started":"2023-07-28T18:22:34.691523Z","shell.execute_reply":"2023-07-28T18:22:56.42525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os,shutil, pandas as pd\nfrom glob import glob\nfrom tqdm import tqdm\nimport zipfile\nimport cv2\nimport numpy as np","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:47.632397Z","iopub.execute_input":"2023-07-28T18:27:47.633136Z","iopub.status.idle":"2023-07-28T18:27:47.639199Z","shell.execute_reply.started":"2023-07-28T18:27:47.63309Z","shell.execute_reply":"2023-07-28T18:27:47.638006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_PATH = '/kaggle/input/rsna-2023-abdominal-trauma-detection'\nIMG_DIR = '/tmp/Dataset/rsna-atd'","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:49.037175Z","iopub.execute_input":"2023-07-28T18:27:49.037919Z","iopub.status.idle":"2023-07-28T18:27:49.042541Z","shell.execute_reply.started":"2023-07-28T18:27:49.037871Z","shell.execute_reply":"2023-07-28T18:27:49.04137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resize_dim = 512\nimg_size = [resize_dim, resize_dim]\nIDX = 0\nPARTS = 1\nprint(f\"Image Size: {img_size}\")\nprint(f\"Dim: {np.prod(img_size)**0.5: 0.2f}\")","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:49.369439Z","iopub.execute_input":"2023-07-28T18:27:49.369876Z","iopub.status.idle":"2023-07-28T18:27:49.379763Z","shell.execute_reply.started":"2023-07-28T18:27:49.369826Z","shell.execute_reply":"2023-07-28T18:27:49.378554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Meta Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(f'{ROOT_PATH}/train.csv')\nimg_lvl_df = pd.read_csv(f'{ROOT_PATH}/image_level_labels.csv')\ntrain_df = train_df.merge(img_lvl_df, on=['patient_id'], how='right')\n\ntrain_df['image_path'] = (f'{ROOT_PATH}/train_images' + '/'\n                          + train_df.patient_id.astype(str)\n                          + '/' + train_df.series_id.astype(str)\n                          + '/' + train_df.instance_number.astype(str) + '.dcm')\n\nprint('Train:')\nprint(f'# Size: {len(train_df)}')\ndisplay(train_df.head(2))","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:51.320623Z","iopub.execute_input":"2023-07-28T18:27:51.321007Z","iopub.status.idle":"2023-07-28T18:27:51.500083Z","shell.execute_reply.started":"2023-07-28T18:27:51.320974Z","shell.execute_reply":"2023-07-28T18:27:51.498919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_paths = glob('/kaggle/input/rsna-2023-abdominal-trauma-detection/test_images/*/*/*dcm')\n\ntest_df = pd.DataFrame(test_paths, columns=[\"image_path\"])\ntest_df['patient_id'] = test_df.image_path.map(lambda x: x.split('/')[-3]).astype(int)\ntest_df['series_id'] = test_df.image_path.map(lambda x: x.split('/')[-2]).astype(int)\ntest_df['instance_number'] = test_df.image_path.map(lambda x: x.split('/')[-1].replace('.dcm','')).astype(int)\nprint('Test:')\nprint(f'# Size: {len(test_df)}')\ndisplay(test_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:53.29379Z","iopub.execute_input":"2023-07-28T18:27:53.294577Z","iopub.status.idle":"2023-07-28T18:27:53.351185Z","shell.execute_reply.started":"2023-07-28T18:27:53.29453Z","shell.execute_reply":"2023-07-28T18:27:53.350429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# `.dcm` to `.png`","metadata":{}},{"cell_type":"code","source":"!rm -r {IMG_DIR}\nos.makedirs(f'{IMG_DIR}/train_images', exist_ok = True)\nos.makedirs(f'{IMG_DIR}/test_images', exist_ok = True)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:55.178531Z","iopub.execute_input":"2023-07-28T18:27:55.179244Z","iopub.status.idle":"2023-07-28T18:27:56.301041Z","shell.execute_reply.started":"2023-07-28T18:27:55.179192Z","shell.execute_reply":"2023-07-28T18:27:56.299847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pydicom\nimport math\nimport cv2\n    \ndef standardize_pixel_array(dcm: pydicom.dataset.FileDataset) -> np.ndarray:\n    # Correct DICOM pixel_array if PixelRepresentation == 1.\n    pixel_array = dcm.pixel_array\n    if dcm.PixelRepresentation == 1:\n        bit_shift = dcm.BitsAllocated - dcm.BitsStored\n        dtype = pixel_array.dtype \n        new_array = (pixel_array << bit_shift).astype(dtype) >>  bit_shift\n        pixel_array = pydicom.pixel_data_handlers.util.apply_modality_lut(new_array, dcm)\n    return pixel_array\n\ndef read_xray(path, fix_monochrome = True):\n    dicom = pydicom.dcmread(path)\n    data = standardize_pixel_array(dicom)\n    data = data - np.min(data)\n    data = data / (np.max(data) + 1e-5)\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = 1.0 - data\n    return data\n\ndef resize_and_save(file_path):\n    img = read_xray(file_path)\n    h, w = img.shape[:2]  # orig hw\n    img = cv2.resize(img, (resize_dim, resize_dim), cv2.INTER_LINEAR)\n    img = (img * 255).astype(np.uint8)\n    \n    sub_path = file_path.split(\"/\",4)[-1].split('.dcm')[0] + '.png'\n    infos = sub_path.split('/')\n    pid = infos[-3]\n    sid = infos[-2]\n    iid = infos[-1]; iid = iid.replace('.png','')\n    new_path = os.path.join(IMG_DIR, sub_path)\n    os.makedirs(new_path.rsplit('/',1)[0], exist_ok=True)\n    cv2.imwrite(new_path, img,\n#                 [cv2.IMWRITE_PNG_COMPRESSION, 1],\n               )\n    return pid,sid,iid,w,h","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-07-28T18:27:56.303879Z","iopub.execute_input":"2023-07-28T18:27:56.304406Z","iopub.status.idle":"2023-07-28T18:27:56.620489Z","shell.execute_reply.started":"2023-07-28T18:27:56.304349Z","shell.execute_reply":"2023-07-28T18:27:56.619556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"%%time\nfrom joblib import Parallel, delayed\nSIZE = -(-len(train_df) // PARTS)\nfile_paths = train_df.image_path.tolist()[IDX*SIZE:(IDX+1)*SIZE]\nimgsize_train = Parallel(n_jobs=-1,backend='threading')(delayed(resize_and_save)(file_path)\\\n                                                  for file_path in tqdm(file_paths, leave=True, position=0))","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:27:57.745337Z","iopub.execute_input":"2023-07-28T18:27:57.745755Z","iopub.status.idle":"2023-07-28T18:33:15.546493Z","shell.execute_reply.started":"2023-07-28T18:27:57.745719Z","shell.execute_reply":"2023-07-28T18:33:15.544627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid, sid, iid, width, height = list(zip(*imgsize_train))\nmeta_df = pd.DataFrame({'patient_id':pid,\n                        'series_id':sid,\n                       'instance_number':iid,\n                       'width':width,\n                       'height':height})\nmeta_df[['patient_id','series_id','instance_number']] = meta_df[['patient_id','series_id','instance_number']].astype(int)\ntrain_df = train_df.merge(meta_df, on=['patient_id','series_id','instance_number'], how='right')\ntrain_df.head(2)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:34:50.565675Z","iopub.execute_input":"2023-07-28T18:34:50.566116Z","iopub.status.idle":"2023-07-28T18:34:50.642367Z","shell.execute_reply.started":"2023-07-28T18:34:50.566079Z","shell.execute_reply":"2023-07-28T18:34:50.641304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test","metadata":{}},{"cell_type":"code","source":"%%time\nfrom joblib import Parallel, delayed\nfile_paths = test_df.image_path.tolist()\nimgsize_test = Parallel(n_jobs=-1,backend='threading')(delayed(resize_and_save)(file_path)\\\n                                                  for file_path in tqdm(file_paths))","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-28T18:34:52.086356Z","iopub.execute_input":"2023-07-28T18:34:52.086781Z","iopub.status.idle":"2023-07-28T18:34:52.201741Z","shell.execute_reply.started":"2023-07-28T18:34:52.086738Z","shell.execute_reply":"2023-07-28T18:34:52.20047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pid, sid, iid, width, height = list(zip(*imgsize_test))\nmeta_df = pd.DataFrame({'patient_id':pid,\n                        'series_id':sid,\n                       'instance_number':iid,\n                       'width':width,\n                       'height':height})\nmeta_df[['patient_id','series_id','instance_number']] = meta_df[['patient_id','series_id','instance_number']].astype(int)\ntest_df = test_df.merge(meta_df, on=['patient_id','series_id','instance_number'], how='right')\ntest_df.head(2)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-07-28T18:34:52.927815Z","iopub.execute_input":"2023-07-28T18:34:52.928198Z","iopub.status.idle":"2023-07-28T18:34:52.954559Z","shell.execute_reply.started":"2023-07-28T18:34:52.928164Z","shell.execute_reply":"2023-07-28T18:34:52.95344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Check","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(8,8))\nplt.scatter(train_df['width'], train_df['height'])","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:34:54.018038Z","iopub.execute_input":"2023-07-28T18:34:54.018507Z","iopub.status.idle":"2023-07-28T18:34:54.295873Z","shell.execute_reply.started":"2023-07-28T18:34:54.018465Z","shell.execute_reply":"2023-07-28T18:34:54.294623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Check Image","metadata":{}},{"cell_type":"code","source":"path = train_df[train_df.width<700].image_path.iloc[0].replace(ROOT_PATH, IMG_DIR).replace('.dcm','.png')\nimg = cv2.imread(path, cv2.IMREAD_UNCHANGED)\nplt.figure(figsize=(10,10))\nplt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:34:55.870872Z","iopub.execute_input":"2023-07-28T18:34:55.871271Z","iopub.status.idle":"2023-07-28T18:34:56.203079Z","shell.execute_reply.started":"2023-07-28T18:34:55.871237Z","shell.execute_reply":"2023-07-28T18:34:56.202168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = train_df[train_df.width>700].image_path.iloc[0].replace(ROOT_PATH, IMG_DIR).replace('.dcm','.png')\nimg = cv2.imread(path, cv2.IMREAD_UNCHANGED)\nplt.figure(figsize=(10,10))\nplt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T18:35:02.052064Z","iopub.execute_input":"2023-07-28T18:35:02.052783Z","iopub.status.idle":"2023-07-28T18:35:02.35526Z","shell.execute_reply.started":"2023-07-28T18:35:02.052738Z","shell.execute_reply":"2023-07-28T18:35:02.354269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CSV","metadata":{}},{"cell_type":"code","source":"train_df.to_csv(f'{IMG_DIR}/train.csv',index = False)\ntest_df.to_csv(f'{IMG_DIR}/test.csv',index = False)\n\nshutil.copy(f'{ROOT_PATH}/train_series_meta.csv',f'{IMG_DIR}/')\nshutil.copy(f'{ROOT_PATH}/test_series_meta.csv',f'{IMG_DIR}/')\nshutil.copy(f'{ROOT_PATH}/sample_submission.csv',f'{IMG_DIR}/')","metadata":{"execution":{"iopub.status.busy":"2023-07-27T17:30:37.858029Z","iopub.execute_input":"2023-07-27T17:30:37.858537Z","iopub.status.idle":"2023-07-27T17:30:38.247049Z","shell.execute_reply.started":"2023-07-27T17:30:37.858466Z","shell.execute_reply":"2023-07-27T17:30:38.246127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -al {IMG_DIR}","metadata":{"execution":{"iopub.status.busy":"2023-07-27T17:30:40.894047Z","iopub.execute_input":"2023-07-27T17:30:40.894488Z","iopub.status.idle":"2023-07-27T17:30:42.001582Z","shell.execute_reply.started":"2023-07-27T17:30:40.894446Z","shell.execute_reply":"2023-07-27T17:30:42.000188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset","metadata":{}},{"cell_type":"code","source":"from zipfile import ZipFile\n\nzipObj = ZipFile(f'/kaggle/working/rsna-acd.zip', 'w')\n\nfile_paths = glob(f'{IMG_DIR}/**/*',recursive = True)\nprint(f'Total Files:{len(file_paths)}')\nprint('Zippping...')\nfor file_path in tqdm(file_paths):\n    zipObj.write(file_path, file_path[len(IMG_DIR):])\n    os.remove(file_path) if os.path.isfile(file_path) else None\nzipObj.close()","metadata":{"execution":{"iopub.status.busy":"2023-07-27T17:30:43.871679Z","iopub.execute_input":"2023-07-27T17:30:43.87219Z","iopub.status.idle":"2023-07-27T17:31:20.449626Z","shell.execute_reply.started":"2023-07-27T17:30:43.872146Z","shell.execute_reply":"2023-07-27T17:31:20.448357Z"},"trusted":true},"execution_count":null,"outputs":[]}]}