{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install -q \"python-gdcm\" pydicom pylibjpeg pylibjpeg-libjpeg \"opencv-python-headless==4.5.4.60\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pwd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport glob\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pydicom\nfrom tqdm import tqdm\nimport numpy as np\nfrom datetime import datetime\n\n\ndata_path = '/kaggle/input/rsna-breast-cancer-detection/'\ntrain_data_path = '/kaggle/input/rsna-breast-cancer-detection/train_images/'\n\nos.listdir(data_path)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T02:30:53.277383Z","iopub.execute_input":"2023-01-28T02:30:53.278073Z","iopub.status.idle":"2023-01-28T02:30:53.449375Z","shell.execute_reply.started":"2023-01-28T02:30:53.277954Z","shell.execute_reply":"2023-01-28T02:30:53.448581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try to \nfile_names = glob.glob(data_path + 'train_images' + '/*/*.dcm')\nlen(file_names)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T02:30:58.09824Z","iopub.execute_input":"2023-01-28T02:30:58.098835Z","iopub.status.idle":"2023-01-28T02:31:07.007083Z","shell.execute_reply.started":"2023-01-28T02:30:58.098801Z","shell.execute_reply":"2023-01-28T02:31:07.00628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(file_names[1].rsplit('/',2))","metadata":{"execution":{"iopub.status.busy":"2023-01-28T02:31:10.99315Z","iopub.execute_input":"2023-01-28T02:31:10.99388Z","iopub.status.idle":"2023-01-28T02:31:11.00181Z","shell.execute_reply.started":"2023-01-28T02:31:10.993843Z","shell.execute_reply":"2023-01-28T02:31:11.000541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient_id = []\nfor i in range(len(file_names)):\n    patient_id.append(file_names[i].rsplit('/',2)[1])\n    \npatient_id = list(set(patient_id))\nlen(patient_id) # stoped here","metadata":{"execution":{"iopub.status.busy":"2023-01-28T02:32:02.052656Z","iopub.execute_input":"2023-01-28T02:32:02.053092Z","iopub.status.idle":"2023-01-28T02:32:02.089812Z","shell.execute_reply.started":"2023-01-28T02:32:02.053054Z","shell.execute_reply":"2023-01-28T02:32:02.088619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_shape = []\n\nfor i in tqdm(range(len(file_names))):\n    dcm_file = pydicom.dcmread(file_names[i])\n    shape = dcm_file.pixel_array.shape\n    file_shape.append(shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-28T03:08:01.913506Z","iopub.execute_input":"2023-01-28T03:08:01.913895Z","iopub.status.idle":"2023-01-28T03:12:34.691855Z","shell.execute_reply.started":"2023-01-28T03:08:01.913864Z","shell.execute_reply":"2023-01-28T03:12:34.690403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-01-28T02:34:49.838926Z","iopub.execute_input":"2023-01-28T02:34:49.83934Z","iopub.status.idle":"2023-01-28T02:34:49.846129Z","shell.execute_reply.started":"2023-01-28T02:34:49.839303Z","shell.execute_reply":"2023-01-28T02:34:49.844925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_path = '/kaggle/working/'\n\nos.listdir(output_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n\n# # Define the directory path\n# directory = data_path\n\n# # Collect the names and file paths of all files in the directory\n# file_paths = []\n# file_names = []\n# for root, dirs, files in os.walk(directory):\n#     for file in files:\n#         file_path = os.path.join(root, file)\n#         file_paths.append(file_path)\n#         file_names.append(file)\n\n# # Print the results\n# print(\"File Names:\", file_names)\n# print(\"File Paths:\", file_paths)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Split the 'file_paths' column into several columns\n# new_cols = df['file_paths'].str.split('/', expand=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_cols = new_cols.iloc[:, [5, 6]]\n# new_cols.columns = ['patient_ID', 'image_ID']\n# new_cols.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df = df.join(new_cols)\n# df.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find the number of images for each patient in train_iamges\n\ndef count_files(directory):\n    data = []\n    for dirpath, dirnames, filenames in os.walk(directory):\n        data.append([dirpath, len(filenames)])\n    return data\n\ndirectory = train_data_path # train_images\ndata = count_files(directory)\n\ndf_file_counts = pd.DataFrame(data, columns=['Directory', 'Number_of_Files'])\nprint(df_file_counts)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_file_counts['Number_of_Files'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_file_counts = df_file_counts[df_file_counts['Number_of_Files'] >0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_file_counts['Number_of_Files'].value_counts() # remove the parent directory","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find out all sub directories name under train_images\n\nimport os\n\ndef get_subdirs(directory):\n    subdirs = []\n    for dirpath, dirnames, filenames in os.walk(directory):\n        for dirname in dirnames:\n            subdirs.append(dirname)\n    return subdirs\n\ndirectory = train_data_path\nsubdirs = get_subdirs(directory)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create DataFrame from the list\ndf_patient_train = pd.DataFrame(subdirs, columns=['patient_id'])\n\n# Add a new column named 'train_test' and fill it with 'train'\ndf_patient_train['train_test'] = 'train'\n\nprint(df_patient_train) # not used yet. ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import train.csv, try to label patient_id\ndf_train_csv = pd.read_csv(data_path + 'train.csv')\ndf_train_csv.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find out file name and file path\n\ndef find_files(directory):\n    files_collector = []\n    for root, dirs, files in os.walk(directory):\n        for file in files:\n            file_path = os.path.join(root, file)\n            files_collector.append([file, file_path])\n    return pd.DataFrame(files_collector, columns=['file_name', 'file_path'])\n\ndf_train_data_path = find_files(train_data_path)\ndf_train_data_path.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data_path.to_csv('/kaggle/input/df-file-shape-mergedcsv/df_train_data_path.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_data_path.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find out all images size in train_images. It takes more than 8 hours.\n\n\n# def find_files(directory):\n#     files_collector = []\n#     for root, dirs, files in tqdm(os.walk(directory)):\n#         for file in files:\n#             if file.endswith('.dcm'):\n#                 file_path = os.path.join(root, file)\n#                 dcm_file = pydicom.dcmread(file_path)\n#                 shape = dcm_file.pixel_array.shape\n#                 files_collector.append([file, file_path, shape])\n#     return pd.DataFrame(files_collector, columns=['file_name', 'file_path', 'file_shape'])\n\n# df_train_data = find_files(train_data_path)\n# df_train_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # find out image size, the first 20k images. saved into .csv files. \n\n# files_collector = []\n    \n# for i in tqdm(np.arange(26_000,30_000)):\n\n#     file_path = df_train_data_path['file_path'][i]\n#     dcm_file = pydicom.dcmread(file_path)\n#     shape = dcm_file.pixel_array.shape\n#     files_collector.append([df_train_data_path['file_name'][i], df_train_data_path['file_path'][i], shape])\n\n# df_file_shape_30 = pd.DataFrame(files_collector, columns=['file_name', 'file_path', 'file_shape'])\n# df_file_shape_30.to_csv('df_file_shape_30.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Try rename the .csv file\n# print('File_name_{}.csv'.format(datetime.now().strftime(\"%Y-%m-%dT%H%M%S\")))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in [26000,27000,28000,29000,30000]:\n    print('df_file_shape_{}.csv'.format(str(int(j/1000))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find out image size, saved into .csv files. \n\nfor j in tqdm([31000,32000,33000,34000,35000]):\n    \n    files_collector = []\n    \n    start_num = j\n    end_num = j+1000\n    for i in tqdm(np.arange(start_num,end_num)):\n\n        file_path = df_train_data_path['file_path'][i]\n        dcm_file = pydicom.dcmread(file_path)\n        shape = dcm_file.pixel_array.shape\n        files_collector.append([df_train_data_path['file_name'][i], df_train_data_path['file_path'][i], shape])\n\n    file_name = pd.DataFrame(files_collector, columns=['file_name', 'file_path', 'file_shape'])\n    \n    file_name.to_csv('file_name_{}.csv'.format(str(int(end_num/1000))))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for j in [26000,27000,28000,29000,30000]:\n    start_num = j\n    end_num = j+2000\n    \n    for i in np.arange(start_num, end_num):\n\n        print(i)\n        \n        \n    tmp_num = int(end_num/1000) \n    tmp_str = str(tmp_num)\n    print('df_file_shape_{}'.format(tmp_str))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_file_shape_merged['file_shape'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display one image\n\nds = dcmread(df_train_data_path['file_path'][0])\n# `arr` is a numpy.ndarray\narr = ds.pixel_array\n\nplt.imshow(arr, cmap=\"gray\")\nplt.show()\n\narr.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"look into the images size","metadata":{}},{"cell_type":"code","source":"df_images_size = pd.read_csv('/kaggle/input/all-images-shape/merged_df.csv')\ndf_iamges_size = df_images_size[['file_name','file_path','file_shape']]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_images_size['file_shape'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}