{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"code","source":"!pip install -qU \"python-gdcm\" pydicom pylibjpeg \"opencv-python-headless\"","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:44:39.329753Z","iopub.execute_input":"2022-08-20T13:44:39.330145Z","iopub.status.idle":"2022-08-20T13:44:52.868478Z","shell.execute_reply.started":"2022-08-20T13:44:39.330116Z","shell.execute_reply":"2022-08-20T13:44:52.867386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport sys\nimport cv2\nimport glob\nimport random\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport nibabel as nib\nimport matplotlib.pyplot as plt\n\nfrom tqdm import tqdm\nfrom matplotlib import cm\nfrom PIL import Image as Img\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import train_test_split\nfrom IPython.display import display_html, display, Image\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:03.27555Z","iopub.execute_input":"2022-08-20T13:45:03.276348Z","iopub.status.idle":"2022-08-20T13:45:03.283359Z","shell.execute_reply.started":"2022-08-20T13:45:03.276316Z","shell.execute_reply":"2022-08-20T13:45:03.282687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !export LD_LIBRARY_PATH=/usr/local/cuda-11.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:04.258017Z","iopub.execute_input":"2022-08-20T13:45:04.258803Z","iopub.status.idle":"2022-08-20T13:45:04.263383Z","shell.execute_reply.started":"2022-08-20T13:45:04.258765Z","shell.execute_reply":"2022-08-20T13:45:04.262428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    root_dir = \"../input/rsna-2022-cervical-spine-fracture-detection/\"\n    train_dir = root_dir + \"train_images/\"\n    num_samples = 1000\n    num_slices = 50\n    slice_size = (128, 128)\n    seed = 42 ","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:05.24018Z","iopub.execute_input":"2022-08-20T13:45:05.240851Z","iopub.status.idle":"2022-08-20T13:45:05.245704Z","shell.execute_reply.started":"2022-08-20T13:45:05.240809Z","shell.execute_reply":"2022-08-20T13:45:05.244758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed=42):\n    \"\"\"\n    sets the integer starting value used in generating random numbers.\n    comment out packages/frameworks not in use as per your choice \n    \"\"\"\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:05.423312Z","iopub.execute_input":"2022-08-20T13:45:05.4239Z","iopub.status.idle":"2022-08-20T13:45:05.428537Z","shell.execute_reply.started":"2022-08-20T13:45:05.42387Z","shell.execute_reply":"2022-08-20T13:45:05.427684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"set_seed(CFG.seed)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:05.929878Z","iopub.execute_input":"2022-08-20T13:45:05.930544Z","iopub.status.idle":"2022-08-20T13:45:05.93446Z","shell.execute_reply.started":"2022-08-20T13:45:05.93051Z","shell.execute_reply":"2022-08-20T13:45:05.93362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(CFG.root_dir + \"train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:06.173869Z","iopub.execute_input":"2022-08-20T13:45:06.174406Z","iopub.status.idle":"2022-08-20T13:45:06.195504Z","shell.execute_reply.started":"2022-08-20T13:45:06.174374Z","shell.execute_reply":"2022-08-20T13:45:06.194513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:06.770495Z","iopub.execute_input":"2022-08-20T13:45:06.771085Z","iopub.status.idle":"2022-08-20T13:45:06.778349Z","shell.execute_reply.started":"2022-08-20T13:45:06.771051Z","shell.execute_reply":"2022-08-20T13:45:06.777526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3D slices\n`X` : 3D input sequential scans per StudyInstanceID  (max `slice_num` can be 69 since minimum number of slices out of all studies for real XD )\n\n`y` : one hot encoded vector representing `C1-C7`","metadata":{}},{"cell_type":"code","source":"%time \nnum_samples = CFG.num_samples\nnum_slices = CFG.num_slices\nX = np.zeros((num_samples, CFG.slice_size[0], CFG.slice_size[0], num_slices))\ny = np.zeros((num_samples, 7))\n\n\nfor i in tqdm(range(num_samples)):\n    study_id = train_df.loc[i, \"StudyInstanceUID\"]\n\n    # Get .dcm paths\n    study_scans_paths = glob.glob(os.path.join(CFG.train_dir, study_id, \"*.dcm\"))\n    # sorting w.r.t slice number \n    study_scans_paths = sorted(study_scans_paths, key=lambda x : int(x.split('/')[-1].split('.')[0])) \n    study_scans_paths = study_scans_paths[:num_slices]\n    \n    # Get corresponging datasets and images\n    datasets = [pydicom.dcmread(path) for path in study_scans_paths]\n    images = [apply_voi_lut(dataset.pixel_array, dataset) for dataset in datasets]\n    images = [cv2.resize(image, CFG.slice_size) for image in images]\n    images = np.moveaxis(np.array(images), 0, -1) # (69,256,256) => (256,256,69)\n\n    \n    scaler = MinMaxScaler(feature_range=(0,1))\n    images = scaler.fit_transform(images.reshape(-1, images.shape[-1])).reshape(images.shape) # reshaping to 2D then back to 3D for MinMax Scaler\n\n    X[i] = np.asarray(images)\n    y[i] = np.asarray(train_df.iloc[i, -7:].values.astype('int'))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T13:45:09.552707Z","iopub.execute_input":"2022-08-20T13:45:09.553845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_name = f\"X_num_samples_{num_samples}_num_slices_{num_slices}.npy\"\ny_name = f\"y_num_samples_{num_samples}_num_slices_{num_slices}.npy\"","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:50:07.184388Z","iopub.execute_input":"2022-08-19T10:50:07.184873Z","iopub.status.idle":"2022-08-19T10:50:07.190112Z","shell.execute_reply.started":"2022-08-19T10:50:07.184839Z","shell.execute_reply":"2022-08-19T10:50:07.188856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save(X_name, X)\nnp.save(y_name, y)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:50:10.265297Z","iopub.execute_input":"2022-08-19T10:50:10.265642Z","iopub.status.idle":"2022-08-19T10:50:52.039166Z","shell.execute_reply.started":"2022-08-19T10:50:10.265612Z","shell.execute_reply":"2022-08-19T10:50:52.037871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X, y","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:51:25.471397Z","iopub.execute_input":"2022-08-19T10:51:25.471861Z","iopub.status.idle":"2022-08-19T10:51:26.118636Z","shell.execute_reply.started":"2022-08-19T10:51:25.471828Z","shell.execute_reply":"2022-08-19T10:51:26.117542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.load(X_name)\ny = np.load(y_name)","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:53:34.638585Z","iopub.execute_input":"2022-08-19T10:53:34.639392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[0,:,:,0], cmap='bone');","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:53:21.101349Z","iopub.execute_input":"2022-08-19T10:53:21.101727Z","iopub.status.idle":"2022-08-19T10:53:21.356302Z","shell.execute_reply.started":"2022-08-19T10:53:21.101693Z","shell.execute_reply":"2022-08-19T10:53:21.355356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-19T10:53:26.70215Z","iopub.execute_input":"2022-08-19T10:53:26.702503Z","iopub.status.idle":"2022-08-19T10:53:26.719679Z","shell.execute_reply.started":"2022-08-19T10:53:26.702473Z","shell.execute_reply":"2022-08-19T10:53:26.718489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[69,:,:,0], cmap='bone');","metadata":{"execution":{"iopub.status.busy":"2022-08-19T07:43:15.805395Z","iopub.status.idle":"2022-08-19T07:43:15.805869Z","shell.execute_reply.started":"2022-08-19T07:43:15.805629Z","shell.execute_reply":"2022-08-19T07:43:15.805652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y[2]","metadata":{"execution":{"iopub.status.busy":"2022-08-19T07:43:15.807558Z","iopub.status.idle":"2022-08-19T07:43:15.808058Z","shell.execute_reply.started":"2022-08-19T07:43:15.807786Z","shell.execute_reply":"2022-08-19T07:43:15.807809Z"},"trusted":true},"execution_count":null,"outputs":[]}]}