{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Create IDwise TF Records\n\n## references:\n\n## [How To Create TFRecords](https://www.kaggle.com/cdeotte/how-to-create-tfrecords)\n\n## [How to properly split folds](https://www.kaggle.com/code/underwearfitting/how-to-properly-split-folds/notebook)\n\nand [This disccusion](https://www.kaggle.com/competitions/rsna-2022-cervical-spine-fracture-detection/discussion/341412) helps to open anomally files","metadata":{}},{"cell_type":"markdown","source":"## My Other Works\n### [Create IDwise TF Records](https://www.kaggle.com/code/itsuki9180/create-idwise-tf-records) <- Here\n### [RSNA 2022 TF-Keras Traing Phase With TPU](https://www.kaggle.com/code/itsuki9180/rsna-2022-tf-keras-traing-phase-with-tpu) \n### [RSNA 2022 TF-Keras Inference Phase With GPU](https://www.kaggle.com/itsuki9180/rsna-2022-tf-keras-inference-phase-with-gpu) ","metadata":{}},{"cell_type":"markdown","source":"## Import and Easy EDA","metadata":{}},{"cell_type":"code","source":"!conda install '../input/rsna2022-pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '../input/rsna2022-pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '../input/rsna2022-pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '../input/rsna2022-pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '../input/rsna2022-pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '../input/rsna2022-pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:41:16.230226Z","iopub.execute_input":"2022-08-20T12:41:16.231179Z","iopub.status.idle":"2022-08-20T12:42:21.572628Z","shell.execute_reply.started":"2022-08-20T12:41:16.23106Z","shell.execute_reply":"2022-08-20T12:42:21.571513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD LIBRARIES\nimport numpy as np, pandas as pd, os\nimport matplotlib.pyplot as plt, cv2\nimport tensorflow as tf, re, math\nimport glob\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.utils import check_random_state\nimport nibabel as nib\nimport pydicom as dicom\nfrom collections import Counter, defaultdict\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:21.576171Z","iopub.execute_input":"2022-08-20T12:42:21.576591Z","iopub.status.idle":"2022-08-20T12:42:27.420386Z","shell.execute_reply.started":"2022-08-20T12:42:21.576549Z","shell.execute_reply":"2022-08-20T12:42:27.419183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOLDS = 5\nIMGS_PER_CHANNEL = 16 # must be square number\nN_CHANNEL = 3\nIMG_SIZE = int(512 // (IMGS_PER_CHANNEL**0.5)) \nSEED = 42","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.422267Z","iopub.execute_input":"2022-08-20T12:42:27.422901Z","iopub.status.idle":"2022-08-20T12:42:27.428582Z","shell.execute_reply.started":"2022-08-20T12:42:27.422867Z","shell.execute_reply":"2022-08-20T12:42:27.427464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\ndef tileImage(imgs):\n d = int(math.sqrt(IMGS_PER_CHANNEL))\n h = imgs[0].shape[0]\n w = imgs[0].shape[1]\n r = np.zeros((h*d,w*d,1),dtype=np.uint8)\n for idx,img in enumerate(imgs):\n idx_y = int(idx/d)\n idx_x = idx-idx_y*d\n r[idx_y*h:(idx_y+1)*h,idx_x*w:(idx_x+1)*w,:] = img\n return r","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.431542Z","iopub.execute_input":"2022-08-20T12:42:27.432298Z","iopub.status.idle":"2022-08-20T12:42:27.442314Z","shell.execute_reply.started":"2022-08-20T12:42:27.432246Z","shell.execute_reply":"2022-08-20T12:42:27.440821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE = '../input/rsna-2022-cervical-spine-fracture-detection'","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.443916Z","iopub.execute_input":"2022-08-20T12:42:27.445074Z","iopub.status.idle":"2022-08-20T12:42:27.454764Z","shell.execute_reply.started":"2022-08-20T12:42:27.445025Z","shell.execute_reply":"2022-08-20T12:42:27.453252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\ntest = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\")\nsamplesub = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.456371Z","iopub.execute_input":"2022-08-20T12:42:27.456985Z","iopub.status.idle":"2022-08-20T12:42:27.492861Z","shell.execute_reply.started":"2022-08-20T12:42:27.456932Z","shell.execute_reply":"2022-08-20T12:42:27.491789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.494458Z","iopub.execute_input":"2022-08-20T12:42:27.495744Z","iopub.status.idle":"2022-08-20T12:42:27.508149Z","shell.execute_reply.started":"2022-08-20T12:42:27.495692Z","shell.execute_reply":"2022-08-20T12:42:27.506766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.509937Z","iopub.execute_input":"2022-08-20T12:42:27.510946Z","iopub.status.idle":"2022-08-20T12:42:27.531729Z","shell.execute_reply.started":"2022-08-20T12:42:27.510906Z","shell.execute_reply":"2022-08-20T12:42:27.53072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.533137Z","iopub.execute_input":"2022-08-20T12:42:27.533561Z","iopub.status.idle":"2022-08-20T12:42:27.545052Z","shell.execute_reply.started":"2022-08-20T12:42:27.533524Z","shell.execute_reply":"2022-08-20T12:42:27.543553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samplesub.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.549064Z","iopub.execute_input":"2022-08-20T12:42:27.550023Z","iopub.status.idle":"2022-08-20T12:42:27.562795Z","shell.execute_reply.started":"2022-08-20T12:42:27.54998Z","shell.execute_reply":"2022-08-20T12:42:27.56144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path, size = IMG_SIZE):\n try:\n img=dicom.dcmread(path)\n img.PhotometricInterpretation = 'YBR_FULL'\n data=img.pixel_array\n data=data-np.min(data)\n if np.max(data) != 0:\n data=data/np.max(data)\n data=(data*255).astype(np.uint8)\n data = cv2.resize(data, (IMG_SIZE, IMG_SIZE))\n data = np.expand_dims(data, -1)\n return data\n except: \n return np.zeros((IMG_SIZE, IMG_SIZE, 1))\n\ndef listdirs(folder):\n return [d for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))] \n\ntrain_dir = f'{BASE}/train_images'\ntest_dir = f'{BASE}/test_images'\npatients = sorted(os.listdir(train_dir))\nlen(patients), patients[:5]","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.564581Z","iopub.execute_input":"2022-08-20T12:42:27.565773Z","iopub.status.idle":"2022-08-20T12:42:27.678453Z","shell.execute_reply.started":"2022-08-20T12:42:27.565731Z","shell.execute_reply":"2022-08-20T12:42:27.67722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## split Multilabeled-Stratificated-KFold","metadata":{}},{"cell_type":"code","source":"class RepeatedStratifiedGroupKFold():\n\n def __init__(self, n_splits=5, n_repeats=1, random_state=None):\n self.n_splits = n_splits\n self.n_repeats = n_repeats\n self.random_state = random_state\n \n def split(self, X, y=None, groups=None):\n k = self.n_splits\n def eval_y_counts_per_fold(y_counts, fold):\n y_counts_per_fold[fold] += y_counts\n std_per_label = []\n for label in range(labels_num):\n label_std = np.std(\n [y_counts_per_fold[i][label] / y_distr[label] for i in range(k)]\n )\n std_per_label.append(label_std)\n y_counts_per_fold[fold] -= y_counts\n return np.mean(std_per_label)\n \n rnd = check_random_state(self.random_state)\n for repeat in range(self.n_repeats):\n labels_num = np.max(y) + 1\n y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))\n y_distr = Counter()\n for label, g in zip(y, groups):\n y_counts_per_group[g][label] += 1\n y_distr[label] += 1\n\n y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))\n groups_per_fold = defaultdict(set)\n \n groups_and_y_counts = list(y_counts_per_group.items())\n rnd.shuffle(groups_and_y_counts)\n\n for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])):\n best_fold = None\n min_eval = None\n for i in range(k):\n fold_eval = eval_y_counts_per_fold(y_counts, i)\n if min_eval is None or fold_eval < min_eval:\n min_eval = fold_eval\n best_fold = i\n y_counts_per_fold[best_fold] += y_counts\n groups_per_fold[best_fold].add(g)\n \n all_groups = set(groups)\n for i in range(k):\n train_groups = all_groups - groups_per_fold[i]\n test_groups = groups_per_fold[i]\n\n train_indices = [i for i, g in enumerate(groups) if g in train_groups]\n test_indices = [i for i, g in enumerate(groups) if g in test_groups]\n\n yield train_indices, test_indices","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.680834Z","iopub.execute_input":"2022-08-20T12:42:27.681793Z","iopub.status.idle":"2022-08-20T12:42:27.698217Z","shell.execute_reply.started":"2022-08-20T12:42:27.681743Z","shell.execute_reply":"2022-08-20T12:42:27.697094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's first concat all the labels \n# e.g 00000000010 \ntarget_cols = train.drop(['StudyInstanceUID'],axis=1).columns.values.tolist()\ntargets = train[target_cols].astype(str)\n# create a new col to store the label\ntrain['combined_tar'] = ''\nfor i in tqdm(range(targets.shape[1])):\n train['combined_tar'] += targets.iloc[:,i]\n# take a look at it\ntrain.combined_tar.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.700214Z","iopub.execute_input":"2022-08-20T12:42:27.700719Z","iopub.status.idle":"2022-08-20T12:42:27.753389Z","shell.execute_reply.started":"2022-08-20T12:42:27.700672Z","shell.execute_reply":"2022-08-20T12:42:27.752099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['combined_tar'] = LabelEncoder().fit_transform(train['combined_tar'])\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.755055Z","iopub.execute_input":"2022-08-20T12:42:27.756296Z","iopub.status.idle":"2022-08-20T12:42:27.77679Z","shell.execute_reply.started":"2022-08-20T12:42:27.756245Z","shell.execute_reply":"2022-08-20T12:42:27.775508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['fold'] = -1\nrskf = RepeatedStratifiedGroupKFold(n_splits=FOLDS, random_state=SEED)\nfor i, (train_idx, valid_idx) in enumerate(rskf.split(train, train.combined_tar, train.StudyInstanceUID)): #(df, targets, group)\n train.loc[valid_idx, 'fold'] = int(i)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:27.778212Z","iopub.execute_input":"2022-08-20T12:42:27.778615Z","iopub.status.idle":"2022-08-20T12:42:47.96717Z","shell.execute_reply.started":"2022-08-20T12:42:27.77858Z","shell.execute_reply":"2022-08-20T12:42:47.96583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.query('fold==0').combined_tar.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:47.968874Z","iopub.execute_input":"2022-08-20T12:42:47.969238Z","iopub.status.idle":"2022-08-20T12:42:47.984107Z","shell.execute_reply.started":"2022-08-20T12:42:47.969204Z","shell.execute_reply":"2022-08-20T12:42:47.983165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('combined_tar', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:47.98553Z","iopub.execute_input":"2022-08-20T12:42:47.986069Z","iopub.status.idle":"2022-08-20T12:42:48.011059Z","shell.execute_reply.started":"2022-08-20T12:42:47.986034Z","shell.execute_reply":"2022-08-20T12:42:48.009937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('combined_tar', axis=1).to_csv('train_folds.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:48.012323Z","iopub.execute_input":"2022-08-20T12:42:48.01308Z","iopub.status.idle":"2022-08-20T12:42:48.028926Z","shell.execute_reply.started":"2022-08-20T12:42:48.013045Z","shell.execute_reply":"2022-08-20T12:42:48.027629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## create TF-Records","metadata":{}},{"cell_type":"code","source":"def _bytes_feature(value):\n \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n if isinstance(value, type(tf.constant(0))):\n value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n \"\"\"Returns a float_list from a float / double.\"\"\"\n return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _int64_feature(value):\n \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:48.030271Z","iopub.execute_input":"2022-08-20T12:42:48.030957Z","iopub.status.idle":"2022-08-20T12:42:48.040765Z","shell.execute_reply.started":"2022-08-20T12:42:48.03092Z","shell.execute_reply":"2022-08-20T12:42:48.03911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def serialize_example(img, pati_oa, c1, c2, c3, c4, c5, c6, c7):\n feature = {\n 'image': _bytes_feature(img),\n 'patient_overall': _int64_feature(pati_oa),\n 'C1': _int64_feature(c1),\n 'C2': _int64_feature(c2),\n 'C3': _int64_feature(c3),\n 'C4': _int64_feature(c4),\n 'C5': _int64_feature(c5),\n 'C6': _int64_feature(c6),\n 'C7': _int64_feature(c7),\n }\n example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n return example_proto.SerializeToString()","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:48.0426Z","iopub.execute_input":"2022-08-20T12:42:48.043854Z","iopub.status.idle":"2022-08-20T12:42:48.057057Z","shell.execute_reply.started":"2022-08-20T12:42:48.043802Z","shell.execute_reply":"2022-08-20T12:42:48.05609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"folds = train","metadata":{"execution":{"iopub.status.busy":"2022-08-20T12:42:48.058543Z","iopub.execute_input":"2022-08-20T12:42:48.059455Z","iopub.status.idle":"2022-08-20T12:42:48.069327Z","shell.execute_reply.started":"2022-08-20T12:42:48.059377Z","shell.execute_reply":"2022-08-20T12:42:48.068255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for f in range(FOLDS):\n ct = (folds['fold'] == f).sum()\n idx = folds[folds['fold'] == f].index\n print(idx)\n print(ct)\n print('Writing TFRecord %i of %i...'%(f,ct))\n flg = 0\n flg2 = 1\n with tf.io.TFRecordWriter('train%.2i-%i.tfrec'%(f,ct)) as writer:\n for k in tqdm(range(ct)):\n imgs = glob.glob(BASE+'/train_images/'+folds['StudyInstanceUID'][idx[k]]+'/*')\n n_imgs = len(imgs)\n stride = n_imgs // (N_CHANNEL * IMGS_PER_CHANNEL)\n if k==0 : print(n_imgs, stride, n_imgs//stride)\n for j in range(1, stride, 1):\n pimgs = range(j, n_imgs, stride)\n img_arr_b = []\n img_arr_g = []\n img_arr_r = []\n for impath in pimgs: \n img = load_dicom(BASE+'/train_images/'+folds['StudyInstanceUID'][idx[k]]+f'/{impath}.dcm') \n \n if len(img_arr_b)