{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook uses pydicom-conda-helper dataset https://www.kaggle.com/code/awsaf49/pydicom-conda-helper to set up the libraries to read dcm files. Also, data and code for extracting vertibrae by SAMUEL CORTINHAS: https://www.kaggle.com/code/samuelcortinhas/extracting-vertebrae-c1-c7/notebook","metadata":{}},{"cell_type":"code","source":"!conda install '/kaggle/input/pydicom-conda-helper/certifi-2020.12.5-py37h89c1867_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/conda-4.10.1-py37h89c1867_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/gdcm-2.8.9-py37h500ead1_1.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libgcc-ng-9.3.0-h2828fa1_19.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/libjpeg-turbo-2.1.0-h7f98852_0.tar.bz2' -c conda-forge -y\n!conda install '/kaggle/input/pydicom-conda-helper/openssl-1.1.1k-h7f98852_0.tar.bz2' -c conda-forge -y","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:48:56.623294Z","iopub.execute_input":"2022-11-27T12:48:56.624131Z","iopub.status.idle":"2022-11-27T12:49:57.262275Z","shell.execute_reply.started":"2022-11-27T12:48:56.623637Z","shell.execute_reply":"2022-11-27T12:49:57.260943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n\ndef read_xray(path, voi_lut = True, fix_monochrome = True):\n    dicom = pydicom.read_file(path)\n    \n    # VOI LUT (if available by DICOM device) is used to transform raw DICOM data to \"human-friendly\" view\n    if voi_lut:\n        data = apply_voi_lut(dicom.pixel_array, dicom)\n    else:\n        data = dicom.pixel_array\n               \n    # depending on this value, X-ray may look inverted - fix that:\n    if fix_monochrome and dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        data = np.amax(data) - data\n    \n    data = data - np.min(data)\n    data = data / np.max(data)\n    data = (data * 255).astype(np.uint8)\n        \n    return data","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:49:57.266351Z","iopub.execute_input":"2022-11-27T12:49:57.26673Z","iopub.status.idle":"2022-11-27T12:49:57.432658Z","shell.execute_reply.started":"2022-11-27T12:49:57.26669Z","shell.execute_reply":"2022-11-27T12:49:57.431579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\npd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\", index_col = \"StudyInstanceUID\")","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:49:57.433903Z","iopub.execute_input":"2022-11-27T12:49:57.434255Z","iopub.status.idle":"2022-11-27T12:49:57.474793Z","shell.execute_reply.started":"2022-11-27T12:49:57.434219Z","shell.execute_reply":"2022-11-27T12:49:57.473224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n# From https://www.kaggle.com/code/andradaolteanu/rsna-fracture-detection-dicom-images-explore\ndef get_observation_data(path):\n    '''\n    Get information from the .dcm files\n    '''\n\n    dataset = pydicom.read_file(path)\n    \n    # Dictionary to store the information from the image\n    observation_data = {\n        \"Rows\" : dataset.get(\"Rows\"),\n        \"Columns\" : dataset.get(\"Columns\"),\n        \"SOPInstanceUID\" : dataset.get(\"SOPInstanceUID\"),\n        \"ContentDate\" : dataset.get(\"ContentDate\"),\n        \"SliceThickness\" : dataset.get(\"SliceThickness\"),\n        \"InstanceNumber\" : dataset.get(\"InstanceNumber\"),\n        \"ImagePositionPatient\" : dataset.get(\"ImagePositionPatient\"),\n        \"ImageOrientationPatient\" : dataset.get(\"ImageOrientationPatient\"),\n    }\n\n    # String columns\n    str_columns = [\"SOPInstanceUID\", \"ContentDate\", \n                   \"SliceThickness\", \"InstanceNumber\"]\n    for k in str_columns:\n        observation_data[k] = str(dataset.get(k)) if k in dataset else None\n\n    return observation_data\n\ndef get_metadata():\n    '''\n    Retrieves the desired metadata from the .dcm files and saves it into dataframe.\n    '''\n    \n    exceptions = 0\n    dicts = []\n    \n    test_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\")\n\n    for dir_ in os.listdir('../input/rsna-2022-cervical-spine-fracture-detection/test_images'):\n\n        # Get all .dcm paths for this Instance\n        dcm_paths = glob(f\"{base_path}/test_images/{dir_}/*\")\n        print(dcm_paths)\n\n        for path in dcm_paths:\n            try:\n                # Get datasets\n                dataset = get_observation_data(path)\n                dicts.append(dataset)\n            except Exception as e:\n                exceptions += 1\n                continue\n\n    # Convert into df\n    meta_train_data = pd.DataFrame(data=dicts, columns=md_example.keys())\n    \n    # Export information\n    meta_train_data.to_csv(\"meta_test.csv\", index=False)\n    \n    print(f\"Metadata created. Number of total fails: {exceptions}.\")","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:49:57.477818Z","iopub.execute_input":"2022-11-27T12:49:57.478165Z","iopub.status.idle":"2022-11-27T12:49:57.489033Z","shell.execute_reply.started":"2022-11-27T12:49:57.478129Z","shell.execute_reply":"2022-11-27T12:49:57.488082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#md_example = get_observation_data(ex_path)\n#print(md_example)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:49:57.490431Z","iopub.execute_input":"2022-11-27T12:49:57.490905Z","iopub.status.idle":"2022-11-27T12:49:57.500501Z","shell.execute_reply.started":"2022-11-27T12:49:57.490856Z","shell.execute_reply":"2022-11-27T12:49:57.499498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport matplotlib.patches as patches\nimport seaborn as sns\nsns.set(style='darkgrid', font_scale=1.6)\nimport cv2\nimport os\nfrom os import listdir\nimport re\nimport gc\nimport sys\nimport pydicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nfrom tqdm import tqdm\nfrom pprint import pprint\nfrom time import time\nimport itertools\n# from skimage import measure \nfrom mpl_toolkits.mplot3d.art3d import Poly3DCollection\nimport nibabel as nib\nfrom glob import glob\n\nfrom sklearn.model_selection import train_test_split, StratifiedGroupKFold, GroupKFold\nfrom sklearn.metrics import confusion_matrix, accuracy_score\nfrom tensorflow import keras\nfrom keras import layers\nfrom keras import callbacks\n\n# Models\nfrom sklearn.linear_model import LinearRegression, LogisticRegression\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom xgboost import XGBClassifier\nfrom lightgbm import LGBMClassifier\nfrom catboost import CatBoostClassifier\nfrom sklearn.naive_bayes import GaussianNB\n\n# Load dataframes\ntrain_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\ntrain_bbox = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train_bounding_boxes.csv\")\ntest_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\")\nss = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/sample_submission.csv\")\n\n# Store segmentation paths in a dataframe\nbase_path = \"../input/rsna-2022-cervical-spine-fracture-detection\"\nseg_paths = glob(f\"{base_path}/segmentations/*\")\nseg_df = pd.DataFrame({'path': seg_paths})\nseg_df['StudyInstanceUID'] = seg_df['path'].apply(lambda x:x.split('/')[-1][:-4])\nseg_df = seg_df[['StudyInstanceUID','path']]\n\n# Metadata was extracted previously (check out my RSNA dataset)\n#make csv\n#############################~~~~~~~~#####################################################################\nmeta_train = pd.read_csv(\"../input/rsna-2022-spine-fracture-detection-metadata/meta_train_clean.csv\")\n\n# Only select patients with segmentations\nmeta_seg = meta_train[meta_train['StudyInstanceUID'].isin(seg_df['StudyInstanceUID'])].reset_index(drop=True)\nprint('meta_seg shape:', meta_seg.shape)\nmeta_seg.head(3)\n\n# Example\nex_path = \"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/1.2.826.0.1.3680043.12281.nii\"\nexample = nib.load(ex_path)\nexample = example.get_fdata()  # convert to numpy array\nexample = example[:, ::-1, ::-1].transpose(2, 1, 0)  # align orientation with train image\nnp.unique(example[119])\n\n# plt.figure()\n# plt.imshow(example[119])\n# plt.title('Segmentation example')\n# plt.colorbar()\n# plt.axis('off')\n# plt.show()\n\n\n# Initialise targets\ntargets = ['C1','C2','C3','C4','C5','C6','C7']\nmeta_seg[targets]=0\n\n\nmeta_seg = pd.read_csv('../input/rsna-2022-spine-fracture-detection-metadata/meta_segmentation.csv')\nmeta_seg.head(3)\n\n\n# Example\nmeta_seg[['StudyInstanceUID','Slice']+targets].iloc[199:204,:]\n\n# Print example of extracted vertebrae\nprint('UID:', meta_seg['StudyInstanceUID'].unique()[0])\npd.set_option('display.max_rows', 500)\nmeta_seg[meta_seg['StudyInstanceUID']==meta_seg['StudyInstanceUID'].unique()[0]].loc[110:340,targets]\n\n# Calculate slice ratio (to generalise better)\nslice_max_seg = meta_seg.groupby('StudyInstanceUID')['Slice'].max().to_dict()\nmeta_seg['SliceRatio'] = 0\nmeta_seg['SliceRatio'] = meta_seg['Slice']/meta_seg['StudyInstanceUID'].map(slice_max_seg)\n\nfeatures = ['SliceRatio','SliceThickness','ImagePositionPatient_x','ImagePositionPatient_y','ImagePositionPatient_z']\n\n# Features and targets\nX = meta_seg[['StudyInstanceUID']+features]\ny = meta_seg[targets]\n\n# Train-valid split, grouped by patient (80/20 split)\ngkf = GroupKFold(n_splits=5)\n(train_idx, valid_idx) = next(gkf.split(X, y, groups = X['StudyInstanceUID']))\n\n# Train set\nX_train, y_train = X.iloc[train_idx,:], y.iloc[train_idx,:]\n\n# Validation set\nX_valid, y_valid = X.iloc[valid_idx,:], y.iloc[valid_idx,:]\n\n# Drop patient id\nX_train = X_train.drop('StudyInstanceUID', axis=1)\nX_valid = X_valid.drop('StudyInstanceUID', axis=1)\n\n# Train classifier\nclf = RandomForestClassifier()\nclf.fit(X_train, y_train)\n\n\n# Evaluate model\ny_preds = clf.predict(X_valid)\n\ntotal_acc = 0 \nfor i in range(7):\n    acc = (y_valid[f'C{i+1}']==y_preds[:,i]).sum()/len(y_preds[:,i])\n    total_acc+=acc/7\n    print(f'Accuracy of C{i+1}: {acc} %')\n\nprint('')\nprint(f'Overall accuracy: {total_acc} %')\n\n# Feature importances\npd.DataFrame({'Feature':features, 'Importance':clf.feature_importances_}).sort_values(by='Importance', ascending=False)\n\npreds = clf.predict(X_valid)\n\n# Confusion matrices\n# fig = plt.figure(figsize=(20,10))\n# for i in range(7):\n#     cm = confusion_matrix(preds[:,i], y_valid.values[:,i])\n#     plt.subplot(2,4,i+1)\n#     CBAR=False\n#     if (i==3) or (i==6):\n#         CBAR=True\n#     sns.heatmap(cm, annot=True, fmt='d', cbar=CBAR, cmap='Blues')\n#     plt.xlabel('Predicted label')\n#     plt.ylabel('True label')\n#     plt.title(f'C{i+1}')\n# fig.tight_layout()\n\nnp.set_printoptions(threshold=np.inf)\nclf.predict(X.drop('StudyInstanceUID',axis=1))[110:250,:]\n\n# Read in metadata for entire train set\nmeta_train = pd.read_csv('../input/rsna-2022-spine-fracture-detection-metadata/meta_train_clean.csv')\n\n# Calculate slice ratio (to generalise better)\nslice_max_train = meta_train.groupby('StudyInstanceUID')['Slice'].max().to_dict()\nmeta_train['SliceRatio'] = 0\nmeta_train['SliceRatio'] = meta_train['Slice']/meta_train['StudyInstanceUID'].map(slice_max_train)\n\n# Initialise targets\nmeta_train[targets]=0\n\nprint(\"TRAIN\", meta_train[features])\n\n# Predict targets for entire train set\nmeta_train[targets] = clf.predict(meta_train[features])\n\n# We know images with segmentations have 100% accurate targets so put these back in\nmeta_train.loc[meta_train['StudyInstanceUID'].isin(meta_seg['StudyInstanceUID']),targets] = meta_seg[targets].values\n\n# Save to csv\nmeta_train.to_csv('meta_train_with_vertebrae.csv', index=False)\n\n# Preview\nmeta_train.head(3)\n\n# fig = plt.figure(figsize=(20,16))\n# for i, Cx in enumerate(targets):\n#     plt.subplot(4,2,i+1)\n#     sns.histplot(meta_train.groupby('StudyInstanceUID')[Cx].sum())\n#     plt.title(f'Number of slices: {Cx}')\n# fig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:49:57.502253Z","iopub.execute_input":"2022-11-27T12:49:57.502626Z","iopub.status.idle":"2022-11-27T12:51:08.393752Z","shell.execute_reply.started":"2022-11-27T12:49:57.50259Z","shell.execute_reply":"2022-11-27T12:51:08.392582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"import pydicom\n#import pylibjpeg\n#import gdcm\nimport nibabel as nib\nimport matplotlib.pyplot as plt\nimport os\nfrom tqdm import tqdm\nimport pandas as pd\nimport cv2\nds = read_xray('../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.20981/200.dcm')\n#print(pydicom.dcmread('../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.20981/1.dcm'))\nplt.imshow(ds, cmap=plt.cm.bone)\n#print(ds)\ndata_x = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\", index_col = \"StudyInstanceUID\")\nim_list = []\nlabel_list = []\nvert_list = []\ni = 0\nl_ = list(tqdm(os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/train_images\", topdown=False)))\nfor _, dirs, _ in l_:\n    for dir_ in dirs:\n        #if not os.path.isfile(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\"): continue\n        if i%100 == 0: print(i)\n        i += 1\n#        if i==6:break\n        #segmentations = nib.load(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\").get_fdata()[:, ::-1, ::-1].transpose(2, 1, 0)\n        for _, _, files in os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/train_images/\" + dir_, topdown=False):\n            #print(len(files), len(segmentations))\n            start_per_vert = [0 for _ in range(7)]\n            finish_per_vert = [0 for _ in range(7)]\n            first_visit = [True for _ in range(7)]\n            first_disappearance = [False for _ in range(7)]\n            img_vert = meta_train[meta_train[\"StudyInstanceUID\"] == dir_][[\"C1\", \"C2\", \"C3\", \"C4\", \"C5\", \"C6\", \"C7\"]].to_numpy()\n            for ii in range(len(files)):\n                for vert in range(7):\n                    if (vert+1) in set(np.where(img_vert[ii] == 1)[0] + 1):\n                        if first_visit[vert]: \n                            first_visit[vert] = False\n                            start_per_vert[vert] = ii\n                    else:\n                        if not first_visit[vert] and not first_disappearance[vert]:\n                            first_disappearance[vert] = True\n                            finish_per_vert[vert] = ii\n                        \n            for start, finish in zip(start_per_vert, finish_per_vert):\n                for ii in range(2,5):\n                    if set(np.where(img_vert[int(start + ii*(- start + finish)/5)] == 1)[0] + 1) & set(range(1,8)) and files[int(start + ii*(- start + finish)/5)] not in [\"1.dcm\",\"10.dcm\"]:\n                        im_list.append(cv2.resize(read_xray(\"../input/rsna-2022-cervical-spine-fracture-detection/train_images/\" + dir_ + \"/\" + files[int(start + ii*(- start + finish)/5)])[512//4:3*512//4, 512//4:3*512//4], (512//4, 512//4)))\n                        label_list.append(max([data_x.loc[dir_][\"C\" + str(int(vert))] for vert in set(np.where(img_vert[int(start + ii*(- start + finish)/5)] == 1)[0] + 1) & set(range(1,8))]))\n                        vert_list.append(img_vert[int(start + ii*(- start + finish)/5)])\n                                   \n#-----------#########----\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:08.395501Z","iopub.execute_input":"2022-11-27T12:51:08.396152Z","iopub.status.idle":"2022-11-27T12:51:08.405892Z","shell.execute_reply.started":"2022-11-27T12:51:08.396111Z","shell.execute_reply":"2022-11-27T12:51:08.404801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#l_ = None\n#print(len(im_list), len(label_list))\n#492","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:08.407422Z","iopub.execute_input":"2022-11-27T12:51:08.408856Z","iopub.status.idle":"2022-11-27T12:51:08.41972Z","shell.execute_reply.started":"2022-11-27T12:51:08.408817Z","shell.execute_reply":"2022-11-27T12:51:08.418698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\n'''filehandler = open(b\"im_list_.pkl\",\"wb\")\npickle.dump(im_list, filehandler)\nfilehandler = open(b\"label_list.pkl\",\"wb\")\npickle.dump(label_list, filehandler)\nfilehandler = open(b\"vert_list.pkl\",\"wb\")\npickle.dump(vert_list, filehandler)'''\nwith open('../input/rnsadata/im_list_.pkl', 'rb') as f:\n    im_list = pickle.load(f)\nwith open('../input/rnsadata/label_list.pkl', 'rb') as f:\n    label_list = pickle.load(f)\nwith open('../input/rnsadata/vert_list.pkl', 'rb') as f:\n    vert_list = pickle.load(f)\n","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:08.421217Z","iopub.execute_input":"2022-11-27T12:51:08.421578Z","iopub.status.idle":"2022-11-27T12:51:15.134259Z","shell.execute_reply.started":"2022-11-27T12:51:08.421543Z","shell.execute_reply":"2022-11-27T12:51:15.133213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"import os\nos.chdir(r'/kaggle/working')\n\nfrom IPython.display import FileLink\n\nFileLink(r'im_list_.pkl')\"\"\"\nim_list = im_list[:int(len(im_list))]\nlabel_list = label_list[:int(len(label_list))]\nvert_list = vert_list[:int(len(vert_list))]\n\n#im_list = [im_list[i] for i in range(len(im_list)) if vert_list[i][1] == 1]\n#label_list = [label_list[i] for i in range(len(label_list)) if vert_list[i][1] == 1]\n#vert_list = [vert_list[i] for i in range(len(vert_list)) if vert_list[i][1] == 1]","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.139501Z","iopub.execute_input":"2022-11-27T12:51:15.140486Z","iopub.status.idle":"2022-11-27T12:51:15.148321Z","shell.execute_reply.started":"2022-11-27T12:51:15.140453Z","shell.execute_reply":"2022-11-27T12:51:15.14719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(sum(label_list))\nprint(len(label_list))","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.150016Z","iopub.execute_input":"2022-11-27T12:51:15.150352Z","iopub.status.idle":"2022-11-27T12:51:15.166512Z","shell.execute_reply.started":"2022-11-27T12:51:15.150325Z","shell.execute_reply":"2022-11-27T12:51:15.165365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_list_1 = [elem for elem in label_list if elem == 1]\nim_list_1 = [im_list[i] for i in range(len(im_list)) if label_list[i] == 1]\nvert_list_1 = [vert_list[i] for i in range(len(vert_list)) if label_list[i] == 1]\n\nim_list.extend(im_list_1)\nim_list.extend(im_list_1)\nlabel_list.extend(label_list_1)\nlabel_list.extend(label_list_1)\nvert_list.extend(vert_list_1)\nvert_list.extend(vert_list_1)\nX = []\nY = []\nV = []\ncount = 0\nprint(sum(label_list)/len(label_list))\nfor i, elem in enumerate(label_list):\n    if elem == 1 or(elem == 0 and count < 20000):\n        X.append(im_list[i])\n        Y.append(elem)\n        V.append(vert_list[i])\n        if elem == 0:\n            count +=1\nsum(Y)/len(Y)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.167948Z","iopub.execute_input":"2022-11-27T12:51:15.16869Z","iopub.status.idle":"2022-11-27T12:51:15.252791Z","shell.execute_reply.started":"2022-11-27T12:51:15.168635Z","shell.execute_reply":"2022-11-27T12:51:15.251836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n#allakse to na krataei thn seira\nX_test, X_train, y_test, y_train = X[:int(0.1*len(X))], X[int(0.1*len(X)):], Y[:int(0.1*len(Y))], Y[int(0.1*len(Y)):]","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.25415Z","iopub.execute_input":"2022-11-27T12:51:15.254768Z","iopub.status.idle":"2022-11-27T12:51:15.26111Z","shell.execute_reply.started":"2022-11-27T12:51:15.254731Z","shell.execute_reply":"2022-11-27T12:51:15.259975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"import nibabel as nib\nsegmentations = nib.load(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/1.2.826.0.1.3680043.10921.nii\").get_fdata()[:, ::-1, ::-1].transpose(2, 1, 0)\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.262622Z","iopub.execute_input":"2022-11-27T12:51:15.263108Z","iopub.status.idle":"2022-11-27T12:51:15.274444Z","shell.execute_reply.started":"2022-11-27T12:51:15.263073Z","shell.execute_reply":"2022-11-27T12:51:15.273563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"import matplotlib.pyplot as plt\nfor segment in segmentations:\n    plt.figure()\n    plt.imshow(segment)\n    print(np.unique(segment))\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.276021Z","iopub.execute_input":"2022-11-27T12:51:15.276445Z","iopub.status.idle":"2022-11-27T12:51:15.28617Z","shell.execute_reply.started":"2022-11-27T12:51:15.276406Z","shell.execute_reply":"2022-11-27T12:51:15.285086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del model\nimport tensorflow as tf\n\ndef vgg_block(num_convs, num_channels):\n    blk = tf.keras.models.Sequential()\n    for _ in range(num_convs):\n        blk.add(tf.keras.layers.Conv2D(num_channels,kernel_size=3,\n                                    padding='same',activation='relu'))\n    blk.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2))\n    return blk","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.287919Z","iopub.execute_input":"2022-11-27T12:51:15.288253Z","iopub.status.idle":"2022-11-27T12:51:15.297045Z","shell.execute_reply.started":"2022-11-27T12:51:15.288219Z","shell.execute_reply":"2022-11-27T12:51:15.296108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_arch = ((1, 32), (1, 64), (1, 128), (1, 256), (2, 256))","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.298516Z","iopub.execute_input":"2022-11-27T12:51:15.298869Z","iopub.status.idle":"2022-11-27T12:51:15.311241Z","shell.execute_reply.started":"2022-11-27T12:51:15.298835Z","shell.execute_reply":"2022-11-27T12:51:15.310282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def vgg(conv_arch):\n    net = tf.keras.models.Sequential()\n    # The convulational part\n    for (num_convs, num_channels) in conv_arch:\n        net.add(vgg_block(num_convs, num_channels))\n    # The fully-connected part\n    net.add(tf.keras.models.Sequential([\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(4096, activation='relu'),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(4096, activation='relu'),\n        tf.keras.layers.Dropout(0.5),\n        tf.keras.layers.Dense(1, activation='sigmoid')]))\n    return net\n\nnet = vgg(conv_arch)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:15.31422Z","iopub.execute_input":"2022-11-27T12:51:15.314534Z","iopub.status.idle":"2022-11-27T12:51:19.432262Z","shell.execute_reply.started":"2022-11-27T12:51:15.314509Z","shell.execute_reply":"2022-11-27T12:51:19.43125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nnp.array(X_train).shape","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:19.433607Z","iopub.execute_input":"2022-11-27T12:51:19.434627Z","iopub.status.idle":"2022-11-27T12:51:19.633826Z","shell.execute_reply.started":"2022-11-27T12:51:19.434578Z","shell.execute_reply":"2022-11-27T12:51:19.632696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\nLOSS = tf.keras.losses.BinaryCrossentropy(\n    label_smoothing=0.0,\n    axis=-1,\n    name='binary_crossentropy'\n)\n\nOPTIMIZER = tf.keras.optimizers.Adam(\n    learning_rate=0.0001,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=1e-05,\n    amsgrad=False,\n    name='Adam')\nEPOCHS = 40\n\nmodel = vgg(conv_arch)\n\nmodel.compile(optimizer=OPTIMIZER,\n              loss= LOSS,\n              metrics=['accuracy'])\n\nhistory = model.fit(np.array(X_train).reshape(-1, 128, 128, 1)/255, np.array(y_train), epochs=EPOCHS, batch_size=BATCH_SIZE, verbose = 1,\n                    validation_data=(np.array(X_test).reshape(-1, 128, 128, 1)/255, np.array(y_test)))","metadata":{"execution":{"iopub.status.busy":"2022-11-27T12:51:19.635987Z","iopub.execute_input":"2022-11-27T12:51:19.636752Z","iopub.status.idle":"2022-11-27T13:04:10.581056Z","shell.execute_reply.started":"2022-11-27T12:51:19.636711Z","shell.execute_reply":"2022-11-27T13:04:10.580062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#regression στο τελος\n\n\n#briskoume elaxisto arithmo eikonwn ana spondylo\n#taizoume sto cnn eikones pou exoun auto to bathos\n\n#h\n\n#na taizoume mia mia tis eikones kai na bgazoume mia katanomh pithanothtas\n#gia kathe spondylo","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:10.582569Z","iopub.execute_input":"2022-11-27T13:04:10.582955Z","iopub.status.idle":"2022-11-27T13:04:10.588153Z","shell.execute_reply.started":"2022-11-27T13:04:10.58292Z","shell.execute_reply":"2022-11-27T13:04:10.587099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tail (vl,iml):\n    vert_arr = np.array(vl)\n    prediction = model.predict(\n        np.array(iml).reshape(-1, 128, 128, 1)/255,\n        batch_size=None,\n        verbose='auto',\n        steps=None,\n        callbacks=None,\n        max_queue_size=10,\n        workers=1,\n        use_multiprocessing=False\n    )\n    print(\"vert\", vert_arr)\n    print(\"pred\", prediction)\n    res_splitted = prediction * vert_arr[:len(prediction)]\n    print(\"res_spl\", res_splitted)\n    denom = vert_arr.sum(axis = 0)\n    res = res_splitted.sum(axis = 0)/denom\n    res[np.isnan(res)] = 0\n    return res , 1-np.prod(np.sort(1 - res)[:2])","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:20:30.315792Z","iopub.execute_input":"2022-11-27T13:20:30.316159Z","iopub.status.idle":"2022-11-27T13:20:30.323603Z","shell.execute_reply.started":"2022-11-27T13:20:30.316128Z","shell.execute_reply":"2022-11-27T13:20:30.322492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"from tqdm import tqdm\nmeta_test = pd.read_csv(\"./meta_test.csv\")\nim_list_test = []\nlabel_list_test = []\nvert_list_test = []\ni = 0\nl_ = list(tqdm(os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images\", topdown=False)))\nfor _, dirs, _ in l_:\n    for dir_ in dirs:\n        #if not os.path.isfile(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\"): continue\n        if i%100 == 0: print(i)\n        i += 1\n#        if i==6:break\n        #segmentations = nib.load(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\").get_fdata()[:, ::-1, ::-1].transpose(2, 1, 0)\n        for _, _, files in os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images/\" + dir_, topdown=False):\n            #print(len(files), len(segmentations))\n            start_per_vert = [0 for _ in range(7)]\n            finish_per_vert = [0 for _ in range(7)]\n            first_visit = [True for _ in range(7)]\n            first_disappearance = [False for _ in range(7)]\n            img_vert = meta_test[meta_test[\"SOPInstanceUID\"] == dir_][[\"C1\", \"C2\", \"C3\", \"C4\", \"C5\", \"C6\", \"C7\"]].to_numpy()\n            for ii in range(len(files)):\n                for vert in range(7):\n                    if (vert+1) in set(np.where(img_vert[ii] == 1)[0] + 1):\n                        if first_visit[vert]: \n                            first_visit[vert] = False\n                            start_per_vert[vert] = ii\n                    else:\n                        if not first_visit[vert] and not first_disappearance[vert]:\n                            first_disappearance[vert] = True\n                            finish_per_vert[vert] = ii\n                        \n            for start, finish in zip(start_per_vert, finish_per_vert):\n                for ii in range(2,5):\n                    if set(np.where(img_vert[int(start + ii*(- start + finish)/5)] == 1)[0] + 1) & set(range(1,8)) and files[int(start + ii*(- start + finish)/5)] not in [\"1.dcm\",\"10.dcm\"]:\n                        im_list_test.append(cv2.resize(read_xray(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images/\" + dir_ + \"/\" + files[int(start + ii*(- start + finish)/5)])[512//4:3*512//4, 512//4:3*512//4], (512//4, 512//4)))\n                        label_list_test.append(max([data_x.loc[dir_][\"C\" + str(int(vert))] for vert in set(np.where(img_vert[int(start + ii*(- start + finish)/5)] == 1)[0] + 1) & set(range(1,8))]))\n                        vert_list_test.append(img_vert[int(start + ii*(- start + finish)/5)])\n                                   \n#-----------#########----\"\"\"","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:10.604581Z","iopub.execute_input":"2022-11-27T13:04:10.6055Z","iopub.status.idle":"2022-11-27T13:04:10.621771Z","shell.execute_reply.started":"2022-11-27T13:04:10.605442Z","shell.execute_reply":"2022-11-27T13:04:10.620826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nimport os\nex_path = \"../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.10001/104.dcm\"\nmd_example = get_observation_data(ex_path)\nprint(md_example)\nget_metadata()","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:10.623155Z","iopub.execute_input":"2022-11-27T13:04:10.623413Z","iopub.status.idle":"2022-11-27T13:04:24.36244Z","shell.execute_reply.started":"2022-11-27T13:04:10.623389Z","shell.execute_reply":"2022-11-27T13:04:24.361379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_test_images_verts(clf):\n    meta_test = pd.read_csv(\"./meta_test.csv\")\n    meta_seg = meta_test.copy()\n    targets = ['C1','C2','C3','C4','C5','C6','C7']\n    print(meta_seg)\n    # Calculate slice ratio (to generalise better)\n    print(meta_test)\n    meta_seg[['StudyInstanceUID','Slice']] = meta_seg[\"SOPInstanceUID\"].apply(lambda x : pd.Series([\".\".join(x.split('.')[:-2]), int(x.split('.')[-1])]))\n    meta_seg[['ImagePositionPatient_x','ImagePositionPatient_y','ImagePositionPatient_z']] = meta_seg[\"ImagePositionPatient\"].apply(lambda x : pd.Series(x.strip(\"[]\").split(',')))\n\n    # Calculate slice ratio (to generalise better)\n    slice_max_seg = meta_seg.groupby('StudyInstanceUID')['Slice'].max().to_dict()\n    meta_seg['SliceRatio'] = 0\n    meta_seg['SliceRatio'] = meta_seg['Slice']/meta_seg['StudyInstanceUID'].map(slice_max_seg)\n\n\n    features = ['SliceRatio','SliceThickness','ImagePositionPatient_x','ImagePositionPatient_y','ImagePositionPatient_z']\n\n    # Features and targets\n    X = meta_seg[['StudyInstanceUID']+features]\n\n    # Drop patient id\n    X = X.drop('StudyInstanceUID', axis=1)\n    \n    print(X[features])\n\n    # Evaluate model\n    y_preds = clf.predict(X[features])\n\n    np.set_printoptions(threshold=np.inf)\n    \n    #print(y_preds)\n\n    # Read in metadata for entire train set\n    meta_train = pd.read_csv('./meta_test.csv')\n    #meta_train = meta_train.rename(columns={'SOPInstanceUID': 'StudyInstanceUID'})\n    meta_train[['StudyInstanceUID','Slice']] = meta_train[\"SOPInstanceUID\"].apply(lambda x : pd.Series([\".\".join(x.split('.')[:-2]), int(x.split('.')[-1])]))\n    meta_train[['ImagePositionPatient_x','ImagePositionPatient_y','ImagePositionPatient_z']] = meta_train[\"ImagePositionPatient\"].apply(lambda x : pd.Series(x.strip(\"[]\").split(',')))\n\n    # Calculate slice ratio (to generalise better)\n    slice_max_seg = meta_train.groupby('StudyInstanceUID')['Slice'].max().to_dict()\n    meta_train['SliceRatio'] = 0\n    meta_train['SliceRatio'] = meta_train['Slice']/meta_train['StudyInstanceUID'].map(slice_max_seg)\n    # Initialise targets\n    meta_train[targets]=0\n\n    # Predict targets for entire train set\n    meta_train[targets] = clf.predict(meta_train[features])\n\n    # Save to csv\n    meta_train.to_csv('meta_test_with_vertebrae.csv', index=False)\n\n    # Preview\n    meta_train.head(3)\n    \n    return meta_train\n\n","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:24.364765Z","iopub.execute_input":"2022-11-27T13:04:24.365449Z","iopub.status.idle":"2022-11-27T13:04:24.378094Z","shell.execute_reply.started":"2022-11-27T13:04:24.365406Z","shell.execute_reply":"2022-11-27T13:04:24.37697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#meta_train.index = meta_train[\"StudyInstanceUID\"] + \"_\" + meta_train[\"Slice\"].apply(str)\nmeta_test = get_test_images_verts(clf)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:24.379556Z","iopub.execute_input":"2022-11-27T13:04:24.380051Z","iopub.status.idle":"2022-11-27T13:04:25.863435Z","shell.execute_reply.started":"2022-11-27T13:04:24.380014Z","shell.execute_reply":"2022-11-27T13:04:25.862423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sum(meta_test['C1'])\nmeta_test = meta_test.sort_values([\"StudyInstanceUID\", \"Slice\"])","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:25.864876Z","iopub.execute_input":"2022-11-27T13:04:25.86535Z","iopub.status.idle":"2022-11-27T13:04:25.877309Z","shell.execute_reply.started":"2022-11-27T13:04:25.86531Z","shell.execute_reply":"2022-11-27T13:04:25.876395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"meta_test","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:25.882689Z","iopub.execute_input":"2022-11-27T13:04:25.882995Z","iopub.status.idle":"2022-11-27T13:04:25.9113Z","shell.execute_reply.started":"2022-11-27T13:04:25.882969Z","shell.execute_reply":"2022-11-27T13:04:25.9104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nim_list_test = []\nvert_list_test = []\nverts_per_person = []\nfolder_per_person = []\ni\ni = 0\nl_ = list(tqdm(os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images\", topdown=False)))\nfor _, dirs, _ in l_:\n    for dir_ in dirs:\n        verts_per_person.append([0])\n        folder_per_person.append(dir_)\n        #if not os.path.isfile(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\"): continue\n        if i%100 == 0: print(i)\n        i += 1\n#        if i==6:break\n        #segmentations = nib.load(f\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/\" + dir_ + \".nii\").get_fdata()[:, ::-1, ::-1].transpose(2, 1, 0)\n        for _, _, files in os.walk(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images/\" + dir_, topdown=False):\n            #print(len(files), len(segmentations))\n            start_per_vert = [0 for _ in range(7)]\n            finish_per_vert = [0 for _ in range(7)]\n            first_visit = [True for _ in range(7)]\n            first_disappearance = [False for _ in range(7)]\n            img_vert = meta_test[meta_test[\"StudyInstanceUID\"] == dir_][[\"C1\", \"C2\", \"C3\", \"C4\", \"C5\", \"C6\", \"C7\"]].to_numpy()\n            for ii in range(len(files)):\n                for vert in range(7):\n                    if (vert+1) in set(np.where(img_vert[ii] == 1)[0] + 1):\n                        if first_visit[vert]: \n                            first_visit[vert] = False\n                            start_per_vert[vert] = ii\n                    else:\n                        if not first_visit[vert] and not first_disappearance[vert]:\n                            first_disappearance[vert] = True\n                            finish_per_vert[vert] = ii\n                        \n            for start, finish in zip(start_per_vert, finish_per_vert):\n                for ii in range(2,5):\n                    if set(np.where(img_vert[int(start + ii*(- start + finish)/5)] == 1)[0] + 1) & set(range(1,8)) and files[int(start + ii*(- start + finish)/5)] not in [\"1.dcm\",\"10.dcm\"]:\n                        verts_per_person[-1][0] += 1\n                        im_list_test.append(cv2.resize(read_xray(\"../input/rsna-2022-cervical-spine-fracture-detection/test_images/\" + dir_ + \"/\" + files[int(start + ii*(- start + finish)/5)])[512//4:3*512//4, 512//4:3*512//4], (512//4, 512//4)))\n                        vert_list_test.append(img_vert[int(start + ii*(- start + finish)/5)])\n                                   \n#-----------#########----","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:04:25.912762Z","iopub.execute_input":"2022-11-27T13:04:25.913312Z","iopub.status.idle":"2022-11-27T13:04:26.714518Z","shell.execute_reply.started":"2022-11-27T13:04:25.913278Z","shell.execute_reply":"2022-11-27T13:04:26.71355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count = 0\ndf = pd.DataFrame(columns=[\"row_id\", \"fractured\"])\nfor p_n, verts in enumerate(verts_per_person):\n    curr_vert = []\n    curr_im_list = []\n    for i in range(verts[0]):\n        curr_vert.append(vert_list_test[count+i])\n        curr_im_list.append(im_list_test[count+i])\n    print(curr_vert)\n    res, pat_ovr = tail(curr_vert,curr_im_list)\n    print(res)\n    for ii in range(1, 8):\n        df = df.append({\"row_id\": folder_per_person[p_n] + f\"_C{ii}\", \"fractured\": res[ii-1]}, ignore_index=True)\n    df = df.append({\"row_id\": folder_per_person[p_n] + f\"_patient_overall\", \"fractured\": pat_ovr}, ignore_index=True)\n    count += verts[0]","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:20:37.089986Z","iopub.execute_input":"2022-11-27T13:20:37.090351Z","iopub.status.idle":"2022-11-27T13:20:37.272534Z","shell.execute_reply.started":"2022-11-27T13:20:37.090318Z","shell.execute_reply":"2022-11-27T13:20:37.269289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-27T13:11:20.785464Z","iopub.execute_input":"2022-11-27T13:11:20.786462Z","iopub.status.idle":"2022-11-27T13:11:20.79363Z","shell.execute_reply.started":"2022-11-27T13:11:20.786421Z","shell.execute_reply":"2022-11-27T13:11:20.792695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}