{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Notebook from a kaggle novice❗\n\nIf you like this and think it helpful, please give a like 👍. Many thanks ！🏄‍\n","metadata":{"execution":{"iopub.status.busy":"2022-07-29T07:11:03.457277Z"}}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt \nimport cv2 as cv\nfrom path import Path\nimport os \nimport glob\nimport tensorflow_hub as hub\nimport os \nimport pydicom as dicom\nfrom pydicom.pixel_data_handlers.util import apply_voi_lut\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras import layers\nfrom tqdm import tqdm\nfrom tensorflow.keras.preprocessing.image import load_img, img_to_array\nfrom tensorflow.keras.utils import to_categorical\nfrom pydicom import dcmread\nimport nibabel as nib\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.133204Z","iopub.execute_input":"2023-02-07T19:26:29.133569Z","iopub.status.idle":"2023-02-07T19:26:29.140516Z","shell.execute_reply.started":"2023-02-07T19:26:29.133539Z","shell.execute_reply":"2023-02-07T19:26:29.13936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/train.csv\")\n# test_df = pd.read_csv(\"../input/rsna-2022-cervical-spine-fracture-detection/test.csv\") useless ,sample not exist","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.142421Z","iopub.execute_input":"2023-02-07T19:26:29.142989Z","iopub.status.idle":"2023-02-07T19:26:29.166458Z","shell.execute_reply.started":"2023-02-07T19:26:29.142951Z","shell.execute_reply":"2023-02-07T19:26:29.165293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df #guess it's a multi-label classify","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.168866Z","iopub.execute_input":"2023-02-07T19:26:29.169243Z","iopub.status.idle":"2023-02-07T19:26:29.186641Z","shell.execute_reply.started":"2023-02-07T19:26:29.169207Z","shell.execute_reply":"2023-02-07T19:26:29.185794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_dicom(path):\n    img=dicom.dcmread(path)\n    data=img.pixel_array\n    data=data-np.min(data)\n    if np.max(data) != 0:\n        data=data/np.max(data)\n    data=(data*255).astype(np.uint8)\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.189109Z","iopub.execute_input":"2023-02-07T19:26:29.189444Z","iopub.status.idle":"2023-02-07T19:26:29.196308Z","shell.execute_reply.started":"2023-02-07T19:26:29.189418Z","shell.execute_reply":"2023-02-07T19:26:29.195127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def listdirs(folder):\n    return [d for d in os.listdir(folder) if os.path.isdir(os.path.join(folder, d))]","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.199127Z","iopub.execute_input":"2023-02-07T19:26:29.199483Z","iopub.status.idle":"2023-02-07T19:26:29.207012Z","shell.execute_reply.started":"2023-02-07T19:26:29.199449Z","shell.execute_reply":"2023-02-07T19:26:29.206037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir='../input/rsna-2022-cervical-spine-fracture-detection/train_images'\npatients = sorted(os.listdir(train_dir))\npatients[:5]","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.208206Z","iopub.execute_input":"2023-02-07T19:26:29.211692Z","iopub.status.idle":"2023-02-07T19:26:29.222403Z","shell.execute_reply.started":"2023-02-07T19:26:29.211664Z","shell.execute_reply":"2023-02-07T19:26:29.221406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_file = glob.glob(\"../input/rsna-2022-cervical-spine-fracture-detection/train_images/1.2.826.0.1.3680043.10001/*.dcm\")\nplt.figure(figsize=(20, 20))\n\nfor i in range(28):\n    ax = plt.subplot(7, 7, i + 1)\n    # specify your dcm image path\n    image_path = image_file[i]\n\n    image = load_dicom(image_path)\n\n    plt.axis('off')\n    \n    plt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:29.22409Z","iopub.execute_input":"2023-02-07T19:26:29.224509Z","iopub.status.idle":"2023-02-07T19:26:31.523348Z","shell.execute_reply.started":"2023-02-07T19:26:29.224475Z","shell.execute_reply":"2023-02-07T19:26:31.522511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nibabel as nib\n\nimage_file = glob.glob(\"../input/rsna-2022-cervical-spine-fracture-detection/segmentations/*.nii\")\nplt.figure(figsize=(20, 20))\n\nfor i in range(28):\n    ax = plt.subplot(7, 7, i + 1)\n    # specify your nii image path\n    image_path = image_file[i]\n    nii_img = nib.load(image_path).get_fdata()\n    nib_image = nii_img[:,:,59]\n    plt.axis('off')\n    plt.imshow(nib_image)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:31.524603Z","iopub.execute_input":"2023-02-07T19:26:31.52554Z","iopub.status.idle":"2023-02-07T19:26:38.529684Z","shell.execute_reply.started":"2023-02-07T19:26:31.525504Z","shell.execute_reply":"2023-02-07T19:26:38.528583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Note ❗🏃‍\nwhen use 'dicom.read_file' and apply **'.file_meta.TransferSyntaxUID.name'** you can find 3 types of data:\n\nImplicit VR Little Endian\n\nExplicit VR Little Endian\n\nJPEG Lossless, Non-Hierarchical, First-Order Prediction\n\nThe last one requires jpeg related tools, not yet know how to deal with it. Remove it now otherwise it will stuck the notebook.\n","metadata":{}},{"cell_type":"code","source":"from pydicom.data import get_testdata_files\ntrainset=[]\ntrainlabel=[]\ntrainidt=[]\nlimit = 2\nfor i in tqdm(range(len(train_df))): #there are 2019 rows, need much times, so just process 10 rows to test\n    idt=train_df.loc[i,'StudyInstanceUID']\n    \n#     idt2=('00000'+str(idt))[-5:]\n    path=os.path.join(train_dir,idt)   \n    \n    for im in os.listdir(path):\n        \n        \n        dc = dicom.read_file(os.path.join(path,im))\n        if dc.file_meta.TransferSyntaxUID.name =='JPEG Lossless, Non-Hierarchical, First-Order Prediction (Process 14 [Selection Value 1])':\n            continue\n        \n        img=load_dicom(os.path.join(path,im)) \n#         ds = decode(os.path.join(path,im))\n#         print(ds)\n#         ds.decompress(\"pylibjpeg\")\n\n        img=cv.resize(img,(64,64)) \n        image=img_to_array(img)\n        image=image/255.0\n        \n        saved_csv_name = str(im) + \".csv\"\n        np.savetxt(saved_csv_name, image[:,:,0], delimiter = \",\")\n        \n        trainset+=[image]\n        cur_label=[]\n        cur_label.append(train_df.loc[i,'C1'])\n        cur_label.append(train_df.loc[i,'C2'])\n        cur_label.append(train_df.loc[i,'C3'])\n        cur_label.append(train_df.loc[i,'C4'])\n        cur_label.append(train_df.loc[i,'C5'])\n        cur_label.append(train_df.loc[i,'C6'])\n        cur_label.append(train_df.loc[i,'C7'])\n        trainlabel+=[cur_label]\n        trainidt+=[idt]\n    i+=1\n    if i==limit +1:\n        break","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:38.531309Z","iopub.execute_input":"2023-02-07T19:26:38.531776Z","iopub.status.idle":"2023-02-07T19:26:55.684794Z","shell.execute_reply.started":"2023-02-07T19:26:38.531734Z","shell.execute_reply":"2023-02-07T19:26:55.683816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=np.array(trainlabel)\nY_train=y\nX_train=np.array(trainset)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:55.688549Z","iopub.execute_input":"2023-02-07T19:26:55.688816Z","iopub.status.idle":"2023-02-07T19:26:55.700368Z","shell.execute_reply.started":"2023-02-07T19:26:55.68879Z","shell.execute_reply":"2023-02-07T19:26:55.69944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntest_df = ['1.2.826.0.1.3680043.22327', '1.2.826.0.1.3680043.25399', '1.2.826.0.1.3680043.5876']","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:55.701607Z","iopub.execute_input":"2023-02-07T19:26:55.702485Z","iopub.status.idle":"2023-02-07T19:26:55.707534Z","shell.execute_reply.started":"2023-02-07T19:26:55.702448Z","shell.execute_reply":"2023-02-07T19:26:55.706374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dir='../input/rsna-2022-cervical-spine-fracture-detection/test_images'\ntestset=[]\ntestidt=[]\nfor i in tqdm(range(len(test_df))):\n    idt=test_df[i]\n    path=os.path.join(test_dir,idt)   \n    \n    for im in os.listdir(path):\n        dc = dicom.read_file(os.path.join(path,im))\n        \n        if dc.file_meta.TransferSyntaxUID.name =='JPEG Lossless, Non-Hierarchical, First-Order Prediction (Process 14 [Selection Value 1])':\n            continue\n        img=load_dicom(os.path.join(path,im)) \n\n        img=cv.resize(img,(64,64)) \n        image=img_to_array(img)\n        image=image/255.0\n        testset+=[image]\n        testidt+=[idt]","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:26:55.709037Z","iopub.execute_input":"2023-02-07T19:26:55.709608Z","iopub.status.idle":"2023-02-07T19:27:15.650112Z","shell.execute_reply.started":"2023-02-07T19:26:55.709567Z","shell.execute_reply":"2023-02-07T19:27:15.64913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!zip -r 111file.zip /kaggle/working  #zipping all the files will create a single file which you can download","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:15.65151Z","iopub.execute_input":"2023-02-07T19:27:15.653413Z","iopub.status.idle":"2023-02-07T19:27:17.513338Z","shell.execute_reply.started":"2023-02-07T19:27:15.653375Z","shell.execute_reply":"2023-02-07T19:27:17.512185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = np.array(testset)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:17.515Z","iopub.execute_input":"2023-02-07T19:27:17.515319Z","iopub.status.idle":"2023-02-07T19:27:17.528581Z","shell.execute_reply.started":"2023-02-07T19:27:17.515287Z","shell.execute_reply":"2023-02-07T19:27:17.527603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.Sequential()\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),input_shape=(64,64,1),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.Dropout(0.20))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Conv2D(filters=64,kernel_size=(4,4),activation='relu',kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(keras.layers.Dropout(0.25))\nmodel.add(keras.layers.BatchNormalization())\nmodel.add(keras.layers.Flatten())\nmodel.add(keras.layers.Dense(100,activation=\"relu\",kernel_initializer=\"he_normal\"))\nmodel.add(keras.layers.Dense(7,\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:17.530388Z","iopub.execute_input":"2023-02-07T19:27:17.530756Z","iopub.status.idle":"2023-02-07T19:27:20.409539Z","shell.execute_reply.started":"2023-02-07T19:27:17.530721Z","shell.execute_reply":"2023-02-07T19:27:20.408515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=True, rankdir='TB')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:20.410891Z","iopub.execute_input":"2023-02-07T19:27:20.411751Z","iopub.status.idle":"2023-02-07T19:27:21.428279Z","shell.execute_reply.started":"2023-02-07T19:27:20.411712Z","shell.execute_reply":"2023-02-07T19:27:21.427139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss=\"categorical_crossentropy\",\n              optimizer = \"RMSprop\",metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:21.430049Z","iopub.execute_input":"2023-02-07T19:27:21.431114Z","iopub.status.idle":"2023-02-07T19:27:21.448292Z","shell.execute_reply.started":"2023-02-07T19:27:21.431071Z","shell.execute_reply":"2023-02-07T19:27:21.447349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"callback = keras.callbacks.EarlyStopping(monitor='loss', patience=10)","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:21.449646Z","iopub.execute_input":"2023-02-07T19:27:21.450002Z","iopub.status.idle":"2023-02-07T19:27:21.456195Z","shell.execute_reply.started":"2023-02-07T19:27:21.449967Z","shell.execute_reply":"2023-02-07T19:27:21.455201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = model.fit(X_train, Y_train,epochs=50, batch_size=64, verbose=1,callbacks=[callback])","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:21.458Z","iopub.execute_input":"2023-02-07T19:27:21.458433Z","iopub.status.idle":"2023-02-07T19:27:33.611116Z","shell.execute_reply.started":"2023-02-07T19:27:21.458396Z","shell.execute_reply":"2023-02-07T19:27:33.610005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=model.predict(X_test)\nvalues = np.max(y_pred, axis = 1)\npred=np.argmax(y_pred,axis=1)\npred[0:10]","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:33.612531Z","iopub.execute_input":"2023-02-07T19:27:33.612995Z","iopub.status.idle":"2023-02-07T19:27:43.227105Z","shell.execute_reply.started":"2023-02-07T19:27:33.612957Z","shell.execute_reply":"2023-02-07T19:27:43.226183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename = 'finalized_model.sav'\npickle.dump(model, open(filename, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:43.228661Z","iopub.execute_input":"2023-02-07T19:27:43.229032Z","iopub.status.idle":"2023-02-07T19:27:43.262578Z","shell.execute_reply.started":"2023-02-07T19:27:43.228995Z","shell.execute_reply":"2023-02-07T19:27:43.261126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# values = np.zeros((3,7))\n# lens = np.zeros((3, 1))\n# for i in tqdm(range(len(y_pred))):\n#     if testidt[i] == test_df[0]:\n#         values[0]+=y_pred[i]\n#         lens[0]+=1\n#     elif testidt[i] == test_df[1]:\n#         values[1]+=y_pred[i]\n#         lens[1]+=1\n#     else:\n#         values[2]+=y_pred[i]\n#         lens[2]+=1","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:43.263639Z","iopub.status.idle":"2023-02-07T19:27:43.264785Z","shell.execute_reply.started":"2023-02-07T19:27:43.264536Z","shell.execute_reply":"2023-02-07T19:27:43.26456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result=pd.DataFrame(testidt)\nresult[1]=np.array([np.float64(round(i, 1)) for i in values])\nfor i in range(len(result)):\n    \n    result.iloc[i, 0] = str(result.iloc[i, 0])+'_C'+str(pred[i] + 1)\n\n\n# result.columns=['row_id','fractured']\n# print(result)\n# submission = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/sample_submission.csv')\n# submission['fractured'] = np.array([np.float64(round(i, 1)) for i in values[0:3]])\n# submission['row_id'] = np.array([str(i)+'_C1' for i in test_df])\n    \n# submission","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:43.265965Z","iopub.status.idle":"2023-02-07T19:27:43.266637Z","shell.execute_reply.started":"2023-02-07T19:27:43.266381Z","shell.execute_reply":"2023-02-07T19:27:43.266406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission.to_csv('submission.csv', index = 0)\ntest_df = pd.read_csv('../input/rsna-2022-cervical-spine-fracture-detection/test.csv')\nmeans = train_df.mean(numeric_only=True).to_dict()\ntest_df['fractured'] = test_df['prediction_type'].map(means)\ntest_df[['row_id','fractured']].to_csv('submission.csv', index=False, float_format='%.1g')\n","metadata":{"execution":{"iopub.status.busy":"2023-02-07T19:27:43.269506Z","iopub.status.idle":"2023-02-07T19:27:43.270003Z","shell.execute_reply.started":"2023-02-07T19:27:43.269735Z","shell.execute_reply":"2023-02-07T19:27:43.269759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Actual Work 🤔❓\n\nHow to deal with jpeg image?\n\nHow to make submission correcct?\n\nHow to use the segmentation to help predict?","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}