{"cells":[{"metadata":{},"cell_type":"markdown","source":"# To use GDCM for JPEG in DCM\nSometimes this does not work. If a figure is seen, that means GDCM is working."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y\n!pip install pylibjpeg pylibjpeg-libjpeg\n\nimport pydicom\nimport matplotlib.pyplot as plt\n\nimg = pydicom.read_file('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/8bc04c1a5852/b3f09354bb04/ecdfcd5104b6.dcm')\nplt.imshow(img.pixel_array)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert DCM to RGB 224x224 npy"},{"metadata":{"trusted":true},"cell_type":"code","source":"from PIL import Image\nimport numpy as np\n\nimg = pydicom.read_file('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/8bc04c1a5852/b3f09354bb04/ecdfcd5104b6.dcm')\n\narr = img.pixel_array.astype(np.float32)\n\narr -= arr.min()\narr *= 1./arr.max()\narr *= 255\narr = arr.astype(np.uint8)\n\nimage = Image.merge(\"RGB\", (Image.fromarray(arr[0::2,0::2]), Image.fromarray(arr[0::2,1::2]), Image.fromarray(arr[1::2,1::2]))).resize((224, 224))\n\narr = np.array(image)\n\nplt.imshow(arr)\nplt.show()\n\nnp.save(\"arr.npy\", arr)\n!ls -l arr.npy","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup of TensorFlow and Keras"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# https://qiita.com/sasayabaku/items/9e376ba8e38efe3bcf79\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D\nfrom tensorflow.keras.layers import MaxPool2D\nfrom tensorflow.keras.optimizers import Adam\n\nfrom tensorflow.keras.layers import Dense, Activation, Dropout, Flatten\nfrom tensorflow.keras.utils import plot_model, to_categorical\nfrom keras.callbacks import TensorBoard","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define CNN model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(512,1,input_shape=(512,512,1)))\nmodel.add(Activation('relu'))\n\nmodel.add(Conv2D(512,1))\nmodel.add(Activation('relu'))\n\nmodel.add(MaxPool2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(64,1))\nmodel.add(Activation('relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\n\nmodel.add(Flatten())\nmodel.add(Dense(1024//300))\nmodel.add(Activation('relu'))\nmodel.add(Dropout(0.99))\n\nmodel.add(Dense(1, activation='sigmoid'))\n\nadam = Adam(lr=1e-4)\n\nmodel.compile(optimizer=adam, loss='binary_crossentropy', metrics=[\"accuracy\"])\n\nmodel.summary()\n\nplot_model(model, to_file='./model.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load recorded model"},{"metadata":{"trusted":true},"cell_type":"code","source":"LABEL=4\nfrom keras.models import model_from_json\nmodel_arc_str = open('./cnn_model.'+str(LABEL)+'.json').read()\nmodel = model_from_json(model_arc_str)\nmodel.load_weights('./cnn_model_weight.'+str(LABEL)+'.hdf5')\nmodel.compile(optimizer=adam, loss='binary_crossentropy', metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Utility functions for train_on_batch"},{"metadata":{"trusted":true},"cell_type":"code","source":"import csv\nimport pydicom\n# import matplotlib.pyplot as plt\nimport numpy as np\n\ndef storeFileNames(label=4, interval=100, maxSize=2000000, offset=0):\n    PRINT_INT=10000\n    global xf_train, y_train\n    with open('/kaggle/input/rsna-str-pulmonary-embolism-detection/train.csv') as f:\n        reader = csv.reader(f)\n\n        count = 0\n        for index, row in enumerate(reader):\n            #print(row[label])\n            #if count%PRINT_INT==0 and (index==0 or index>interval):\n            #    print(index, \":\", count, \"/\", maxSize)\n            if index==0:\n                continue\n            if (index+offset)%interval!=0:\n                continue\n            if count>=maxSize:\n                return\n            count += 1\n            y_train = np.append(y_train, float(row[label]))\n            #xf_train.append('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/'+row[0]+'/'+row[1]+'/'+row[2]+'.dcm')\n            xf_train.append(row[0]+'/'+row[1]+'/'+row[2])\n\ndef getPixels(fileName):\n    #print(\"AAA\"+fileName)\n    img = pydicom.read_file('/kaggle/input/rsna-str-pulmonary-embolism-detection/train/'+fileName+'.dcm')\n    img = img.pixel_array.astype(np.float32)\n    img -= img.min()\n    img /= img.max()\n    #print(fileName, len(img))\n    return img\n                            \ndef getBatch(batch_size):\n    # https://aotamasaki.hatenablog.com/entry/2018/08/27/124349\n\n    global xf_train, y_train\n    SIZE = len(xf_train)\n\n    n_batches = SIZE//batch_size\n    #print(SIZE, \"SIZE\")\n    #print(batch_size, \"batch_size\")\n    #print(n_batches, \"n_batches\")\n\n    i = 0\n    while (i < n_batches):\n        #print(\"doing\", i, \"/\", n_batches)\n        y_batch = y_train[(i * batch_size):(i * batch_size + batch_size)]\n        \n        x_batch_name = xf_train[(i * batch_size):(i * batch_size + batch_size)]\n        #print(i, x_batch_name, (i * batch_size), (i * batch_size + batch_size))\n\n        x_batch = np.array([getPixels(fileName)\n                            for fileName in x_batch_name]).reshape(batch_size, 512, 512, 1)\n        i += 1\n        yield x_batch, y_batch","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training with partial train sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"LABEL = 4 # 4-16\nINTERVAL = 17\nOFFSET = LABEL%INTERVAL\nSIZE = 100000\nN_BATCH = 4\nN_EPOCHS = 1\n\nxf_train = []\ny_train = np.array([])\n\nstoreFileNames(LABEL, INTERVAL, SIZE, OFFSET)\n\nfor epoch in range(N_EPOCHS):\n    print(\"EPOCH\", epoch, \"/\", N_EPOCHS)\n    acc = []\n    loss = []\n    \n    index = -1\n    for X_batch, Y_batch in getBatch(N_BATCH):\n        # print(len(X_batch))\n        index += 1\n        if index%100==0:\n            print(\" BATCH:\", index, \"/\", SIZE//N_BATCH)\n        model.train_on_batch(X_batch, Y_batch)\n        score = model.evaluate(X_batch, Y_batch, verbose=0)\n        #print(\"batch accuracy:\", score[1], index)\n        loss.append(score[0])\n        acc.append(score[1])\n    print(\"Train loss\", np.mean(loss), \"train accuracy\", np.mean(acc))\n#    score = model.evaluate(X_test, Y_test)\n#    print(\"Test loss:\", score[0])\n#    print(\"Test accuracy:\", score[1])\n\nimport os\njson_string = model.to_json()\nopen(os.path.join('./', 'cnn_model.'+str(LABEL)+'.json'), 'w').write(json_string)\n!ls -l ./cnn_model*.json\n\nmodel.save_weights(os.path.join('./', 'cnn_model_weight.'+str(LABEL)+'.hdf5'))\n!ls -l ./cnn_model_weight.*.hdf5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* LABEL 4 (negative_exam_for_pe)\n  * Train loss 0.779805525456856 train accuracy 0.6635 : 10000 images\n  * Train loss 0.6715116521120071 train accuracy 0.671 : 10000 images\n  * Train loss 0.6602840732336044 train accuracy 0.671 : 10000 images\n  * Train loss 0.6380791419947147 train accuracy 0.67678 : 100000 images"},{"metadata":{},"cell_type":"markdown","source":"labels = [\n        \"negative_exam_for_pe\",\n        \"rv_lv_ratio_gte_1\",\n        \"rv_lv_ratio_lt_1\",\n        \"leftsided_pe\",\n        \"chronic_pe\",\n        \"rightsided_pe\",\n        \"acute_and_chronic_pe\",\n        \"central_pe\",\n        \"indeterminate\"]\n\n#### the label for SOPInstanceUIDs will be pe_present_on_image from the train.csv..\n# https://www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/discussion/181842\n\n* 3 : pe_present_on_image for each image\n* 4 : negative_exam_for_pe for each person\n* 5 : qa_motion (informational)\n* 6 : qa_contrast (informational)\n* 7 : flow_artifact (informational)\n* 8 : rv_lv_ratio_gte_1 for each person\n* 9 : rv_lv_ratio_lt_1 for each person\n* 10 : leftsided_pe for each person\n* 11 : chronic_pe for each person\n* 12 : true_filling_defect_not_pe\n* 13 : rightsided_pe for each person\n* 14 : acute_and_chronic_pe for each person\n* 15 : central_pe for each person\n* 16 : indeterminate for each person\n\n* negative_exam_for_pe : Yes -> no other labels\n* negative_exam_for_pe : No -> indeterminate yes or no\n* indeterminate : Yes -> qa_motion or qa_contrast (at least one label)\n* indeterminate : No -> at least one image with pe_present_on_image\n* At least one label for leftsided_pe, rightsided_pe, central_pe\n* Only one label for rv_lv_ratio_gte_1 or rv_lv_ratio_lt_1\n* Only one label for chronic_pe or acute_and_chronic_pe\n\n* https://www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/data\n* https://www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/discussion/183473"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}