{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook illustrates how to parallelize the process of meta-feature extraction using `joblib` to get meta-data and some pixel-based features from the training images.\n\nThe extracted features are used in [this notebook](https://www.kaggle.com/kozodoi/lightgbm-on-meta-features) that develops a LightGBM pipeline on top of extracted features.\n\nThe pipeline is inspired by [this notebook](https://www.kaggle.com/swarajshinde/rsna-pulmonary-embolism-analysis-eda-meta-data#Extracting-Meta-Data-from-Dicom-Data-and-Storing-in-CSV-Format). You can also check out [this notebook](https://www.kaggle.com/teeyee314/rsna-pe-metadata-with-multithreading?select=test_meta.csv) for an alternative approach to meta-feature extraction with `multiprocessing`."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"##### PACKAGES\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nimport pydicom as dcm\nimport PIL\n\nfrom tqdm import tqdm\nfrom joblib import Parallel, delayed\n\nimport cv2\nimport vtk\nfrom vtk.util import numpy_support","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### IMAGE PATH\n\nim_path = []\ntrain_path = '../input/rsna-str-pulmonary-embolism-detection/train/'\nfor i in tqdm(os.listdir(train_path)): \n    for j in os.listdir(train_path + i):\n        for k in os.listdir(train_path + i + '/' + j):\n            x = i + '/' + j + '/' + k\n            im_path.append(x)\nlen(im_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### EXTRACT META-FEATURES\n\ndef window(img, WL = 50, WW = 350):\n    upper, lower = WL + WW//2, WL - WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    return X\n\ndef extract_meta_feats(img):\n\n    img_id = img.split('/')[2].replace('.dcm', '')\n    image  = dcm.dcmread(train_path + img)\n    \n    ### META-FEATURES\n    \n    pixelspacing      = image.PixelSpacing[0]\n    slice_thicknesses = image.SliceThickness\n    kvp               = image.KVP\n    table_height      = image.TableHeight\n    x_ray             = image.XRayTubeCurrent\n    exposure          = image.Exposure\n    modality          = image.Modality\n    rot_direction     = image.RotationDirection \n    instance_number   = image.InstanceNumber\n    \n    \n    ### PIXEL-BASED FEATURES\n    \n    reader = vtk.vtkDICOMImageReader()\n    reader.SetFileName(train_path + img)\n    reader.Update()\n    _extent = reader.GetDataExtent()\n    ConstPixelDims = [_extent[1]-_extent[0]+1, _extent[3]-_extent[2]+1, _extent[5]-_extent[4]+1]\n\n    ConstPixelSpacing = reader.GetPixelSpacing()\n    imageData  = reader.GetOutput()\n    pointData  = imageData.GetPointData()\n    arrayData  = pointData.GetArray(0)\n    ArrayDicom = numpy_support.vtk_to_numpy(arrayData)\n    ArrayDicom = ArrayDicom.reshape(ConstPixelDims, order = 'F')\n    ArrayDicom = cv2.resize(ArrayDicom, (512,512))\n\n    img = ArrayDicom.astype(np.int16)\n    img[img <= -1000] = 0\n\n    intercept = reader.GetRescaleOffset()\n    slope     = reader.GetRescaleSlope()\n    if slope != 1:\n        img = slope * img.astype(np.float64)\n        img = img.astype(np.int16)\n    img += np.int16(intercept)\n\n    hu_min  = np.min(img)\n    hu_mean = np.mean(img)\n    hu_max  = np.max(img)\n    hu_std  = np.std(img)\n    \n    \n    ### WINDOW-BASED FEATURES\n    \n    img_lung = window(img, WL = -600, WW = 1500)\n    img_medi = window(img, WL = 40,   WW = 400)\n    img_pesp = window(img, WL = 100,  WW = 700)\n    \n    lung_mean = np.mean(img_lung)\n    lung_std  = np.std(img_lung)\n    \n    medi_mean = np.mean(img_medi)\n    medi_std = np.std(img_medi)\n    \n    pesp_mean = np.mean(img_pesp)\n    pesp_std  = np.std(img_pesp)\n    \n    \n    return [img_id, \n            pixelspacing, slice_thicknesses, kvp, table_height, x_ray, exposure, modality, rot_direction, instance_number,\n            hu_min, hu_mean, hu_max, hu_std,\n            lung_mean, lung_std, medi_mean, medi_std, pesp_mean, pesp_std\n           ]\n\nresults = Parallel(n_jobs = -1, verbose = 1)(map(delayed(extract_meta_feats), im_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##### SAVE FEATURES\n\ndf = pd.DataFrame(results, columns = ['SOPInstanceUID', \n                                      'pixelspacing',\n                                      'slice_thicknesses',\n                                      'kvp',\n                                      'table_height',\n                                      'x_ray_tube_current',\n                                      'exposure',\n                                      'modality',\n                                      'rotation_direction',\n                                      'instance_number',\n                                      'hu_min',\n                                      'hu_mean',\n                                      'hu_max',\n                                      'hu_std',\n                                      'lung_mean',\n                                      'lung_std',\n                                      'medi_mean',\n                                      'medi_std',\n                                      'pesp_mean',\n                                      'pesp_std'])\ndf.to_csv('train_meta.csv', index = False)\ndf.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}