{"cells":[{"metadata":{},"cell_type":"markdown","source":"The following notebook creates the submission file for the compeition. The complete pipeline is fastai based.\nUse the following notebook to create the resnet model for prediction.\nhttps://www.kaggle.com/imnishantg/fastai-model-training-pipeline\n\nYou would see that the submission metric is not very optimized for this notebook. I used a very small sample of studies to train the model. You can tweak around certain configurations to get the desired results."},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/fastai-model-training-pipeline/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2\nprint(\"done\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n## Loading Libraries\nimport numpy as np\nimport pandas as pd\n\nimport fastai\nfrom fastai.basics import *\nfrom fastai.callback.all import *\nfrom fastai.vision.all import *\nfrom fastai.medical.imaging import *\nimport torchvision.models as models\nimport pydicom\nimport shutil\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"source = '../input/rsna-str-pulmonary-embolism-detection'\nupload = '../input/fastai-model-training-pipeline/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = os.listdir(source)\nfiles","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"upl = os.listdir(upload)\nupl","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(f'{source}/test.csv')\nprint(df.shape)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# getter functions\n\nget_x = lambda x:f'{source}/test/{x.StudyInstanceUID}/{x.SeriesInstanceUID}/{x.SOPInstanceUID}.dcm'\n\nvocab = ['pe_present_on_image', 'negative_exam_for_pe', 'indeterminate', \n         'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', # Only one label should be true at a time\n         'chronic_pe', 'acute_and_chronic_pe', # Only one label can be true at a time\n         'leftsided_pe', 'central_pe', 'rightsided_pe', # More than one label can be true at a time\n         'qa_motion', 'qa_contrast', 'flow_artifact', 'true_filling_defect_not_pe'] # These are only informational. Maybe use it for study level inferences\n\n# get_y = ColReader(vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfms = [IntToFloatTensor(div=1000.0, div_mask=1), \n        *aug_transforms(size=224), \n        Normalize.from_stats(*imagenet_stats)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"block = DataBlock(blocks=(ImageBlock(cls=PILDicom)),\n                  get_x=get_x,\n                  batch_tfms=tfms)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dls = block.dataloaders(df, bs=512, num_workers=0)\ndls.show_batch(max_n=9, nrows=3, ncols=3, figsize=(20,20))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"head = create_head(nf=1024, n_out=14, lin_ftrs=[512, 128], concat_pool=True)\nconfig = cnn_config(custom_head=head)\n\nlearn = cnn_learner(dls, resnet34, config=config, n_out=14, pretrained=False, loss_func=nn.BCEWithLogitsLoss())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.model_dir = '.'\nlearn.load('../input/fastai-model-training-pipeline/resnet34')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = dls.test_dl(df)   # df[:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = learn.get_preds(dl=test_data, act=sigmoid)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Creating Submission File"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"test_filepaths = get_dicom_files(f'{source}/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_filepaths[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parse_StudyInstanceUID = lambda x: x.__str__().split('/')[-3]\nparse_SeriesInstanceUID = lambda x: x.__str__().split('/')[-2]\nparse_SOPInstanceUID = lambda x: x.__str__().split('/')[-1].split('.')[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(preds[0])\nsub.columns = vocab\nsub['StudyInstanceUID'] = [parse_StudyInstanceUID(x) for x in test_filepaths]\nsub['SeriesInstanceUID'] = [parse_SeriesInstanceUID(x) for x in test_filepaths]\nsub['SOPInstanceUID'] = [parse_SOPInstanceUID(x) for x in test_filepaths]\n\nprint(sub.shape)\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_p1 = sub.loc[:, ['SOPInstanceUID', 'pe_present_on_image']]\nsub_p1.columns = ['id', 'label']\n\nprint(sub_p1.shape)\nsub_p1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sel_cols = ['negative_exam_for_pe', 'indeterminate', \n            'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', # Only one label should be true at a time\n            'chronic_pe', 'acute_and_chronic_pe', # Only one label can be true at a time\n            'leftsided_pe', 'central_pe', 'rightsided_pe'] # More than one label can be true at a time\n\nsub_p2 = sub.loc[:, ['StudyInstanceUID']+sel_cols]\n\n# Temporary summary for study level predictions\nagg_func = {'negative_exam_for_pe' : ['min'], \n            'indeterminate' : ['min'], \n            'rv_lv_ratio_gte_1' : ['mean'], \n            'rv_lv_ratio_lt_1' : ['mean'], \n            'chronic_pe' : ['mean'], \n            'acute_and_chronic_pe' : ['mean'], \n            'leftsided_pe' : ['max'], \n            'central_pe' : ['max'], \n            'rightsided_pe' : ['max']}\n\nsub_p2 = sub_p2.groupby(['StudyInstanceUID']).agg(agg_func)\nsub_p2.columns = sub_p2.columns.droplevel(1)\nsub_p2 = sub_p2.reset_index()\n\n# Data Reshaping\nsub_p2 = pd.melt(sub_p2, id_vars=['StudyInstanceUID'], value_vars=sel_cols)\nsub_p2['id'] = sub_p2['StudyInstanceUID']+'_'+sub_p2['variable']\nsub_p2.drop(['StudyInstanceUID', 'variable'], inplace=True, axis=1)\nsub_p2 = sub_p2.loc[:, ['id', 'value']]\nsub_p2.columns = ['id', 'label']\n\nprint(sub_p2.shape)\nsub_p2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"finsub = pd.concat([sub_p2, sub_p1])\n\nprint(finsub.shape)\nfinsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"finsub.to_csv('../working/submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that the study level predictions are just a summary statistic for the predictions at image level.\nWe can definitely do better."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}