{"cells":[{"metadata":{},"cell_type":"markdown","source":"# PANDA: pandas-profiling for each data provider"},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport time\nimport pandas as pd\nimport numpy as np\nfrom openslide import OpenSlide\nimport openslide\nfrom pathlib import Path\nfrom pandas_profiling import ProfileReport","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_os_df(path, index, columns):\n    os_df = pd.DataFrame(columns=columns).set_index(index)\n    i = 0\n    start = time.time()\n    files = list(path.glob('*.tiff'))\n    max = len(files)\n    for file in files:\n        os=OpenSlide(str(file))\n        prp = os.properties\n        m = re.match(r'([a-z0-9]*)(_mask)?',file.stem)\n        image_id = str(m.groups(0)[0])\n        data = [ \n            os.dimensions[0],\n            os.dimensions[1],\n            os.level_dimensions[0][0],\n            os.level_dimensions[0][1],\n            os.level_dimensions[1][0],\n            os.level_dimensions[1][1],\n            os.level_dimensions[2][0],\n            os.level_dimensions[2][1],\n            os.level_downsamples[0],\n            os.level_downsamples[1],\n            os.level_downsamples[2],\n            prp[openslide.PROPERTY_NAME_QUICKHASH1] if openslide.PROPERTY_NAME_QUICKHASH1 in prp else np.nan,\n        ]\n\n        se = pd.Series(data, index=os_df.columns, name=image_id)\n        os_df = os_df.append(se)\n        os.close()\n        i += 1\n#         if i % 100 == 0: print('progress: {}/{}'.format(i, max))\n    elapsed_time = time.time() - start\n    print (\"elapsed_time:{0} [sec]\".format(elapsed_time))\n    return os_df\n\ndata_dir_path = Path('../input/prostate-cancer-grade-assessment/train_images/')\nlabel_dir_path = Path('../input/prostate-cancer-grade-assessment/train_label_masks/')\nindex = 'image_id'\ncolumns = [\n    index,\n    'dimensions_x',\n    'dimensions_y',\n    'level_dimensions_0_x',\n    'level_dimensions_0_y',\n    'level_dimensions_1_x',\n    'level_dimensions_1_y',\n    'level_dimensions_2_x',\n    'level_dimensions_2_y',\n    'level_downsamples_0',\n    'level_downsamples_1',\n    'level_downsamples_2',\n    'quickhash1',\n]\n\nlabel_columns = [\n    index,\n    'label_dimensions_x',\n    'label_dimensions_y',\n    'label_level_dimensions_0_x',\n    'label_level_dimensions_0_y',\n    'label_level_dimensions_1_x',\n    'label_level_dimensions_1_y',\n    'label_level_dimensions_2_x',\n    'label_level_dimensions_2_y',\n    'label_level_downsamples_0',\n    'label_level_downsamples_1',\n    'label_level_downsamples_2',\n    'label_quickhash1',\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create Dataframes"},{"metadata":{"trusted":true},"cell_type":"code","source":"csv_df = pd.read_csv('../input/prostate-cancer-grade-assessment/train.csv').set_index('image_id')\ndata_df = create_os_df(data_dir_path, index, columns)\nlabel_df = create_os_df(label_dir_path, index, label_columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Combine Dataframes"},{"metadata":{"trusted":true},"cell_type":"code","source":"os_df =  pd.DataFrame.join(pd.DataFrame.join(csv_df, data_df, on='image_id', how='left'), label_df, on='image_id', how='left')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Profiling report for karolinska's data"},{"metadata":{"trusted":true},"cell_type":"code","source":"karolinska = ProfileReport(os_df[os_df['data_provider'] == 'karolinska'], title=\"Profiling report for radboud's data\")\nkarolinska.to_file(output_file=\"ProfileReport_karolinska.html\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"karolinska.to_widgets()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Profiling report for radboud's data"},{"metadata":{"trusted":true},"cell_type":"code","source":"radboud = ProfileReport(os_df[os_df['data_provider'] == 'radboud'], title=\"Profiling report for radboud's data\")\nradboud.to_file(output_file=\"ProfileReport_radboud.html\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"radboud.to_widgets()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os_df.to_csv('joined.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}