{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**1. Load modules**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom tqdm import tqdm\nimport openslide\nfrom openslide import OpenSlide\nimport cv2 \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-06T14:55:58.930939Z","iopub.execute_input":"2022-09-06T14:55:58.931817Z","iopub.status.idle":"2022-09-06T14:55:58.940435Z","shell.execute_reply.started":"2022-09-06T14:55:58.931777Z","shell.execute_reply":"2022-09-06T14:55:58.939435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/mayo-clinic-strip-ai/\"\ntrain_df = pd.read_csv(data_path + 'train.csv')\ntest_df  = pd.read_csv(data_path + 'test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:55:58.942219Z","iopub.execute_input":"2022-09-06T14:55:58.943155Z","iopub.status.idle":"2022-09-06T14:55:58.965774Z","shell.execute_reply.started":"2022-09-06T14:55:58.943085Z","shell.execute_reply":"2022-09-06T14:55:58.964486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"file_path\"] = train_df[\"image_id\"].apply(lambda x: data_path + \"train/\" + x + \".tif\")\ntrain_df[\"target\"] = train_df[\"label\"].apply(lambda x : 1 if x==\"CE\" else 0)\ntest_df[\"file_path\"] = test_df[\"image_id\"].apply(lambda x: data_path + \"test/\" + x + \".tif\")","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:55:58.967374Z","iopub.execute_input":"2022-09-06T14:55:58.967699Z","iopub.status.idle":"2022-09-06T14:55:58.977899Z","shell.execute_reply.started":"2022-09-06T14:55:58.967671Z","shell.execute_reply":"2022-09-06T14:55:58.976764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2.Calculate aspect ratio with log scale**\n\nIn log scale, the differences between height and width is symmetrical to origin.","metadata":{}},{"cell_type":"code","source":"# calculate aspect ratio\nimport math\ntrain_df['height'] = np.nan\ntrain_df['width'] = np.nan\nfor i,file in enumerate(train_df[\"file_path\"]):\n    slide=OpenSlide(file)\n    train_df['height'].iloc[i] = int(slide.properties.get(\"openslide.level[0].height\"))\n    train_df['width'].iloc[i] = int(slide.properties.get(\"openslide.level[0].width\"))\ntrain_df['image_aspect_ratio'] = train_df['width'] / train_df['height']\ntrain_df['log_aspect_ratio'] = train_df.image_aspect_ratio.apply(lambda x: math.log(x))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:55:58.98159Z","iopub.execute_input":"2022-09-06T14:55:58.982264Z","iopub.status.idle":"2022-09-06T14:56:06.275415Z","shell.execute_reply.started":"2022-09-06T14:55:58.98223Z","shell.execute_reply":"2022-09-06T14:56:06.274491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['height'] = np.nan\ntest_df['width'] = np.nan\nfor i,file in enumerate(test_df[\"file_path\"]):\n    slide=OpenSlide(file)\n    test_df['height'].iloc[i] = int(slide.properties.get(\"openslide.level[0].height\"))\n    test_df['width'].iloc[i] = int(slide.properties.get(\"openslide.level[0].width\"))\ntest_df['image_aspect_ratio'] = test_df['width'] / test_df['height']\ntrain_df['log_aspect_ratio'] = train_df.image_aspect_ratio.apply(lambda x: math.log(x))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:56:06.277259Z","iopub.execute_input":"2022-09-06T14:56:06.278542Z","iopub.status.idle":"2022-09-06T14:56:06.331449Z","shell.execute_reply.started":"2022-09-06T14:56:06.278497Z","shell.execute_reply":"2022-09-06T14:56:06.330432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3. Visualize train and test data**","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nfig, ax = plt.subplots(1,2, figsize=(16,5))\nsns.histplot(x='image_aspect_ratio', data = train_df, bins=100, ax=ax[0])\nax[0].set_title(\"Distribution of aspect ratio @ train\"), ax[0].set_ylabel(\"%\")\nsns.histplot(x='image_aspect_ratio', data = test_df, bins=100, ax=ax[1])\nax[1].set_title(\"Distribution of aspect ratio @ test\"), ax[1].set_ylabel(\"%\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:56:06.335076Z","iopub.execute_input":"2022-09-06T14:56:06.335483Z","iopub.status.idle":"2022-09-06T14:56:07.014813Z","shell.execute_reply.started":"2022-09-06T14:56:06.335449Z","shell.execute_reply":"2022-09-06T14:56:07.013936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**4. Visualize histgram of log aspect ratio for each center_id and all train data**","metadata":{}},{"cell_type":"code","source":"wn=3\nhn=4\nfig, ax = plt.subplots(hn, wn, figsize=(16,20))\nbins=20\n\ncenter_id_list = train_df.groupby('center_id').count().index.to_list()\nfor i in range(len(center_id_list)):\n    \n    sns.histplot(x='log_aspect_ratio',hue='target' ,data = train_df[train_df.center_id == i+1], bins=bins, kde=True, ax=ax[i//wn, i%wn])\n\n    ax[i//wn, i%wn].set_title(\"LogAspectRatio center_id={}, count={}\".format(i+1,train_df.groupby('center_id').count().iloc[i].target)), ax[i//wn, i%wn].set_ylabel(\"count\")\n\nsns.histplot(x='log_aspect_ratio',hue='target', data = train_df, bins=bins, ax=ax[(i+1)//wn, (i+1)%wn])\nax[(i+1)//wn, (i+1)%wn].set_title(\"LogAspectRatio total, count={}\".format(train_df.groupby('center_id').count().sum()[0])), ax[(i+1)//wn, (i+1)%wn].set_ylabel(\"count\")\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:56:07.016164Z","iopub.execute_input":"2022-09-06T14:56:07.016805Z","iopub.status.idle":"2022-09-06T14:56:10.520432Z","shell.execute_reply.started":"2022-09-06T14:56:07.016751Z","shell.execute_reply":"2022-09-06T14:56:10.519452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The result show that the train data set has bias in aspect ratio regardless of center_id.\nBelow are the results with bins separated by -2,5, -0.5, 0.5, and 2.5 to make it easier to see the quantitative bias.","metadata":{}},{"cell_type":"code","source":"wn=3\nhn=4\nfig, ax = plt.subplots(hn, wn, figsize=(16,20))\n\nbins=[-2.5, -0.5, 0.5, 2.5]\n\ncenter_id_list = train_df.groupby('center_id').count().index.to_list()\nfor i in range(len(center_id_list)):    \n    sns.histplot(x='log_aspect_ratio',hue='target' ,data = train_df[train_df.center_id == i+1], bins=bins, kde=True, ax=ax[i//wn, i%wn])\n    ax[i//wn, i%wn].set_title(\"LogAspectRatio center_id={}, count={}\".format(i+1,train_df.groupby('center_id').count().iloc[i].target)), ax[i//wn, i%wn].set_ylabel(\"count\")\n\nsns.histplot(x='log_aspect_ratio',hue='target', data = train_df, bins=bins, ax=ax[(i+1)//wn, (i+1)%wn])\nax[(i+1)//wn, (i+1)%wn].set_title(\"LogAspectRatio total, count={}\".format(train_df.groupby('center_id').count().sum()[0])), ax[(i+1)//wn, (i+1)%wn].set_ylabel(\"count\")\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-06T14:56:10.521892Z","iopub.execute_input":"2022-09-06T14:56:10.522887Z","iopub.status.idle":"2022-09-06T14:56:12.771045Z","shell.execute_reply.started":"2022-09-06T14:56:10.522851Z","shell.execute_reply":"2022-09-06T14:56:12.769921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The image near the origin has an aspect ratio of 1, or close to square, but most of the data is less than -0.5.\nFor example, training data with a log aspect ratio of 0.5 or greater and data with an aspect ratio of -0.5 or less together may adversely affect prediction accuracy.\n\nI hope this work book helps you.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}