{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of Contents\n* [Target](#target)\n* [Age](#age)\n* [Implant](#implant)\n* [Site](#site)\n* [Machine](#machine)\n* [Laterality](#laterality)","metadata":{}},{"cell_type":"code","source":"# packages\n\n# standard\nimport numpy as np\nimport pandas as pd\n\n# plots\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# statistics tools\nfrom statsmodels.graphics.mosaicplot import mosaic","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-05T11:02:29.865469Z","iopub.execute_input":"2022-12-05T11:02:29.865889Z","iopub.status.idle":"2022-12-05T11:02:30.555842Z","shell.execute_reply.started":"2022-12-05T11:02:29.865856Z","shell.execute_reply":"2022-12-05T11:02:30.554823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remember default plot settings\nplt_para_save = plt.rcParams['figure.figsize']","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:31.103216Z","iopub.execute_input":"2022-12-05T11:02:31.103997Z","iopub.status.idle":"2022-12-05T11:02:31.108528Z","shell.execute_reply.started":"2022-12-05T11:02:31.103957Z","shell.execute_reply":"2022-12-05T11:02:31.107456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read training data\ndf_train = pd.read_csv('../input/rsna-breast-cancer-detection/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:32.25735Z","iopub.execute_input":"2022-12-05T11:02:32.258146Z","iopub.status.idle":"2022-12-05T11:02:32.425574Z","shell.execute_reply.started":"2022-12-05T11:02:32.258098Z","shell.execute_reply":"2022-12-05T11:02:32.424385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# overview of data frame\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:33.485472Z","iopub.execute_input":"2022-12-05T11:02:33.485909Z","iopub.status.idle":"2022-12-05T11:02:33.524159Z","shell.execute_reply.started":"2022-12-05T11:02:33.485872Z","shell.execute_reply":"2022-12-05T11:02:33.52291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We observe a few missings in age and a lot of missing values in BIRADS and density!","metadata":{}},{"cell_type":"markdown","source":"<a id='target'></a>\n# Target","metadata":{}},{"cell_type":"code","source":"# group data by patient first to achieve uniqueness\ndf_unique = df_train.groupby('patient_id', as_index=False).agg(\n    site = pd.NamedAgg(column='site_id', aggfunc=max),\n    machine = pd.NamedAgg(column='machine_id', aggfunc=max),\n    age = pd.NamedAgg(column='age', aggfunc=np.mean),\n    implant = pd.NamedAgg(column='implant', aggfunc=max),\n    cancer = pd.NamedAgg(column='cancer', aggfunc=max))\ndf_unique.head(6)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:36.642497Z","iopub.execute_input":"2022-12-05T11:02:36.642924Z","iopub.status.idle":"2022-12-05T11:02:36.686843Z","shell.execute_reply.started":"2022-12-05T11:02:36.642885Z","shell.execute_reply":"2022-12-05T11:02:36.685416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimensions\ndf_unique.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:38.61106Z","iopub.execute_input":"2022-12-05T11:02:38.611512Z","iopub.status.idle":"2022-12-05T11:02:38.618987Z","shell.execute_reply.started":"2022-12-05T11:02:38.61147Z","shell.execute_reply":"2022-12-05T11:02:38.618114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### We have 11'913 patients in our data.","metadata":{}},{"cell_type":"code","source":"# frequencies\nfreqs = df_unique.cancer.value_counts()\nprint('Absolute counts:')\nprint(freqs)\nprint()\nprint('Relative frequencies:')\nprint(freqs/freqs.sum())\nfreqs.plot(kind='bar')\nplt.title('Occurrence of cancer')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:41.458492Z","iopub.execute_input":"2022-12-05T11:02:41.45957Z","iopub.status.idle":"2022-12-05T11:02:41.700816Z","shell.execute_reply.started":"2022-12-05T11:02:41.459529Z","shell.execute_reply":"2022-12-05T11:02:41.699474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='age'></a>\n# Age","metadata":{}},{"cell_type":"code","source":"# age distribution\ndf_unique.age.plot(kind='hist', bins=15, figsize=(12,3))\nplt.title('Age distribution')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:44.06797Z","iopub.execute_input":"2022-12-05T11:02:44.068764Z","iopub.status.idle":"2022-12-05T11:02:44.320897Z","shell.execute_reply.started":"2022-12-05T11:02:44.068695Z","shell.execute_reply":"2022-12-05T11:02:44.319714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# detailed version, show each age separately\ndf_unique.age.value_counts().sort_index().plot(kind='bar', figsize=(12,3))\nplt.title('Age distribution')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:45.393047Z","iopub.execute_input":"2022-12-05T11:02:45.393484Z","iopub.status.idle":"2022-12-05T11:02:46.213938Z","shell.execute_reply.started":"2022-12-05T11:02:45.393448Z","shell.execute_reply":"2022-12-05T11:02:46.21253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Impact of age on cancer","metadata":{}},{"cell_type":"code","source":"# violinplot, split by target\nplt.figure(figsize=(6,5))\nsns.violinplot(data=df_unique, x='cancer', y='age')\nplt.title('Age vs Cancer')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:48.196695Z","iopub.execute_input":"2022-12-05T11:02:48.197089Z","iopub.status.idle":"2022-12-05T11:02:48.53181Z","shell.execute_reply.started":"2022-12-05T11:02:48.197057Z","shell.execute_reply":"2022-12-05T11:02:48.530624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# binned version of age\ndf_unique['age_binned'] = pd.cut(df_unique.age, \n                                 [25,45,55,65,75,90])","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:49.853021Z","iopub.execute_input":"2022-12-05T11:02:49.853804Z","iopub.status.idle":"2022-12-05T11:02:49.871584Z","shell.execute_reply.started":"2022-12-05T11:02:49.85374Z","shell.execute_reply":"2022-12-05T11:02:49.870013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mosaic plot\nplt.rcParams['figure.figsize'] = (12,7)\nmosaic(df_unique, ['age_binned', 'cancer'], title='Cancer vs age[binned]')\nplt.ylim(0,1)\nplt.show()\nplt.rcParams['figure.figsize'] = plt_para_save","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:02:51.725874Z","iopub.execute_input":"2022-12-05T11:02:51.726275Z","iopub.status.idle":"2022-12-05T11:02:52.034208Z","shell.execute_reply.started":"2022-12-05T11:02:51.726241Z","shell.execute_reply":"2022-12-05T11:02:52.032941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Occurrence of cancer increases with age in our data.","metadata":{}},{"cell_type":"code","source":"# show rows with missing age\ndf_unique[df_unique.age.isna()]","metadata":{"execution":{"iopub.status.busy":"2022-12-05T11:03:56.400084Z","iopub.execute_input":"2022-12-05T11:03:56.400522Z","iopub.status.idle":"2022-12-05T11:03:56.420428Z","shell.execute_reply.started":"2022-12-05T11:03:56.400485Z","shell.execute_reply":"2022-12-05T11:03:56.418818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id='implant'></a>\n# Implant","metadata":{}},{"cell_type":"code","source":"# frequencies\nfreqs = df_unique.implant.value_counts()\nprint('Absolute counts:')\nprint(freqs)\nprint()\nprint('Relative frequencies:')\nprint(freqs/freqs.sum())\nfreqs.plot(kind='bar')\nplt.title('Occurrence of implant')\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:17.936897Z","iopub.execute_input":"2022-11-29T19:37:17.938346Z","iopub.status.idle":"2022-11-29T19:37:18.086127Z","shell.execute_reply.started":"2022-11-29T19:37:17.938281Z","shell.execute_reply":"2022-11-29T19:37:18.085263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 We can observe that the occurrence of implants is much rarer than the occurrence of cancer itself!","metadata":{}},{"cell_type":"markdown","source":"### Impact of implant on cancer","metadata":{}},{"cell_type":"code","source":"# cross table of implant and cancer\nctab = pd.crosstab(df_unique.implant, df_unique.cancer)\nctab","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.087336Z","iopub.execute_input":"2022-11-29T19:37:18.088148Z","iopub.status.idle":"2022-11-29T19:37:18.110587Z","shell.execute_reply.started":"2022-11-29T19:37:18.088112Z","shell.execute_reply":"2022-11-29T19:37:18.109069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's normalize this by row\n(ctab.transpose() / ctab.sum(axis=1)).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.112553Z","iopub.execute_input":"2022-11-29T19:37:18.113758Z","iopub.status.idle":"2022-11-29T19:37:18.125669Z","shell.execute_reply.started":"2022-11-29T19:37:18.113697Z","shell.execute_reply":"2022-11-29T19:37:18.124581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Occurrence of cancer is much rarer in our data when implants are present (1.75% vs 4.11%). However, the sample of patients with implants is relatively small here.","metadata":{}},{"cell_type":"markdown","source":"<a id='site'></a>\n# Site","metadata":{}},{"cell_type":"code","source":"# cross table of site and cancer\nctab = pd.crosstab(df_unique.site, df_unique.cancer)\nctab","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.126688Z","iopub.execute_input":"2022-11-29T19:37:18.127812Z","iopub.status.idle":"2022-11-29T19:37:18.155455Z","shell.execute_reply.started":"2022-11-29T19:37:18.127738Z","shell.execute_reply":"2022-11-29T19:37:18.154102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's normalize again\n(ctab.transpose() / ctab.sum(axis=1)).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.156991Z","iopub.execute_input":"2022-11-29T19:37:18.157353Z","iopub.status.idle":"2022-11-29T19:37:18.171865Z","shell.execute_reply.started":"2022-11-29T19:37:18.15732Z","shell.execute_reply":"2022-11-29T19:37:18.170692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Cancer rate on site 1 is somewhat higher.","metadata":{}},{"cell_type":"markdown","source":"<a id='machine'></a>\n# Machine","metadata":{}},{"cell_type":"code","source":"# cross table of machine and cancer\nctab = pd.crosstab(df_unique.machine, df_unique.cancer)\nctab","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.173609Z","iopub.execute_input":"2022-11-29T19:37:18.174228Z","iopub.status.idle":"2022-11-29T19:37:18.205409Z","shell.execute_reply.started":"2022-11-29T19:37:18.17419Z","shell.execute_reply":"2022-11-29T19:37:18.203348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's normalize again\n(ctab.transpose() / ctab.sum(axis=1)).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.208718Z","iopub.execute_input":"2022-11-29T19:37:18.209176Z","iopub.status.idle":"2022-11-29T19:37:18.224517Z","shell.execute_reply.started":"2022-11-29T19:37:18.20914Z","shell.execute_reply":"2022-11-29T19:37:18.22281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Note that machines with id 190 and 197 have only few observations. But also for the other machines we see quite some differences in the rates.","metadata":{}},{"cell_type":"markdown","source":"<a id='laterality'></a>\n# Laterality","metadata":{}},{"cell_type":"markdown","source":"#### Let's check if there is some asymmetry.","metadata":{}},{"cell_type":"code","source":"# group data by patient AND laterality now to achieve uniqueness\ndf_unique_LR = df_train.groupby(['patient_id','laterality'], as_index=False).agg(\n    cancer = pd.NamedAgg(column='cancer', aggfunc=max))\ndf_unique_LR.head(6)","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.226467Z","iopub.execute_input":"2022-11-29T19:37:18.227152Z","iopub.status.idle":"2022-11-29T19:37:18.265127Z","shell.execute_reply.started":"2022-11-29T19:37:18.227111Z","shell.execute_reply":"2022-11-29T19:37:18.264186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cross table of laterality and cancer\nctab = pd.crosstab(df_unique_LR.laterality, df_unique_LR.cancer)\nctab","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.266918Z","iopub.execute_input":"2022-11-29T19:37:18.267508Z","iopub.status.idle":"2022-11-29T19:37:18.290693Z","shell.execute_reply.started":"2022-11-29T19:37:18.267472Z","shell.execute_reply":"2022-11-29T19:37:18.289014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's normalize again\n(ctab.transpose() / ctab.sum(axis=1)).transpose()","metadata":{"execution":{"iopub.status.busy":"2022-11-29T19:37:18.294789Z","iopub.execute_input":"2022-11-29T19:37:18.29516Z","iopub.status.idle":"2022-11-29T19:37:18.309258Z","shell.execute_reply.started":"2022-11-29T19:37:18.295128Z","shell.execute_reply":"2022-11-29T19:37:18.307425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 💡 Ok, this looks pretty balanced.","metadata":{}},{"cell_type":"markdown","source":"### 🚧 work in progress 🚧","metadata":{}}]}