{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 1. Data Overview","metadata":{}},{"cell_type":"code","source":"# imports \nimport numpy as np\nimport pandas as pd \nimport os\nimport glob\n\n# visualisation'\nimport matplotlib.pyplot as plt\nimport seaborn as sns \nsns.set_style(style = 'darkgrid')\nsns.set_palette('viridis')\n\n%matplotlib inline","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T07:07:31.862423Z","iopub.execute_input":"2022-12-28T07:07:31.862829Z","iopub.status.idle":"2022-12-28T07:07:32.876967Z","shell.execute_reply.started":"2022-12-28T07:07:31.862789Z","shell.execute_reply":"2022-12-28T07:07:32.875404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntest = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv')\n\nprint('Train data')\ntrain.head()","metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T07:07:32.879325Z","iopub.execute_input":"2022-12-28T07:07:32.880205Z","iopub.status.idle":"2022-12-28T07:07:33.02767Z","shell.execute_reply.started":"2022-12-28T07:07:32.880141Z","shell.execute_reply":"2022-12-28T07:07:33.026447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test data')\ntest.head()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T07:07:33.029977Z","iopub.execute_input":"2022-12-28T07:07:33.031188Z","iopub.status.idle":"2022-12-28T07:07:33.048396Z","shell.execute_reply.started":"2022-12-28T07:07:33.031125Z","shell.execute_reply":"2022-12-28T07:07:33.047299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('---Train data---')\nprint('Data Shape: {:,d} rows X {:d} columns'.format(train.shape[0], train.shape[1]))\nprint('Total no. of scans: {:,d}'.format(len(train)))\nprint('Unique patients: {:,d}'.format(train['patient_id'].nunique()))\nprint()\n\nprint('---Test data---')\nprint('Data Shape: {:,d} rows X {:d} columns'.format(test.shape[0], test.shape[1]))\nprint('Total no. of scans: {:,d}'.format(len(test)))\nprint('Unique patients: {:,d}'.format(test['patient_id'].nunique()))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T07:07:33.051034Z","iopub.execute_input":"2022-12-28T07:07:33.05209Z","iopub.status.idle":"2022-12-28T07:07:33.073116Z","shell.execute_reply.started":"2022-12-28T07:07:33.052049Z","shell.execute_reply":"2022-12-28T07:07:33.071584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Feature types\n\nTaking a look at the features and their type (categorical/continuous).","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows = 5, ncols = 2, figsize =(15,25))\n\ntrain_copy = train.copy()\nfeatures = [\"site_id\", \"laterality\", \"view\", \"age\", \"biopsy\", \"invasive\",\"BIRADS\", \"implant\", \"density\", \"machine_id\"]\naxs = axs.flatten()\n\nfor i, feature in enumerate(features):\n    _ = sns.histplot(train_copy, x=\"{}\".format(feature), ax=axs[i])\n    _ = axs[i].set_title(\"{} Distribution\".format(feature))\n    _ = axs[i].set_ylabel(\"\")\n    _ = axs[i].set_xlabel(\"\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T08:35:43.567036Z","iopub.execute_input":"2022-12-28T08:35:43.568104Z","iopub.status.idle":"2022-12-28T08:35:46.095448Z","shell.execute_reply.started":"2022-12-28T08:35:43.568056Z","shell.execute_reply":"2022-12-28T08:35:46.094006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Categorical features: `site_id`, `laterality`, `view`, `age`, `biopsy`, `invasive`,`BIRADS`, `implant`, `density`, `machine_id`\n* Continuos Features: `age`","metadata":{}},{"cell_type":"markdown","source":"## 3. Spearman Correlation\n\nWill help to look at the features that are correlated and to what extent. The correlation values range from -1 to 1. The more the value is towards either extremity, the more the correlation. ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (12,12))\nfeatures = [\"site_id\", \"laterality\", \"view\", \"age\", \"biopsy\", \"invasive\",\"BIRADS\", \"implant\", \n            \"density\", \"machine_id\", \"difficult_negative_case\", \"cancer\"]\n\ncorr_matrix = train[features].corr('spearman')\nsns.heatmap(corr_matrix, \n            annot = True, \n            cmap = 'flare', \n            mask = np.triu(np.ones_like(corr_matrix, dtype = bool)),  \n            center = 0,\n            square = True, \n            linewidths = 0.1 )","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T08:33:08.928262Z","iopub.execute_input":"2022-12-28T08:33:08.928693Z","iopub.status.idle":"2022-12-28T08:33:09.495168Z","shell.execute_reply.started":"2022-12-28T08:33:08.928618Z","shell.execute_reply":"2022-12-28T08:33:09.493904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* `invasive` and `cancer`\n* `biopsy` and `cancer`\n* `invasive` and `biopsy`\n* `machine_id` and `BIRADS`\n* `difficult_negative_case` and `biopsy`","metadata":{}},{"cell_type":"markdown","source":"## 4. Missing Data\n\nWe look at the missing data and decide what we could do with those features.","metadata":{}},{"cell_type":"code","source":"print('Percentage of missing data per feature:')\n100 * train.isnull().sum() / len(train)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T08:10:39.102126Z","iopub.execute_input":"2022-12-28T08:10:39.102572Z","iopub.status.idle":"2022-12-28T08:10:39.124464Z","shell.execute_reply.started":"2022-12-28T08:10:39.102538Z","shell.execute_reply":"2022-12-28T08:10:39.122651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There seems to be missing data in BIRADS, density, and age columns. Let's check what these mean and what are the categories.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows = 1, ncols = 1, figsize = (10, 7))\n\nmissing_data = train.isnull().sum().loc[lambda x: x > 0]\n_ = sns.barplot(x = missing_data.index, y = missing_data.values, ax = axs)\nfor p in axs.patches:\n    axs.text(x = p.get_x() + (p.get_width() / 2), \n             y = p.get_height() + 300, \n             s = \"{:,d}\".format(round(p.get_height())), \n             ha = \"center\")\n_ = axs.set_xlabel('Features', fontsize = 14)\n_ = axs.set_ylabel('No. of missing records', fontsize = 14)\n_ = axs.set_title('Missing Data', fontsize = 16)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-12-28T07:47:49.702587Z","iopub.execute_input":"2022-12-28T07:47:49.703028Z","iopub.status.idle":"2022-12-28T07:47:49.887016Z","shell.execute_reply.started":"2022-12-28T07:47:49.702992Z","shell.execute_reply":"2022-12-28T07:47:49.885753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n\n * `age` has only 37 records with missing values. With a low number of missing records and `age` being a continuous feature, we can fill in the   records with the mean age. `age` appears in the test dataset.\n * `BIRADS` has around 52% records with missing data. We can either find a way to fill these values (would need furtur explorartion) or ignore this feature altogether since it does not appear in the test dataset. \n * `density` also has 46% records with missing data. Same as `BIRADS`, we find a way to fill the null values or ignore the feature altogether since it does not appear in the test set.","metadata":{}},{"cell_type":"markdown","source":"## 4.1 Age","metadata":{}},{"cell_type":"code","source":"features = [\"age\"]\ntrain[features].describe().T.style.bar(\n    subset=['mean'], color='#7BCC70')\\\n    .background_gradient(subset=['std'], cmap='Reds')\\\n    .background_gradient(subset=['50%'], cmap='coolwarm')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T08:05:10.545256Z","iopub.execute_input":"2022-12-28T08:05:10.545664Z","iopub.status.idle":"2022-12-28T08:05:10.633012Z","shell.execute_reply.started":"2022-12-28T08:05:10.545632Z","shell.execute_reply":"2022-12-28T08:05:10.631706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (25, 10))\nsns.countplot(x = 'age', data = train, palette = 'viridis')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T08:32:18.974258Z","iopub.execute_input":"2022-12-28T08:32:18.975073Z","iopub.status.idle":"2022-12-28T08:32:20.351276Z","shell.execute_reply.started":"2022-12-28T08:32:18.975037Z","shell.execute_reply":"2022-12-28T08:32:20.349312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# function to fill null age values\n\ndef fill_age(age):\n    '''\n    takes in age parameter\n    if the age is null, then return the mean age (59)\n    else returns the original age\n    '''\n    if np.isnan(age): \n        return 59\n    else: \n        return age","metadata":{"execution":{"iopub.status.busy":"2022-12-28T08:01:48.286875Z","iopub.execute_input":"2022-12-28T08:01:48.287326Z","iopub.status.idle":"2022-12-28T08:01:48.295073Z","shell.execute_reply.started":"2022-12-28T08:01:48.287289Z","shell.execute_reply":"2022-12-28T08:01:48.292989Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations:**\n* `age` ranges from 26 to 89.\n* The data is almost normally distributed with most records from the age group of 50-70.\n* The mean age is 59 ± 10. We can fill the records with missing age with this value.","metadata":{}},{"cell_type":"markdown","source":"## 4.2 BIRADS\n\n* `0` if the breast required follow-up\n* `1` if the breast was rated as negative for cancer\n* `2` if the breast was rated as normal. \n* Only provided for train.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows = 1, ncols = 1, figsize = (10, 7))\n\n_ = sns.countplot(x = 'BIRADS', data = train, ax = axs, palette = 'flare')\nfor p in axs.patches:\n    axs.text(x = p.get_x() + (p.get_width() / 2), \n             y = p.get_height() + 200, \n             s = \"{:,d}\".format(round(p.get_height())), \n             ha = \"center\")\n_ = axs.set_xlabel('BIRADS Score')\n_ = axs.set_ylabel('No. of Records')\n_ = axs.set_title('BIRADS Score Counts')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T08:48:04.839588Z","iopub.execute_input":"2022-12-28T08:48:04.839987Z","iopub.status.idle":"2022-12-28T08:48:05.034176Z","shell.execute_reply.started":"2022-12-28T08:48:04.839953Z","shell.execute_reply":"2022-12-28T08:48:05.032065Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.3 Density\n* A rating for how dense the breast tissue is, with `A` being the least dense and `D` being the most dense. \n* Extremely dense tissue can make diagnosis more difficult.\n* Only provided for train.","metadata":{}},{"cell_type":"code","source":"fig, axs = plt.subplots(nrows = 1, ncols = 1, figsize = (10, 7))\ndensity_order = ['A', 'B', 'C', 'D']\n\n_ = sns.countplot(x = 'density', data = train, order = density_order, palette = 'flare', ax = axs)\nfor p in axs.patches:\n    axs.text(x = p.get_x() + (p.get_width() / 2), \n             y = p.get_height() + 200, \n             s = \"{:,d}\".format(round(p.get_height())), \n             ha = \"center\")\n_ = axs.set_xlabel('Density Rating')\n_ = axs.set_ylabel('No. of Records')\n_ = axs.set_title('Density Rating Counts')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T08:55:34.073465Z","iopub.execute_input":"2022-12-28T08:55:34.073884Z","iopub.status.idle":"2022-12-28T08:55:34.284712Z","shell.execute_reply.started":"2022-12-28T08:55:34.073853Z","shell.execute_reply":"2022-12-28T08:55:34.283946Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Although it took me a while to organise and make sure the visuals were good, it was a fun process! Feedback and suggestions are highly appreciated! Upvote if you liked reading through :)*","metadata":{}}]}