{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Load library\nimport os\nimport glob\nimport cv2\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:24.940132Z","iopub.execute_input":"2022-12-21T23:26:24.940491Z","iopub.status.idle":"2022-12-21T23:26:24.945917Z","shell.execute_reply.started":"2022-12-21T23:26:24.94046Z","shell.execute_reply":"2022-12-21T23:26:24.944703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset preparation","metadata":{}},{"cell_type":"code","source":"data_path = \"/kaggle/input/rsna-breast-cancer-detection\"\ntrain_image_path = os.path.join(data_path,\"train_images\")\ntest_image_path = os.path.join(data_path,\"test_images\")\nos.path.exists(train_image_path)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:27.753945Z","iopub.execute_input":"2022-12-21T23:26:27.754292Z","iopub.status.idle":"2022-12-21T23:26:27.761872Z","shell.execute_reply.started":"2022-12-21T23:26:27.754263Z","shell.execute_reply":"2022-12-21T23:26:27.760927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find all csv files in the dataset\ncsv_files = glob.glob(data_path+\"/*.csv\")\ncsv_files","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:31.475135Z","iopub.execute_input":"2022-12-21T23:26:31.475516Z","iopub.status.idle":"2022-12-21T23:26:31.4832Z","shell.execute_reply.started":"2022-12-21T23:26:31.475484Z","shell.execute_reply":"2022-12-21T23:26:31.482159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train csv to dataframe\ntrain_df = pd.read_csv(csv_files[1])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:38.751134Z","iopub.execute_input":"2022-12-21T23:26:38.751575Z","iopub.status.idle":"2022-12-21T23:26:38.876539Z","shell.execute_reply.started":"2022-12-21T23:26:38.751537Z","shell.execute_reply":"2022-12-21T23:26:38.875242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.columns","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:42.138592Z","iopub.execute_input":"2022-12-21T23:26:42.139068Z","iopub.status.idle":"2022-12-21T23:26:42.158983Z","shell.execute_reply.started":"2022-12-21T23:26:42.13903Z","shell.execute_reply":"2022-12-21T23:26:42.157978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:44.944199Z","iopub.execute_input":"2022-12-21T23:26:44.944555Z","iopub.status.idle":"2022-12-21T23:26:44.97898Z","shell.execute_reply.started":"2022-12-21T23:26:44.944525Z","shell.execute_reply":"2022-12-21T23:26:44.977975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What is the distribution of cancer cases","metadata":{}},{"cell_type":"code","source":"train_df.head","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:48.401066Z","iopub.execute_input":"2022-12-21T23:26:48.401439Z","iopub.status.idle":"2022-12-21T23:26:48.421388Z","shell.execute_reply.started":"2022-12-21T23:26:48.401408Z","shell.execute_reply":"2022-12-21T23:26:48.420308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"difficult_negative_case\"].value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:26:54.971238Z","iopub.execute_input":"2022-12-21T23:26:54.971601Z","iopub.status.idle":"2022-12-21T23:26:54.981415Z","shell.execute_reply.started":"2022-12-21T23:26:54.971569Z","shell.execute_reply":"2022-12-21T23:26:54.979708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"machine_id\"].value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:09.521343Z","iopub.execute_input":"2022-12-21T23:27:09.521719Z","iopub.status.idle":"2022-12-21T23:27:09.530998Z","shell.execute_reply.started":"2022-12-21T23:27:09.521688Z","shell.execute_reply":"2022-12-21T23:27:09.529899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Is there any difference for the machine's ability in detecting cancer?**","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nplt.figure(figsize=(20,10))\nsns.histplot(data=train_df,x=\"machine_id\",hue=\"cancer\",kde=\"true\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:20.620971Z","iopub.execute_input":"2022-12-21T23:27:20.621341Z","iopub.status.idle":"2022-12-21T23:27:21.95247Z","shell.execute_reply.started":"2022-12-21T23:27:20.621304Z","shell.execute_reply":"2022-12-21T23:27:21.951446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"machine = pd.DataFrame(train_df[\"machine_id\"].value_counts(dropna = False))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:29.402435Z","iopub.execute_input":"2022-12-21T23:27:29.40281Z","iopub.status.idle":"2022-12-21T23:27:29.408914Z","shell.execute_reply.started":"2022-12-21T23:27:29.40278Z","shell.execute_reply":"2022-12-21T23:27:29.407964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"machine_cancer = []\nfor i in range(len(machine)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['machine_id'] == machine.index.values[i]))\n    machine_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:35.430032Z","iopub.execute_input":"2022-12-21T23:27:35.430427Z","iopub.status.idle":"2022-12-21T23:27:35.444326Z","shell.execute_reply.started":"2022-12-21T23:27:35.430386Z","shell.execute_reply":"2022-12-21T23:27:35.443278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"machine_cancer","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:42.227707Z","iopub.execute_input":"2022-12-21T23:27:42.228696Z","iopub.status.idle":"2022-12-21T23:27:42.237473Z","shell.execute_reply.started":"2022-12-21T23:27:42.228643Z","shell.execute_reply":"2022-12-21T23:27:42.236351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp  = [i / j for i, j in zip(machine_cancer, machine.values.tolist())]\ntmp2 = []\nfor i in range(len(tmp)):\n    tmp2.append(tmp[i].item())\nplt.bar(np.arange(len(tmp2)),tmp2)\nplt.xticks(np.arange(len(tmp2)),machine.index.values)\nplt.xlabel(\"Machine id\")\nplt.ylabel(\"Cancer detection frequency\")\nplt.title(\"Cancer detection frequency for different machine\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:27:47.370654Z","iopub.execute_input":"2022-12-21T23:27:47.371737Z","iopub.status.idle":"2022-12-21T23:27:47.625326Z","shell.execute_reply.started":"2022-12-21T23:27:47.371673Z","shell.execute_reply":"2022-12-21T23:27:47.623659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**different machine have different cancer detection frequency**","metadata":{}},{"cell_type":"markdown","source":"**For machine 210 and 197, they found no cancer. For machine 93 and 216, their detection frequency are exceptionaly low.**","metadata":{}},{"cell_type":"code","source":"train_df[\"invasive\"].value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:28:25.14487Z","iopub.execute_input":"2022-12-21T23:28:25.14587Z","iopub.status.idle":"2022-12-21T23:28:25.170776Z","shell.execute_reply.started":"2022-12-21T23:28:25.145777Z","shell.execute_reply":"2022-12-21T23:28:25.169678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"biopsy\"].value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:28:29.546367Z","iopub.execute_input":"2022-12-21T23:28:29.54675Z","iopub.status.idle":"2022-12-21T23:28:29.555685Z","shell.execute_reply.started":"2022-12-21T23:28:29.546717Z","shell.execute_reply":"2022-12-21T23:28:29.55456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**For invasive and biopsy information, we can consider use them as semi learning information source.**","metadata":{}},{"cell_type":"code","source":"train_df[\"density\"].value_counts(dropna = False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:28:54.627995Z","iopub.execute_input":"2022-12-21T23:28:54.628387Z","iopub.status.idle":"2022-12-21T23:28:54.638909Z","shell.execute_reply.started":"2022-12-21T23:28:54.628357Z","shell.execute_reply":"2022-12-21T23:28:54.637849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**What is the relationship between density and cancer derection?**","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nplt.figure(figsize=(20,10))\nsns.histplot(data=train_df,x=\"density\",hue=\"cancer\",kde=\"true\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:28:59.04259Z","iopub.execute_input":"2022-12-21T23:28:59.042952Z","iopub.status.idle":"2022-12-21T23:28:59.542882Z","shell.execute_reply.started":"2022-12-21T23:28:59.042922Z","shell.execute_reply":"2022-12-21T23:28:59.541969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"density = pd.DataFrame(train_df[\"density\"].value_counts(dropna = False))\ndensity_cancer = []\nfor i in range(len(density)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['density'] == density.index.values[i]))\n    density_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:29:03.406738Z","iopub.execute_input":"2022-12-21T23:29:03.407667Z","iopub.status.idle":"2022-12-21T23:29:03.430371Z","shell.execute_reply.started":"2022-12-21T23:29:03.407595Z","shell.execute_reply":"2022-12-21T23:29:03.429503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"density_cancer","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:29:07.539537Z","iopub.execute_input":"2022-12-21T23:29:07.540215Z","iopub.status.idle":"2022-12-21T23:29:07.546678Z","shell.execute_reply.started":"2022-12-21T23:29:07.540181Z","shell.execute_reply":"2022-12-21T23:29:07.545682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"density","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:29:11.157513Z","iopub.execute_input":"2022-12-21T23:29:11.157872Z","iopub.status.idle":"2022-12-21T23:29:11.169542Z","shell.execute_reply.started":"2022-12-21T23:29:11.157844Z","shell.execute_reply":"2022-12-21T23:29:11.16819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp  = [i / j for i, j in zip(density_cancer, density.values.tolist())]\ntmp2 = []\nfor i in range(len(tmp)):\n    tmp2.append(tmp[i].item())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:29:14.839674Z","iopub.execute_input":"2022-12-21T23:29:14.84037Z","iopub.status.idle":"2022-12-21T23:29:14.849136Z","shell.execute_reply.started":"2022-12-21T23:29:14.840319Z","shell.execute_reply":"2022-12-21T23:29:14.847697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(np.arange(len(tmp2)),tmp2)\nplt.xticks(np.arange(len(tmp2)),density.index.values)\nplt.xlabel(\"Density\")\nplt.ylabel(\"Cancer frequency\")\nplt.title(\"Cancer frequency for different frequency\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:29:17.427324Z","iopub.execute_input":"2022-12-21T23:29:17.427694Z","iopub.status.idle":"2022-12-21T23:29:17.630965Z","shell.execute_reply.started":"2022-12-21T23:29:17.427662Z","shell.execute_reply":"2022-12-21T23:29:17.630043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Not exactly the same as description, highest density \"D\" indicates lowest cancer frequency. However, lowest denstiy \"A\" not directly mean highest cancer frequency. If density is \"nan\", the cancer detection frequency is 0**","metadata":{}},{"cell_type":"code","source":"train_df[\"cancer\"].value_counts()\n# Highly imbalanced data","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:39.223451Z","iopub.execute_input":"2022-12-21T23:30:39.22382Z","iopub.status.idle":"2022-12-21T23:30:39.23273Z","shell.execute_reply.started":"2022-12-21T23:30:39.22379Z","shell.execute_reply":"2022-12-21T23:30:39.231778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Cancer distribution by age**","metadata":{}},{"cell_type":"code","source":"train_df[\"view\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:42.072957Z","iopub.execute_input":"2022-12-21T23:30:42.073323Z","iopub.status.idle":"2022-12-21T23:30:42.086415Z","shell.execute_reply.started":"2022-12-21T23:30:42.073284Z","shell.execute_reply":"2022-12-21T23:30:42.084926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"view = pd.DataFrame(train_df[\"view\"].value_counts(dropna = False))\nview_cancer = []\nfor i in range(len(view)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['view'] == view.index.values[i]))\n    view_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:40:59.926721Z","iopub.execute_input":"2022-12-21T23:40:59.92709Z","iopub.status.idle":"2022-12-21T23:40:59.961423Z","shell.execute_reply.started":"2022-12-21T23:40:59.927057Z","shell.execute_reply":"2022-12-21T23:40:59.960537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp  = [i / j for i, j in zip(view_cancer, view.values.tolist())]\ntmp2 = []\nfor i in range(len(tmp)):\n    tmp2.append(tmp[i].item())\n\n    \nplt.bar(np.arange(len(tmp2)),tmp2)\nplt.xticks(np.arange(len(tmp2)),view.index.values)\nplt.xlabel(\"view\")\nplt.ylabel(\"Cancer frequency\")\nplt.title(\"Cancer frequency for view\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:41:44.025214Z","iopub.execute_input":"2022-12-21T23:41:44.025703Z","iopub.status.idle":"2022-12-21T23:41:44.242576Z","shell.execute_reply.started":"2022-12-21T23:41:44.025661Z","shell.execute_reply":"2022-12-21T23:41:44.241664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**MLO and CC have similar cancer detection frequency. Although AT and significantly higer frequency. Its number is small.**","metadata":{}},{"cell_type":"code","source":"train_df[\"laterality\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:46.379435Z","iopub.execute_input":"2022-12-21T23:30:46.379807Z","iopub.status.idle":"2022-12-21T23:30:46.389701Z","shell.execute_reply.started":"2022-12-21T23:30:46.379775Z","shell.execute_reply":"2022-12-21T23:30:46.388607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"laterality = pd.DataFrame(train_df[\"laterality\"].value_counts(dropna = False))\nlaterality_cancer = []\nfor i in range(len(laterality)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['laterality'] == laterality.index.values[i]))\n    laterality_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:36:48.474078Z","iopub.execute_input":"2022-12-21T23:36:48.474443Z","iopub.status.idle":"2022-12-21T23:36:48.493269Z","shell.execute_reply.started":"2022-12-21T23:36:48.474412Z","shell.execute_reply":"2022-12-21T23:36:48.49233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"laterality_cancer","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:36:58.852238Z","iopub.execute_input":"2022-12-21T23:36:58.852633Z","iopub.status.idle":"2022-12-21T23:36:58.859666Z","shell.execute_reply.started":"2022-12-21T23:36:58.852584Z","shell.execute_reply":"2022-12-21T23:36:58.858398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp  = [i / j for i, j in zip(laterality_cancer, laterality.values.tolist())]\ntmp2 = []\nfor i in range(len(tmp)):\n    tmp2.append(tmp[i].item())\n\n    \nplt.bar(np.arange(len(tmp2)),tmp2)\nplt.xticks(np.arange(len(tmp2)),laterality.index.values)\nplt.xlabel(\"laterality\")\nplt.ylabel(\"Cancer frequency\")\nplt.title(\"Cancer frequency for laterality\")\n","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:38:33.315446Z","iopub.execute_input":"2022-12-21T23:38:33.315823Z","iopub.status.idle":"2022-12-21T23:38:33.500364Z","shell.execute_reply.started":"2022-12-21T23:38:33.315789Z","shell.execute_reply":"2022-12-21T23:38:33.499497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Different laterality means trivial difference on cancer detection frequency**","metadata":{}},{"cell_type":"code","source":"len(pd.unique(train_df[\"patient_id\"]))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:50.922661Z","iopub.execute_input":"2022-12-21T23:30:50.923338Z","iopub.status.idle":"2022-12-21T23:30:50.931718Z","shell.execute_reply.started":"2022-12-21T23:30:50.923294Z","shell.execute_reply":"2022-12-21T23:30:50.930673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient = pd.DataFrame(train_df[\"patient_id\"].value_counts(dropna = False))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:54.310182Z","iopub.execute_input":"2022-12-21T23:30:54.310861Z","iopub.status.idle":"2022-12-21T23:30:54.318261Z","shell.execute_reply.started":"2022-12-21T23:30:54.310819Z","shell.execute_reply":"2022-12-21T23:30:54.316954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient.head","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:30:59.3845Z","iopub.execute_input":"2022-12-21T23:30:59.384953Z","iopub.status.idle":"2022-12-21T23:30:59.397505Z","shell.execute_reply.started":"2022-12-21T23:30:59.384914Z","shell.execute_reply":"2022-12-21T23:30:59.396444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(20,10))\nsns.histplot(data=patient,x=\"patient_id\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:02.030982Z","iopub.execute_input":"2022-12-21T23:31:02.031419Z","iopub.status.idle":"2022-12-21T23:31:02.695915Z","shell.execute_reply.started":"2022-12-21T23:31:02.031377Z","shell.execute_reply":"2022-12-21T23:31:02.694757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"patient.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-19T03:54:43.031336Z","iopub.execute_input":"2022-12-19T03:54:43.031787Z","iopub.status.idle":"2022-12-19T03:54:43.044001Z","shell.execute_reply.started":"2022-12-19T03:54:43.031748Z","shell.execute_reply":"2022-12-19T03:54:43.042823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**most patient with 6 or less images**","metadata":{}},{"cell_type":"code","source":"train_df[\"site_id\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:10.479991Z","iopub.execute_input":"2022-12-21T23:31:10.480349Z","iopub.status.idle":"2022-12-21T23:31:10.488433Z","shell.execute_reply.started":"2022-12-21T23:31:10.480318Z","shell.execute_reply":"2022-12-21T23:31:10.487198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site = pd.DataFrame(train_df[\"site_id\"].value_counts(dropna = False))\nsite_cancer = []\nfor i in range(len(site)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['site_id'] == site.index.values[i]))\n    site_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:13.081105Z","iopub.execute_input":"2022-12-21T23:31:13.081471Z","iopub.status.idle":"2022-12-21T23:31:13.092028Z","shell.execute_reply.started":"2022-12-21T23:31:13.08144Z","shell.execute_reply":"2022-12-21T23:31:13.090953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"site_cancer","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:16.441964Z","iopub.execute_input":"2022-12-21T23:31:16.442337Z","iopub.status.idle":"2022-12-21T23:31:16.448881Z","shell.execute_reply.started":"2022-12-21T23:31:16.442289Z","shell.execute_reply":"2022-12-21T23:31:16.447784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp  = [i / j for i, j in zip(site_cancer, site.values.tolist())]\ntmp2 = []\nfor i in range(len(tmp)):\n    tmp2.append(tmp[i].item())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:19.650413Z","iopub.execute_input":"2022-12-21T23:31:19.650823Z","iopub.status.idle":"2022-12-21T23:31:19.656811Z","shell.execute_reply.started":"2022-12-21T23:31:19.650792Z","shell.execute_reply":"2022-12-21T23:31:19.655596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.bar(np.arange(len(tmp2)),tmp2)\nplt.xticks(np.arange(len(tmp2)),site.index.values)\nplt.xlabel(\"site_id\")\nplt.ylabel(\"Cancer frequency\")\nplt.title(\"Cancer frequency for different site\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:21.601901Z","iopub.execute_input":"2022-12-21T23:31:21.602249Z","iopub.status.idle":"2022-12-21T23:31:21.777185Z","shell.execute_reply.started":"2022-12-21T23:31:21.602219Z","shell.execute_reply":"2022-12-21T23:31:21.776281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**For different sites, the frequency is similar. so not informational to cancer prediction**","metadata":{}},{"cell_type":"code","source":"train_df[\"age\"].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:31.235039Z","iopub.execute_input":"2022-12-21T23:31:31.235386Z","iopub.status.idle":"2022-12-21T23:31:31.250036Z","shell.execute_reply.started":"2022-12-21T23:31:31.235358Z","shell.execute_reply":"2022-12-21T23:31:31.249032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nplt.figure(figsize=(20,10))\nsns.histplot(data=train_df,x=\"age\",hue=\"cancer\",kde=\"true\")\n","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:31:33.503927Z","iopub.execute_input":"2022-12-21T23:31:33.50427Z","iopub.status.idle":"2022-12-21T23:31:34.501517Z","shell.execute_reply.started":"2022-12-21T23:31:33.50424Z","shell.execute_reply":"2022-12-21T23:31:34.500534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Populations'age distribution is normal. It is consistent with common sense**","metadata":{}},{"cell_type":"code","source":"age = pd.DataFrame(train_df[\"age\"].value_counts(dropna = False))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:32:33.219199Z","iopub.execute_input":"2022-12-21T23:32:33.219935Z","iopub.status.idle":"2022-12-21T23:32:33.227674Z","shell.execute_reply.started":"2022-12-21T23:32:33.219898Z","shell.execute_reply":"2022-12-21T23:32:33.22663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"age.index.values","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:32:35.978629Z","iopub.execute_input":"2022-12-21T23:32:35.97904Z","iopub.status.idle":"2022-12-21T23:32:35.991913Z","shell.execute_reply.started":"2022-12-21T23:32:35.979008Z","shell.execute_reply":"2022-12-21T23:32:35.99067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"age_cancer = []\nfor i in range(len(age)):\n    s_bool_and = ((train_df['cancer'] == 1) & (train_df['age'] == age.index.values[i]))\n    age_cancer.append(s_bool_and.sum())","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:32:38.542345Z","iopub.execute_input":"2022-12-21T23:32:38.54274Z","iopub.status.idle":"2022-12-21T23:32:38.586214Z","shell.execute_reply.started":"2022-12-21T23:32:38.542689Z","shell.execute_reply":"2022-12-21T23:32:38.585285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(age.index.values,[i / j for i, j in zip(age_cancer, age.values.tolist())])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T23:32:40.741728Z","iopub.execute_input":"2022-12-21T23:32:40.742258Z","iopub.status.idle":"2022-12-21T23:32:40.973436Z","shell.execute_reply.started":"2022-12-21T23:32:40.742213Z","shell.execute_reply":"2022-12-21T23:32:40.972553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Cancer frequency increase with age.**","metadata":{}},{"cell_type":"markdown","source":"**Conclusion**\n1. Different machine have different cancer detection frequency.\n2. Not exactly the same as description, highest density \"D\" indicates lowest cancer frequency. However, lowest denstiy \"A\" not directly mean highest cancer frequency. If density is \"nan\", the cancer detection frequency is 0\n3. For view, MLO and CC have similar cancer detection frequency. Although AT and significantly higer frequency. Its number is small.\n4. Different laterality means trivial difference on cancer detection frequency\n5. Most patient with 6 or less images. The maximum is 14, the minimum is 4\n6. For different sites, the frequency is similar. so not informational to cancer prediction\n7. Populations'age distribution is normal. It is consistent with common sense. Cancer frequency increase with age.\n","metadata":{}}]}