{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# import Libraries and dataset","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nimport pydicom\nimport glob\nfrom scipy import stats\n\nos.chdir('/kaggle/input/rsna-breast-cancer-detection')\n\ntrain_dir = './train_images'\ntrain_df = pd.read_csv('train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T07:23:03.500782Z","iopub.execute_input":"2022-12-28T07:23:03.501199Z","iopub.status.idle":"2022-12-28T07:23:03.599152Z","shell.execute_reply.started":"2022-12-28T07:23:03.501158Z","shell.execute_reply":"2022-12-28T07:23:03.597893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train.csv 내용\")\ndisplay(train_df.head(3))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.371786Z","iopub.execute_input":"2022-12-28T05:59:32.3729Z","iopub.status.idle":"2022-12-28T05:59:32.403843Z","shell.execute_reply.started":"2022-12-28T05:59:32.372852Z","shell.execute_reply":"2022-12-28T05:59:32.402672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 데이터 설명\nsite_id - 데이터의 출처 병원 id\\\npatient_id - 환자 id\\\nimage_id - 이미지 id\\\nlaterality - 이미지 속 breast가 어느 쪽인지. 좌/우 \\\nview - The orientation of the image. 한 breast당 2가지 capture가 default값이다. \\\nage - 환자 나이\\\nimplant - 보형물의 여부.\\\ndensity - 유방조직의 밀도. 가장 밀도 높은 A ~ 낮은 D. 밀도가 너무 높으면 검사 어려움. train에만 존재하는 데이터.\\\nmachine_id - 이미지관련 기계의 id \\\ncancer - 악성암 여부. train에만 존재하는 데이터 \\\nbiopsy -  biopsy가 breast에서 시행 여부. Only provided for train.\\\ninvasive - cancer이 양성인 경우, invasive 판명됐는지? Only provided for train.\\\nBIRADS - 0 지속적인 진료 필요, 1 cancer가 음성인 경우, and 2 breast가 정상인 경우. Only provided for train.\\\nprediction_id - The ID for the matching submission row. Multiple images will share the same prediction ID. Test only.\\\ndifficult_negative_case - True, case가 unusual하게 어려운 경우. Only provided for train.","metadata":{}},{"cell_type":"markdown","source":"### train.csv에만 존재하는 컬럼 \ndensity \\\nbiopsy \\\nBIRADS \\\ndifficult_negative_case\ncancer \n\n### test.csv에만 존재하는 컬럼\nprediction_id","metadata":{}},{"cell_type":"markdown","source":"# NaN값 확인 Missing Values","metadata":{}},{"cell_type":"markdown","source":"- age 에서 37개\n- BIRADS 에서 28420개\n- density 에서 25236개의 missing values를 확인","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.405488Z","iopub.execute_input":"2022-12-28T05:59:32.405846Z","iopub.status.idle":"2022-12-28T05:59:32.424749Z","shell.execute_reply.started":"2022-12-28T05:59:32.405812Z","shell.execute_reply":"2022-12-28T05:59:32.423476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nan값 제외한 데이터 \ntrain_drop = train_df.dropna()\nprint(\"원래 train.csv의 총 데이터 갯수는 \",len(train_df),\"개인데 dropna한 뒤로 \",len(train_drop),\"개가 됐다.\\nNaN값이 같은 행에 있지 않았나보다.\")","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.427809Z","iopub.execute_input":"2022-12-28T05:59:32.428485Z","iopub.status.idle":"2022-12-28T05:59:32.454912Z","shell.execute_reply.started":"2022-12-28T05:59:32.428449Z","shell.execute_reply":"2022-12-28T05:59:32.453993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.site_id.unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.456219Z","iopub.execute_input":"2022-12-28T05:59:32.456557Z","iopub.status.idle":"2022-12-28T05:59:32.471024Z","shell.execute_reply.started":"2022-12-28T05:59:32.456524Z","shell.execute_reply":"2022-12-28T05:59:32.469792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\n    \"site_1에서의 환자별 이미지 데이터 갯수: \",\n    train_df[train_df['site_id']==1].groupby('patient_id').image_id.count().unique(),\n    \"\\nsite_2에서의 환자별 이미지 데이터 갯수: \",\n    train_df[train_df['site_id']==2].groupby('patient_id').image_id.count().unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.473734Z","iopub.execute_input":"2022-12-28T05:59:32.474368Z","iopub.status.idle":"2022-12-28T05:59:32.496485Z","shell.execute_reply.started":"2022-12-28T05:59:32.47432Z","shell.execute_reply":"2022-12-28T05:59:32.495023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"train image data와 train.csv 내 patient_id는 각각 모두 정보가 있다.\")\nprint(\"train.csv의 환자 id 갯수: \",len(train_df.patient_id.unique()), \n      \", train image의 환자 id 갯수: \",len(os.listdir(train_dir)))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.498281Z","iopub.execute_input":"2022-12-28T05:59:32.498972Z","iopub.status.idle":"2022-12-28T05:59:32.649821Z","shell.execute_reply.started":"2022-12-28T05:59:32.498925Z","shell.execute_reply":"2022-12-28T05:59:32.648261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer값에 따른 사진 갯수의 차이\n각 환자별 x-ray 사진 갯수가 다르다. \\\n양성인 경우 5개이하의 이미지 데이터를 가지고 있고 음성인 경우 2개 ~ 14개로 다양하다.","metadata":{}},{"cell_type":"code","source":"print(\"각 환자별 이미지 갯수 : \",train_df.groupby('patient_id').image_id.size().unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.651388Z","iopub.execute_input":"2022-12-28T05:59:32.652671Z","iopub.status.idle":"2022-12-28T05:59:32.664506Z","shell.execute_reply.started":"2022-12-28T05:59:32.652621Z","shell.execute_reply":"2022-12-28T05:59:32.663373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"음성인 경우 이미지 갯수 : \",train_df[train_df['cancer']==0].groupby('patient_id').image_id.size().unique())\nprint(\"양성인 경우 이미지 갯수 : \",train_df[train_df['cancer']==1].groupby('patient_id').image_id.size().unique())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.666011Z","iopub.execute_input":"2022-12-28T05:59:32.667458Z","iopub.status.idle":"2022-12-28T05:59:32.68922Z","shell.execute_reply.started":"2022-12-28T05:59:32.667411Z","shell.execute_reply":"2022-12-28T05:59:32.688004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"각 환자별 사진 데이터 갯수","metadata":{}},{"cell_type":"code","source":"train_df[train_df.cancer==0]['patient_id'].value_counts().value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.695268Z","iopub.execute_input":"2022-12-28T05:59:32.695621Z","iopub.status.idle":"2022-12-28T05:59:32.717096Z","shell.execute_reply.started":"2022-12-28T05:59:32.695587Z","shell.execute_reply":"2022-12-28T05:59:32.715538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.cancer==1]['patient_id'].value_counts().value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.719375Z","iopub.execute_input":"2022-12-28T05:59:32.720259Z","iopub.status.idle":"2022-12-28T05:59:32.732527Z","shell.execute_reply.started":"2022-12-28T05:59:32.720208Z","shell.execute_reply":"2022-12-28T05:59:32.731252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CountStatus = pd.value_counts(train_df[train_df.cancer==1]['patient_id'].value_counts(), sort=True)\nCountStatus.plot.bar()\nCountStatus.plot.bar(grid=True, figsize=(10,8), fontsize=15)  # figsize, fontsize 조정\n\n# percentage = []\n# for i in range(data.shape[0]):\n#     pct = (data.Runs[i] / total_runs) * 100\n#     percentage.append(round(pct,2))\n# data['Percentage'] = percentage\n\n# for p in graph:\n#     width = p.get_width()\n#     height = p.get_height()\n#     x, y = p.get_xy()\n#     plt.text(x+width/2,\n#              y+height*1.01,\n#              str(data.Percentage[i])+'%',\n#              ha='center',\n#              weight='bold')\n#     i+=1\n\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:32.734212Z","iopub.execute_input":"2022-12-28T05:59:32.734776Z","iopub.status.idle":"2022-12-28T05:59:33.046111Z","shell.execute_reply.started":"2022-12-28T05:59:32.734714Z","shell.execute_reply":"2022-12-28T05:59:33.044813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cc_info = dict()\n# cc_idx=0\n# cc_per=0\nfor i in range(len(train_df.cancer.value_counts())):\n    cc_idx=train_df.cancer.value_counts().index[i]\n    cc_per=train_df.cancer.value_counts().values[i]/train_df.cancer.value_counts().values.sum()*100\n    cc_info[cc_idx]= cc_per","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:33.048217Z","iopub.execute_input":"2022-12-28T05:59:33.048712Z","iopub.status.idle":"2022-12-28T05:59:33.064342Z","shell.execute_reply.started":"2022-12-28T05:59:33.048663Z","shell.execute_reply":"2022-12-28T05:59:33.062914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cc_info","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:33.066077Z","iopub.execute_input":"2022-12-28T05:59:33.067238Z","iopub.status.idle":"2022-12-28T05:59:33.080168Z","shell.execute_reply.started":"2022-12-28T05:59:33.067189Z","shell.execute_reply":"2022-12-28T05:59:33.078542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tTof=[]\ni=0\nfor d in train_df[train_df.cancer==0].patient_id.unique():\n    if d in train_df[train_df.cancer==1].patient_id.unique():\n        tTof.append(d)\n    else:\n        i+=1\n\n        \nfTot=[]\nj=0\nfor d in train_df[train_df.cancer==1].patient_id.unique():\n    if d in train_df[train_df.cancer==0].patient_id.unique():\n        fTot.append(d)\n    else:\n        j+=1\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:33.082347Z","iopub.execute_input":"2022-12-28T05:59:33.082863Z","iopub.status.idle":"2022-12-28T05:59:46.795699Z","shell.execute_reply.started":"2022-12-28T05:59:33.082813Z","shell.execute_reply":"2022-12-28T05:59:46.79423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(tTof),len(fTot),i,j)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.797095Z","iopub.execute_input":"2022-12-28T05:59:46.797525Z","iopub.status.idle":"2022-12-28T05:59:46.806123Z","shell.execute_reply.started":"2022-12-28T05:59:46.797472Z","shell.execute_reply":"2022-12-28T05:59:46.804374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.patient_id==tTof[0]]","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.808246Z","iopub.execute_input":"2022-12-28T05:59:46.808791Z","iopub.status.idle":"2022-12-28T05:59:46.838228Z","shell.execute_reply.started":"2022-12-28T05:59:46.808715Z","shell.execute_reply":"2022-12-28T05:59:46.837074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df.patient_id==fTot[0]]","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.839557Z","iopub.execute_input":"2022-12-28T05:59:46.839924Z","iopub.status.idle":"2022-12-28T05:59:46.864781Z","shell.execute_reply.started":"2022-12-28T05:59:46.839892Z","shell.execute_reply":"2022-12-28T05:59:46.863661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.laterality.unique()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.866455Z","iopub.execute_input":"2022-12-28T05:59:46.866838Z","iopub.status.idle":"2022-12-28T05:59:46.88159Z","shell.execute_reply.started":"2022-12-28T05:59:46.866799Z","shell.execute_reply":"2022-12-28T05:59:46.880358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.883337Z","iopub.execute_input":"2022-12-28T05:59:46.883679Z","iopub.status.idle":"2022-12-28T05:59:46.909431Z","shell.execute_reply.started":"2022-12-28T05:59:46.88365Z","shell.execute_reply":"2022-12-28T05:59:46.908166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train_drop.info()","metadata":{"execution":{"iopub.status.busy":"2022-12-26T05:55:05.085516Z","iopub.execute_input":"2022-12-26T05:55:05.086255Z","iopub.status.idle":"2022-12-26T05:55:05.110001Z","shell.execute_reply.started":"2022-12-26T05:55:05.086206Z","shell.execute_reply":"2022-12-26T05:55:05.109155Z"}}},{"cell_type":"code","source":"print(\"정규화나 표준화를 하지않은 상태에서 기술통계량. age의 값이 확연히 달라 정규화나 표준화의 필요성이 느껴진다.\")\n\ntrain_df.iloc[:,3:-2].describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.91092Z","iopub.execute_input":"2022-12-28T05:59:46.911236Z","iopub.status.idle":"2022-12-28T05:59:46.962199Z","shell.execute_reply.started":"2022-12-28T05:59:46.911208Z","shell.execute_reply":"2022-12-28T05:59:46.961014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer +/- count \n- cancer은 Missing value가 없다.\n- 양성인 경우가 음성인 경우보다 많이 적다.","metadata":{}},{"cell_type":"code","source":"print(train_df.groupby('cancer').size())\nplt.figure(figsize=(5,3))\nsns.countplot(data=train_df, x='cancer')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:46.963927Z","iopub.execute_input":"2022-12-28T05:59:46.964278Z","iopub.status.idle":"2022-12-28T05:59:47.085492Z","shell.execute_reply.started":"2022-12-28T05:59:46.964237Z","shell.execute_reply":"2022-12-28T05:59:47.084297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_df.cancer.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.087196Z","iopub.execute_input":"2022-12-28T05:59:47.087543Z","iopub.status.idle":"2022-12-28T05:59:47.101574Z","shell.execute_reply.started":"2022-12-28T05:59:47.08751Z","shell.execute_reply":"2022-12-28T05:59:47.099688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cc_info = dict()\n# cc_idx=0\n# cc_per=0\nfor i in range(len(train_df.cancer.value_counts())):\n    cc_idx=train_df.cancer.value_counts().index[i]\n    cc_per=train_df.cancer.value_counts().values[i]/train_df.cancer.value_counts().values.sum()*100\n    cc_info[cc_idx]= cc_per\n    \nprint(\"cancer의 양/음성 비율은, \", cc_info)  ","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.105029Z","iopub.execute_input":"2022-12-28T05:59:47.106179Z","iopub.status.idle":"2022-12-28T05:59:47.1322Z","shell.execute_reply.started":"2022-12-28T05:59:47.106111Z","shell.execute_reply":"2022-12-28T05:59:47.131044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_df.groupby('biopsy').size())\nplt.figure(figsize=(5,3))\nsns.countplot(data=train_df, x='biopsy')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.133522Z","iopub.execute_input":"2022-12-28T05:59:47.133878Z","iopub.status.idle":"2022-12-28T05:59:47.304824Z","shell.execute_reply.started":"2022-12-28T05:59:47.133844Z","shell.execute_reply":"2022-12-28T05:59:47.303655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer and implant\n보형물을 착용하지 않은 사람 중 양성인 경우가 있다.\\\n아래 통계를 보면 양성이면서 보형물을 가진 사람이 0은 아니다.\n","metadata":{}},{"cell_type":"code","source":"print(train_drop.groupby(['implant','cancer']).patient_id.size())\nplt.figure(figsize=(5,3))\nsns.countplot(data=train_drop, x='implant', hue='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.306192Z","iopub.execute_input":"2022-12-28T05:59:47.306532Z","iopub.status.idle":"2022-12-28T05:59:47.516596Z","shell.execute_reply.started":"2022-12-28T05:59:47.3065Z","shell.execute_reply":"2022-12-28T05:59:47.515312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### cancer and BIRADS ","metadata":{}},{"cell_type":"code","source":"print(\"양성인 경우만 0. 지속적인 진료가 필요하다.\")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_drop, x='BIRADS', hue='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.518103Z","iopub.execute_input":"2022-12-28T05:59:47.518457Z","iopub.status.idle":"2022-12-28T05:59:47.734613Z","shell.execute_reply.started":"2022-12-28T05:59:47.518424Z","shell.execute_reply":"2022-12-28T05:59:47.733454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer and invasive","metadata":{}},{"cell_type":"code","source":"print(\"음성인 경우 invasive가 양성으로 나온적은 없다.\")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_drop, x='invasive', hue='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.741283Z","iopub.execute_input":"2022-12-28T05:59:47.741652Z","iopub.status.idle":"2022-12-28T05:59:47.938411Z","shell.execute_reply.started":"2022-12-28T05:59:47.741617Z","shell.execute_reply":"2022-12-28T05:59:47.937168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby(['invasive','cancer']).size()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.939798Z","iopub.execute_input":"2022-12-28T05:59:47.940738Z","iopub.status.idle":"2022-12-28T05:59:47.957172Z","shell.execute_reply.started":"2022-12-28T05:59:47.940702Z","shell.execute_reply":"2022-12-28T05:59:47.956047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"cancer이 양성인 경우- invasive인지 아닌지 \")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_drop[train_drop['cancer']==1], x='invasive', color='orange')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:47.958406Z","iopub.execute_input":"2022-12-28T05:59:47.958725Z","iopub.status.idle":"2022-12-28T05:59:48.123776Z","shell.execute_reply.started":"2022-12-28T05:59:47.958697Z","shell.execute_reply":"2022-12-28T05:59:48.122544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer and Age","metadata":{}},{"cell_type":"markdown","source":"- cancer이 음성인 경우가 압도적으로 많아서 양성인 경우가 작게 보인다\n- 그래도 육안으로 보면 50 ~75세 사이 비교적 더 많이 분포된 것으로 확인된다.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8,4))\nsns.histplot(data=train_drop, x='age', hue='cancer')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:48.124904Z","iopub.execute_input":"2022-12-28T05:59:48.125267Z","iopub.status.idle":"2022-12-28T05:59:48.604861Z","shell.execute_reply.started":"2022-12-28T05:59:48.125233Z","shell.execute_reply":"2022-12-28T05:59:48.603524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### cancer 과 view","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6,3))\nsns.countplot(data=train_df, x='view')\nplt.show()\nprint(train_df.view.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:48.606515Z","iopub.execute_input":"2022-12-28T05:59:48.607699Z","iopub.status.idle":"2022-12-28T05:59:48.845004Z","shell.execute_reply.started":"2022-12-28T05:59:48.607658Z","shell.execute_reply":"2022-12-28T05:59:48.843815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.groupby(['patient_id','view']).size()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:48.84621Z","iopub.execute_input":"2022-12-28T05:59:48.846538Z","iopub.status.idle":"2022-12-28T05:59:48.871971Z","shell.execute_reply.started":"2022-12-28T05:59:48.846509Z","shell.execute_reply":"2022-12-28T05:59:48.870698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"위 그래프에서 거의 보이지 않는 view의 값들을 자세히 그려보았다.","metadata":{}},{"cell_type":"markdown","source":"찾아보니, 'ML','LM','AT','LMO'는 보충 view 라고 한다.","metadata":{}},{"cell_type":"code","source":"train_view=train_df[train_df.view.isin(['ML','LM','AT','LMO'])]\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_view, x='view')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:48.873726Z","iopub.execute_input":"2022-12-28T05:59:48.874089Z","iopub.status.idle":"2022-12-28T05:59:49.068336Z","shell.execute_reply.started":"2022-12-28T05:59:48.874056Z","shell.execute_reply":"2022-12-28T05:59:49.067444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"cancer 양성인 경우, ML, LM, LMO는 존재하지 않았다.\")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_df[train_df['cancer']==1], x='view')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.069785Z","iopub.execute_input":"2022-12-28T05:59:49.070374Z","iopub.status.idle":"2022-12-28T05:59:49.252974Z","shell.execute_reply.started":"2022-12-28T05:59:49.070338Z","shell.execute_reply":"2022-12-28T05:59:49.251842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"음성인 경우 전체 데이터와 그래프가 비슷하다.\")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_df[train_df['cancer']==0], x='view')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.254479Z","iopub.execute_input":"2022-12-28T05:59:49.254997Z","iopub.status.idle":"2022-12-28T05:59:49.496Z","shell.execute_reply.started":"2022-12-28T05:59:49.25495Z","shell.execute_reply":"2022-12-28T05:59:49.494984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v_cm = train_df[train_df.view.isin(['CC','MLO'])]","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.497346Z","iopub.execute_input":"2022-12-28T05:59:49.497693Z","iopub.status.idle":"2022-12-28T05:59:49.515731Z","shell.execute_reply.started":"2022-12-28T05:59:49.497659Z","shell.execute_reply":"2022-12-28T05:59:49.514849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.517219Z","iopub.execute_input":"2022-12-28T05:59:49.52126Z","iopub.status.idle":"2022-12-28T05:59:49.530332Z","shell.execute_reply.started":"2022-12-28T05:59:49.521219Z","shell.execute_reply":"2022-12-28T05:59:49.529199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_v_cm.cancer.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.531904Z","iopub.execute_input":"2022-12-28T05:59:49.532373Z","iopub.status.idle":"2022-12-28T05:59:49.547592Z","shell.execute_reply.started":"2022-12-28T05:59:49.532339Z","shell.execute_reply":"2022-12-28T05:59:49.54636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# cancer and density\n유방조직의 밀도 \\\n가장 밀도 높은 A ~ 낮은 D.\nnan값 있음","metadata":{}},{"cell_type":"code","source":"print(\"nan값 제외한 density value count\")\nprint(train_drop.density.value_counts())\nprint(\"-\"*40)\nprint(\"cancer 양성인 경우 density value count\")\nprint(train_drop[train_drop['cancer']==1].density.value_counts())\nprint(\"-\"*40)\nprint(\"cancer 음성인 경우 density value count\")\nprint(train_drop[train_drop['cancer']==0].density.value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.549291Z","iopub.execute_input":"2022-12-28T05:59:49.549674Z","iopub.status.idle":"2022-12-28T05:59:49.571381Z","shell.execute_reply.started":"2022-12-28T05:59:49.549636Z","shell.execute_reply":"2022-12-28T05:59:49.570229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.density.unique()\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_drop, x='density', order=['B','C','A','D'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.57302Z","iopub.execute_input":"2022-12-28T05:59:49.573914Z","iopub.status.idle":"2022-12-28T05:59:49.765037Z","shell.execute_reply.started":"2022-12-28T05:59:49.573864Z","shell.execute_reply":"2022-12-28T05:59:49.764182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"cancer positive인 경우\")\nplt.figure(figsize=(6,3))\nsns.countplot(data=train_drop[train_drop['cancer']==1], x='density', order=['B','C','A','D'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.766081Z","iopub.execute_input":"2022-12-28T05:59:49.766948Z","iopub.status.idle":"2022-12-28T05:59:49.948459Z","shell.execute_reply.started":"2022-12-28T05:59:49.766912Z","shell.execute_reply":"2022-12-28T05:59:49.947287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# laterality, view, density 카테고리화","metadata":{}},{"cell_type":"code","source":"train_sc = train_drop[['laterality','view','age','biopsy','BIRADS','implant','density','cancer']]\n\n\ntrain_sc['laterality']=train_sc.loc[:,'laterality'].replace(['L','R'],[0,1])\ntrain_sc['view']=train_sc.loc[:,'view'].replace(['MLO', 'CC', 'ML', 'LM', 'AT', 'LMO'],[0,1,2,3,4,5])\ntrain_sc['density']=train_sc.loc[:,'density'].replace(['C', 'D', 'B', 'A'],[0,1,2,3])","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.94971Z","iopub.execute_input":"2022-12-28T05:59:49.950098Z","iopub.status.idle":"2022-12-28T05:59:49.998228Z","shell.execute_reply.started":"2022-12-28T05:59:49.950064Z","shell.execute_reply":"2022-12-28T05:59:49.996916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"표준화한 값으로 상관관계 구하기","metadata":{}},{"cell_type":"code","source":"# 표준화\n# train_sc = (train_sc - train_sc.mean())/train_sc.std()\n# train_sc = pd.DataFrame(train_sc, columns=train_sc.columns, index=list(train_sc.index.values))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:49.999624Z","iopub.execute_input":"2022-12-28T05:59:49.999997Z","iopub.status.idle":"2022-12-28T05:59:50.004436Z","shell.execute_reply.started":"2022-12-28T05:59:49.999962Z","shell.execute_reply":"2022-12-28T05:59:50.0036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\n\ntrain_sc_col = train_sc.columns\ntrain_sc_idx = train_sc.index.values\nscaler = MinMaxScaler()\ntrain_sc = scaler.fit_transform(train_sc)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.005735Z","iopub.execute_input":"2022-12-28T05:59:50.006836Z","iopub.status.idle":"2022-12-28T05:59:50.225679Z","shell.execute_reply.started":"2022-12-28T05:59:50.006801Z","shell.execute_reply":"2022-12-28T05:59:50.224491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sc = pd.DataFrame(train_sc, columns=train_sc_col, index=list(train_sc_idx))","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.227025Z","iopub.execute_input":"2022-12-28T05:59:50.227371Z","iopub.status.idle":"2022-12-28T05:59:50.24302Z","shell.execute_reply.started":"2022-12-28T05:59:50.227339Z","shell.execute_reply":"2022-12-28T05:59:50.241819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_sc","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.244308Z","iopub.execute_input":"2022-12-28T05:59:50.244646Z","iopub.status.idle":"2022-12-28T05:59:50.272752Z","shell.execute_reply.started":"2022-12-28T05:59:50.244616Z","shell.execute_reply":"2022-12-28T05:59:50.271454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_drop[train_drop.cancer==0].age.describe())\nprint(\"-\"*40)\nprint(train_drop[train_drop.cancer==1].age.describe())\nprint(\"결론\",\"-\"*30)\nprint(\"cancer가 양성인 경우 median값이 더 높다 (56, 65)\")","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.274057Z","iopub.execute_input":"2022-12-28T05:59:50.274425Z","iopub.status.idle":"2022-12-28T05:59:50.294805Z","shell.execute_reply.started":"2022-12-28T05:59:50.274394Z","shell.execute_reply":"2022-12-28T05:59:50.293644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 변수간의 상관관계","metadata":{}},{"cell_type":"code","source":"# train_sc.cov()\n# train_sc.corrwith(train_sc['cancer'], method='spearman')\ntrain_sc.corr(method='pearson')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.2964Z","iopub.execute_input":"2022-12-28T05:59:50.297194Z","iopub.status.idle":"2022-12-28T05:59:50.321713Z","shell.execute_reply.started":"2022-12-28T05:59:50.297147Z","shell.execute_reply":"2022-12-28T05:59:50.320652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"상관계수 값이 0.1로 0에 가까워 음의 상관관계이며 p값이 0.05보다 적어서 귀무가설을 기각하게된다.\\\n귀무가설(null hypothesis)은 우리가 증명하고자 하는 가설의 반대되는 가설, 효과와 차이가 없는 가설","metadata":{}},{"cell_type":"code","source":"stats.spearmanr(train_sc[\"cancer\"], train_sc[\"age\"], nan_policy=\"omit\")","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.323008Z","iopub.execute_input":"2022-12-28T05:59:50.323371Z","iopub.status.idle":"2022-12-28T05:59:50.339027Z","shell.execute_reply.started":"2022-12-28T05:59:50.323334Z","shell.execute_reply":"2022-12-28T05:59:50.337833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = np.zeros_like(train_sc.corr())\nmask[np.triu_indices_from(mask)] = True\nplt.rcParams[\"figure.figsize\"] = (10,6)\nsns.heatmap(data = train_sc.corr(), mask=mask, annot=True, fmt = '.3f', linewidths=0, cmap='Greens')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.340826Z","iopub.execute_input":"2022-12-28T05:59:50.341341Z","iopub.status.idle":"2022-12-28T05:59:50.749055Z","shell.execute_reply.started":"2022-12-28T05:59:50.341295Z","shell.execute_reply":"2022-12-28T05:59:50.747927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test 데이터 확인","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('./test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.751017Z","iopub.execute_input":"2022-12-28T05:59:50.751656Z","iopub.status.idle":"2022-12-28T05:59:50.7615Z","shell.execute_reply.started":"2022-12-28T05:59:50.751618Z","shell.execute_reply":"2022-12-28T05:59:50.760456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.762828Z","iopub.execute_input":"2022-12-28T05:59:50.763812Z","iopub.status.idle":"2022-12-28T05:59:50.780946Z","shell.execute_reply.started":"2022-12-28T05:59:50.763741Z","shell.execute_reply":"2022-12-28T05:59:50.779568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test.csv 파일에는 결측값 nan이 없다. \\\nmissing value 없음!","metadata":{}},{"cell_type":"code","source":"test.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.782581Z","iopub.execute_input":"2022-12-28T05:59:50.783503Z","iopub.status.idle":"2022-12-28T05:59:50.792509Z","shell.execute_reply.started":"2022-12-28T05:59:50.783466Z","shell.execute_reply":"2022-12-28T05:59:50.791264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.view.unique() # train.csv를 사용할 때 고려해야하나?","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:50.7942Z","iopub.execute_input":"2022-12-28T05:59:50.794525Z","iopub.status.idle":"2022-12-28T05:59:50.803315Z","shell.execute_reply.started":"2022-12-28T05:59:50.794496Z","shell.execute_reply":"2022-12-28T05:59:50.802181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 임시 제출용 폼 만들기","metadata":{}},{"cell_type":"code","source":"sub_df=test.copy()\nsub_df['cancer']=0","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:51.634554Z","iopub.status.idle":"2022-12-28T05:59:51.635439Z","shell.execute_reply.started":"2022-12-28T05:59:51.635211Z","shell.execute_reply":"2022-12-28T05:59:51.635235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:51.637254Z","iopub.status.idle":"2022-12-28T05:59:51.638181Z","shell.execute_reply.started":"2022-12-28T05:59:51.637964Z","shell.execute_reply":"2022-12-28T05:59:51.637986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df[[\"prediction_id\",\"cancer\"]].drop_duplicates(subset=\"prediction_id\")","metadata":{"execution":{"iopub.status.busy":"2022-12-28T05:59:51.63923Z","iopub.status.idle":"2022-12-28T05:59:51.640091Z","shell.execute_reply.started":"2022-12-28T05:59:51.639873Z","shell.execute_reply":"2022-12-28T05:59:51.639895Z"},"trusted":true},"execution_count":null,"outputs":[]}]}