{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"An attempt to see if the tabular data alone could be enough to get a \"proper score\". It doesn't seem like it is but could be a starting point for someone else I guess. :-)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.linear_model import Ridge\nfrom sklearn import svm\nimport matplotlib.pyplot as plt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-22T17:35:42.483805Z","iopub.execute_input":"2023-01-22T17:35:42.48462Z","iopub.status.idle":"2023-01-22T17:35:43.458649Z","shell.execute_reply.started":"2023-01-22T17:35:42.484511Z","shell.execute_reply":"2023-01-22T17:35:43.456773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv').fillna(50)\ndf_test = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/test.csv').fillna(50)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:35:49.339823Z","iopub.execute_input":"2023-01-22T17:35:49.340222Z","iopub.status.idle":"2023-01-22T17:35:49.463695Z","shell.execute_reply.started":"2023-01-22T17:35:49.340192Z","shell.execute_reply":"2023-01-22T17:35:49.462742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:35:52.3924Z","iopub.execute_input":"2023-01-22T17:35:52.392882Z","iopub.status.idle":"2023-01-22T17:35:52.430643Z","shell.execute_reply.started":"2023-01-22T17:35:52.392825Z","shell.execute_reply":"2023-01-22T17:35:52.429627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = df.loc[df['cancer'] == 1]['age'].tolist()\nplt.hist(bins, 50)\nplt.title('cancer age')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:35:58.1574Z","iopub.execute_input":"2023-01-22T17:35:58.157759Z","iopub.status.idle":"2023-01-22T17:35:58.563577Z","shell.execute_reply.started":"2023-01-22T17:35:58.157728Z","shell.execute_reply":"2023-01-22T17:35:58.562021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = df.loc[df['cancer'] == 0].loc[df['BIRADS'] != 0]['age'].tolist()\nplt.hist(bins, 70)\nplt.title('Age no cancer birads!=0[no need follow up]')","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:36:07.802916Z","iopub.execute_input":"2023-01-22T17:36:07.803468Z","iopub.status.idle":"2023-01-22T17:36:08.444804Z","shell.execute_reply.started":"2023-01-22T17:36:07.803428Z","shell.execute_reply":"2023-01-22T17:36:08.443801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins = df.loc[df['BIRADS'] == 2]['age'].tolist()\nplt.hist(bins, 50)\nplt.title('age  if the breast was rated as normal')","metadata":{"execution":{"iopub.status.busy":"2023-01-20T13:06:52.127385Z","iopub.execute_input":"2023-01-20T13:06:52.127974Z","iopub.status.idle":"2023-01-20T13:06:52.410034Z","shell.execute_reply.started":"2023-01-20T13:06:52.127939Z","shell.execute_reply":"2023-01-20T13:06:52.409174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n    site_id - ID code for the source hospital.\n    patient_id - ID code for the patient.\n    image_id - ID code for the image.\n    laterality - Whether the image is of the left or right breast.\n    view - The orientation of the image. The default for a screening exam is to capture two views per breast.\n    age - The patient's age in years.\n    implant - Whether or not the patient had breast implants. Site 1 only provides breast implant information at the patient level, not at the breast level.\n    density - A rating for how dense the breast tissue is, with A being the least dense and D being the most dense. Extremely dense tissue can make diagnosis more difficult. Only provided for train.\n    machine_id - An ID code for the imaging device.\n    cancer - Whether or not the breast was positive for malignant cancer. The target value. Only provided for train.\n    biopsy - Whether or not a follow-up biopsy was performed on the breast. Only provided for train.\n    invasive - If the breast is positive for cancer, whether or not the cancer proved to be invasive. Only provided for train.\n    BIRADS - 0 if the breast required follow-up, 1 if the breast was rated as negative for cancer, and 2 if the breast was rated as normal. Only provided for train.\n    prediction_id - The ID for the matching submission row. Multiple images will share the same prediction ID. Test only.\n    difficult_negative_case - True if the case was unusually difficult. Only provided for train.\n","metadata":{}},{"cell_type":"code","source":"df_train = df[:]\n\ncancer = df_train.loc[df_train['cancer'] == 1]\nnocancer = df_train.loc[df_train['cancer'] == 0].loc[df_train['BIRADS'] != 0]\n\ny_cancer = np.ones(len(cancer))\ny_nocancer = np.zeros(len(nocancer))","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:40:02.835173Z","iopub.execute_input":"2023-01-22T17:40:02.835543Z","iopub.status.idle":"2023-01-22T17:40:02.861768Z","shell.execute_reply.started":"2023-01-22T17:40:02.835511Z","shell.execute_reply":"2023-01-22T17:40:02.860883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer[:5]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:40:14.945016Z","iopub.execute_input":"2023-01-22T17:40:14.945563Z","iopub.status.idle":"2023-01-22T17:40:14.989247Z","shell.execute_reply.started":"2023-01-22T17:40:14.945522Z","shell.execute_reply":"2023-01-22T17:40:14.986888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(cancer),len(nocancer))","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:14.33802Z","iopub.execute_input":"2023-01-22T17:42:14.338389Z","iopub.status.idle":"2023-01-22T17:42:14.343983Z","shell.execute_reply.started":"2023-01-22T17:42:14.338358Z","shell.execute_reply":"2023-01-22T17:42:14.342874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## siteid","metadata":{}},{"cell_type":"code","source":"\ncancer_siteid = np.asarray(cancer['site_id'].tolist())\nnocancer_siteid = np.asarray(nocancer['site_id'].tolist())\nX_siteid = np.concatenate((cancer_siteid, nocancer_siteid), axis=0).reshape(cancer_siteid.shape[0] + nocancer_siteid.shape[0], 1) / 2\n","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:22.753315Z","iopub.execute_input":"2023-01-22T17:42:22.753668Z","iopub.status.idle":"2023-01-22T17:42:22.764185Z","shell.execute_reply.started":"2023-01-22T17:42:22.753637Z","shell.execute_reply":"2023-01-22T17:42:22.762995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer_siteid [1:50:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:32.161354Z","iopub.execute_input":"2023-01-22T17:42:32.161716Z","iopub.status.idle":"2023-01-22T17:42:32.169405Z","shell.execute_reply.started":"2023-01-22T17:42:32.161683Z","shell.execute_reply":"2023-01-22T17:42:32.168401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(cancer_siteid)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:35.210241Z","iopub.execute_input":"2023-01-22T17:42:35.2106Z","iopub.status.idle":"2023-01-22T17:42:35.217486Z","shell.execute_reply.started":"2023-01-22T17:42:35.210569Z","shell.execute_reply":"2023-01-22T17:42:35.216522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cancer_siteid.shape ","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:37.369681Z","iopub.execute_input":"2023-01-22T17:42:37.370375Z","iopub.status.idle":"2023-01-22T17:42:37.3768Z","shell.execute_reply.started":"2023-01-22T17:42:37.370337Z","shell.execute_reply":"2023-01-22T17:42:37.375729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nocancer_siteid.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:39.465405Z","iopub.execute_input":"2023-01-22T17:42:39.465765Z","iopub.status.idle":"2023-01-22T17:42:39.472203Z","shell.execute_reply.started":"2023-01-22T17:42:39.465731Z","shell.execute_reply":"2023-01-22T17:42:39.471177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_siteid.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:42.423907Z","iopub.execute_input":"2023-01-22T17:42:42.424272Z","iopub.status.idle":"2023-01-22T17:42:42.42964Z","shell.execute_reply.started":"2023-01-22T17:42:42.424241Z","shell.execute_reply":"2023-01-22T17:42:42.428433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_siteid[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:46.817903Z","iopub.execute_input":"2023-01-22T17:42:46.81861Z","iopub.status.idle":"2023-01-22T17:42:46.825523Z","shell.execute_reply.started":"2023-01-22T17:42:46.81857Z","shell.execute_reply":"2023-01-22T17:42:46.824429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cancer_siteid.shape[0] + nocancer_siteid.shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:49.594161Z","iopub.execute_input":"2023-01-22T17:42:49.594549Z","iopub.status.idle":"2023-01-22T17:42:49.600556Z","shell.execute_reply.started":"2023-01-22T17:42:49.594517Z","shell.execute_reply":"2023-01-22T17:42:49.599422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up view","metadata":{}},{"cell_type":"code","source":"X_view = np.zeros((X_siteid.shape[0], 6))\ni = 0\nfor view in cancer['view'].tolist() + nocancer['view'].tolist():\n    if view == 'CC':\n        X_view[i, 0] = 1\n    elif view == 'MLO':\n        X_view[i, 1] = 1\n    elif view == 'ML':\n        X_view[i, 2] = 1\n    elif view == 'LM':\n        X_view[i, 3] = 1\n    elif view == 'AT':\n        X_view[i, 4] = 1\n    elif view == 'LMO':\n        X_view[i, 5] = 1\n        \n    i += 1","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:54.305775Z","iopub.execute_input":"2023-01-22T17:42:54.306158Z","iopub.status.idle":"2023-01-22T17:42:54.331677Z","shell.execute_reply.started":"2023-01-22T17:42:54.306123Z","shell.execute_reply":"2023-01-22T17:42:54.330671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_view[1:10,:]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:42:59.089079Z","iopub.execute_input":"2023-01-22T17:42:59.089566Z","iopub.status.idle":"2023-01-22T17:42:59.098575Z","shell.execute_reply.started":"2023-01-22T17:42:59.089519Z","shell.execute_reply":"2023-01-22T17:42:59.097362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up age","metadata":{}},{"cell_type":"code","source":"        \ncancer_age = np.asarray(cancer['age'].tolist())\nnocancer_age = np.asarray(nocancer['age'].tolist())\nX_age = np.concatenate((cancer_age, nocancer_age), axis=0).reshape(cancer_siteid.shape[0] + nocancer_siteid.shape[0], 1) / 100\nX_age[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:43:14.18711Z","iopub.execute_input":"2023-01-22T17:43:14.187475Z","iopub.status.idle":"2023-01-22T17:43:14.200882Z","shell.execute_reply.started":"2023-01-22T17:43:14.187442Z","shell.execute_reply":"2023-01-22T17:43:14.1999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Implant","metadata":{}},{"cell_type":"code","source":"\ncancer_implant = np.asarray(cancer['implant'].tolist())\nnocancer_implant = np.asarray(nocancer['implant'].tolist())\nX_implant = np.concatenate((cancer_implant, nocancer_implant), axis=0).reshape(cancer_siteid.shape[0] + nocancer_siteid.shape[0], 1)\nX_implant[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:43:26.78998Z","iopub.execute_input":"2023-01-22T17:43:26.790377Z","iopub.status.idle":"2023-01-22T17:43:26.80452Z","shell.execute_reply.started":"2023-01-22T17:43:26.790343Z","shell.execute_reply":"2023-01-22T17:43:26.803292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(X_implant)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:43:49.72376Z","iopub.execute_input":"2023-01-22T17:43:49.724728Z","iopub.status.idle":"2023-01-22T17:43:49.731943Z","shell.execute_reply.started":"2023-01-22T17:43:49.72469Z","shell.execute_reply":"2023-01-22T17:43:49.730643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Machine id","metadata":{}},{"cell_type":"code","source":"cancer_machineid = np.asarray(cancer['machine_id'].tolist())\nnocancer_machineid = np.asarray(nocancer['machine_id'].tolist())\nX_machineid = np.concatenate((cancer_machineid, nocancer_machineid), axis=0).reshape(cancer_siteid.shape[0] + nocancer_siteid.shape[0], 1) / 200\nnp.unique(X_machineid)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:43:53.845205Z","iopub.execute_input":"2023-01-22T17:43:53.845598Z","iopub.status.idle":"2023-01-22T17:43:53.860275Z","shell.execute_reply.started":"2023-01-22T17:43:53.845567Z","shell.execute_reply":"2023-01-22T17:43:53.859121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build X,y input svm","metadata":{}},{"cell_type":"code","source":"\nX = np.concatenate((X_siteid, X_view, X_age, X_implant, X_machineid), axis=1)\ny = np.concatenate((y_cancer, y_nocancer), axis=0)\nprint(X.shape, y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:44:06.244474Z","iopub.execute_input":"2023-01-22T17:44:06.244828Z","iopub.status.idle":"2023-01-22T17:44:06.2554Z","shell.execute_reply.started":"2023-01-22T17:44:06.244797Z","shell.execute_reply":"2023-01-22T17:44:06.254183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Fit SVm","metadata":{}},{"cell_type":"code","source":"clf = svm.NuSVC(nu=0.04, gamma=\"auto\", class_weight=\"balanced\", kernel=\"rbf\")\nclf.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:44:32.112127Z","iopub.execute_input":"2023-01-22T17:44:32.113076Z","iopub.status.idle":"2023-01-22T17:44:37.716525Z","shell.execute_reply.started":"2023-01-22T17:44:32.113037Z","shell.execute_reply":"2023-01-22T17:44:37.715579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#clf = Ridge(alpha=1.0)\n#clf.fit(X, y)\n\nnum_tests = len(df_test.index)\n\nX_siteid = np.asarray(df_test['site_id'].tolist()).reshape(num_tests, 1) / 2\n\nX_view = np.zeros((num_tests, 6))\ni = 0\nfor view in df_test['view'].tolist():\n    if view == 'CC':\n        X_view[i, 0] = 1\n    elif view == 'MLO':\n        X_view[i, 1] = 1\n    elif view == 'ML':\n        X_view[i, 2] = 1\n    elif view == 'LM':\n        X_view[i, 3] = 1\n    elif view == 'AT':\n        X_view[i, 4] = 1\n    elif view == 'LMO':\n        X_view[i, 5] = 1\n        \n    i += 1\n        \nX_age = np.asarray(df_test['age'].tolist()).reshape(num_tests, 1) / 100\nX_implant = np.asarray(df_test['implant'].tolist()).reshape(num_tests, 1)\nX_machineid = np.asarray(df_test['machine_id'].tolist()).reshape(num_tests, 1) / 200\n\nX = np.concatenate((X_siteid, X_view, X_age, X_implant, X_machineid), axis=1)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:45:26.412269Z","iopub.execute_input":"2023-01-22T17:45:26.412632Z","iopub.status.idle":"2023-01-22T17:45:26.424776Z","shell.execute_reply.started":"2023-01-22T17:45:26.4126Z","shell.execute_reply":"2023-01-22T17:45:26.423884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submission","metadata":{}},{"cell_type":"code","source":"df_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:46:36.75421Z","iopub.execute_input":"2023-01-22T17:46:36.754801Z","iopub.status.idle":"2023-01-22T17:46:36.761597Z","shell.execute_reply.started":"2023-01-22T17:46:36.754757Z","shell.execute_reply":"2023-01-22T17:46:36.760607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame()\n\nsubmission_df['prediction_id'] = df_test['prediction_id']\nsubmission_df['cancer'] = clf.predict(X)\n\nsubmission_df = submission_df.groupby('prediction_id').max()\nsubmission_df['prediction_id'] = submission_df.index    #print(pre_submission_df)\n\n#submission_df['cancer'].where(submission_df['cancer'] > THRESHOLD, 1, inplace=True)\n#submission_df['cancer'].where(submission_df['cancer'] < THRESHOLD, 0, inplace=True)\n\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:45:44.90797Z","iopub.execute_input":"2023-01-22T17:45:44.908592Z","iopub.status.idle":"2023-01-22T17:45:44.92734Z","shell.execute_reply.started":"2023-01-22T17:45:44.908553Z","shell.execute_reply":"2023-01-22T17:45:44.926395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:46:13.461306Z","iopub.execute_input":"2023-01-22T17:46:13.461863Z","iopub.status.idle":"2023-01-22T17:46:13.472245Z","shell.execute_reply.started":"2023-01-22T17:46:13.461747Z","shell.execute_reply":"2023-01-22T17:46:13.469646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-22T17:45:59.528616Z","iopub.execute_input":"2023-01-22T17:45:59.529002Z","iopub.status.idle":"2023-01-22T17:45:59.540457Z","shell.execute_reply.started":"2023-01-22T17:45:59.528968Z","shell.execute_reply":"2023-01-22T17:45:59.539327Z"},"trusted":true},"execution_count":null,"outputs":[]}]}