{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%load_ext autoreload \n%autoreload 2","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:41:44.509591Z","iopub.execute_input":"2022-12-28T14:41:44.510148Z","iopub.status.idle":"2022-12-28T14:41:44.565353Z","shell.execute_reply.started":"2022-12-28T14:41:44.510101Z","shell.execute_reply":"2022-12-28T14:41:44.563186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nfrom fastai.vision.all import *","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:41:44.567682Z","iopub.execute_input":"2022-12-28T14:41:44.568075Z","iopub.status.idle":"2022-12-28T14:41:44.618757Z","shell.execute_reply.started":"2022-12-28T14:41:44.568041Z","shell.execute_reply":"2022-12-28T14:41:44.617445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"/kaggle/working/fastkaggle","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/rwightman/pytorch-image-models","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:55:33.025125Z","iopub.execute_input":"2022-12-28T14:55:33.025559Z","iopub.status.idle":"2022-12-28T14:55:36.230059Z","shell.execute_reply.started":"2022-12-28T14:55:33.025522Z","shell.execute_reply":"2022-12-28T14:55:36.228852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!git clone https://github.com/fastai/fastkaggle.git","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:55:39.123738Z","iopub.execute_input":"2022-12-28T14:55:39.124871Z","iopub.status.idle":"2022-12-28T14:55:41.159556Z","shell.execute_reply.started":"2022-12-28T14:55:39.12482Z","shell.execute_reply":"2022-12-28T14:55:41.158149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\ntimm_path = Path(r\"/kaggle/working/pytorch-image-models\")\n\nfast_kaggle_path = Path(r\"/kaggle/working/fastkaggle\")\n#timm_path.ls()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T15:10:54.114751Z","iopub.execute_input":"2022-12-28T15:10:54.115226Z","iopub.status.idle":"2022-12-28T15:10:54.169002Z","shell.execute_reply.started":"2022-12-28T15:10:54.115185Z","shell.execute_reply":"2022-12-28T15:10:54.168179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/working/pytorch-image-models\n!pip install /kaggle/working/fastkaggle","metadata":{"execution":{"iopub.status.busy":"2022-12-28T15:12:27.556286Z","iopub.execute_input":"2022-12-28T15:12:27.557195Z","iopub.status.idle":"2022-12-28T15:12:54.111701Z","shell.execute_reply.started":"2022-12-28T15:12:27.557141Z","shell.execute_reply":"2022-12-28T15:12:54.10992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nsys_paths = [timm_path, fast_kaggle_path]\n\nfor paths in sys_paths:\n    sys.path.append(paths)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.222057Z","iopub.execute_input":"2022-12-28T14:42:27.223065Z","iopub.status.idle":"2022-12-28T14:42:27.269671Z","shell.execute_reply.started":"2022-12-28T14:42:27.223009Z","shell.execute_reply":"2022-12-28T14:42:27.268303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.272486Z","iopub.execute_input":"2022-12-28T14:42:27.272969Z","iopub.status.idle":"2022-12-28T14:42:27.344401Z","shell.execute_reply.started":"2022-12-28T14:42:27.272901Z","shell.execute_reply":"2022-12-28T14:42:27.338624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!pip install -e pytorch-image-models/\n#!pip install -e fastkaggle/","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.345999Z","iopub.status.idle":"2022-12-28T14:42:27.346852Z","shell.execute_reply.started":"2022-12-28T14:42:27.346551Z","shell.execute_reply":"2022-12-28T14:42:27.346583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\nfrom fastai.vision.all import *\nfrom pathlib import Path\nfrom fastkaggle import *\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.348743Z","iopub.status.idle":"2022-12-28T14:42:27.349583Z","shell.execute_reply.started":"2022-12-28T14:42:27.349288Z","shell.execute_reply":"2022-12-28T14:42:27.349316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not iskaggle:\n    image_data_path = Path.cwd().parent/'data'/'bc_768_roi'\n    csv_path = Path.cwd().parent/'data'/'bc_768_roi'\n    #train_image_path = image_data_path/'train'\n    test_image_path  = image_data_path/'test'\n    train_image_path = Path.home()/'workspace'/'data'/'symlink'\nelse:\n    image_data_path = Path(r'/kaggle/input/rsna-small-for-faster-experimentation/data_small')\n    csv_path = Path(r'/kaggle/input/rsna-breast-cancer-detection')\n    train_image_path = image_data_path\n    test_image_path = Path(r'/kaggle/input/rsna-breast-cancer-detection-poi-images/bc_768_roi/bc_768_roi/test')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.351637Z","iopub.status.idle":"2022-12-28T14:42:27.352114Z","shell.execute_reply.started":"2022-12-28T14:42:27.351852Z","shell.execute_reply":"2022-12-28T14:42:27.351871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#csv_path.ls()\n#train_image_path.ls()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.354253Z","iopub.status.idle":"2022-12-28T14:42:27.354816Z","shell.execute_reply.started":"2022-12-28T14:42:27.354537Z","shell.execute_reply":"2022-12-28T14:42:27.354565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_trn = pd.read_csv(csv_path/'train.csv')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.356425Z","iopub.status.idle":"2022-12-28T14:42:27.3577Z","shell.execute_reply.started":"2022-12-28T14:42:27.357375Z","shell.execute_reply":"2022-12-28T14:42:27.357413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_trn.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.359593Z","iopub.status.idle":"2022-12-28T14:42:27.360176Z","shell.execute_reply.started":"2022-12-28T14:42:27.359866Z","shell.execute_reply":"2022-12-28T14:42:27.359893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fn = get_image_files(train_image_path)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.362025Z","iopub.status.idle":"2022-12-28T14:42:27.363129Z","shell.execute_reply.started":"2022-12-28T14:42:27.36279Z","shell.execute_reply":"2022-12-28T14:42:27.362822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_trn=(\n    df_trn\n    .assign(image_id = lambda df: df['image_id'].astype(str))\n)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.365008Z","iopub.status.idle":"2022-12-28T14:42:27.365566Z","shell.execute_reply.started":"2022-12-28T14:42:27.365282Z","shell.execute_reply":"2022-12-28T14:42:27.36531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef get_y_label(x):\n    return df_trn.loc[df_trn['image_id'] == x.stem.split('_')[1], 'cancer'].values[0]\n\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.367633Z","iopub.status.idle":"2022-12-28T14:42:27.368215Z","shell.execute_reply.started":"2022-12-28T14:42:27.367919Z","shell.execute_reply":"2022-12-28T14:42:27.367946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls_ff = ImageDataLoaders.from_path_func(\n                                      train_image_path,\n                                      fnames=fn,\n                                      label_func=get_y_label,\n                                      item_tfms=Resize(166, method='squish'),\n                                      batch_tfms=aug_transforms(size=128, min_scale=.75),\n                                 \n                                      valid_pct=0.2,\n                                      seed=42)\n                               ","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.370717Z","iopub.status.idle":"2022-12-28T14:42:27.37177Z","shell.execute_reply.started":"2022-12-28T14:42:27.371482Z","shell.execute_reply":"2022-12-28T14:42:27.371511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls_ff.show_batch()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.373436Z","iopub.status.idle":"2022-12-28T14:42:27.374199Z","shell.execute_reply.started":"2022-12-28T14:42:27.373887Z","shell.execute_reply":"2022-12-28T14:42:27.373916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = vision_learner(\n                       dls_ff,\n                      'resnet26d',\n                       metrics=error_rate,\n                       path='.').to_fp16()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.376361Z","iopub.status.idle":"2022-12-28T14:42:27.377033Z","shell.execute_reply.started":"2022-12-28T14:42:27.376722Z","shell.execute_reply":"2022-12-28T14:42:27.376749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = Path('/kaggle/input/first-submission-with-smaller-dataset/learner_first')","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.379258Z","iopub.status.idle":"2022-12-28T14:42:27.379839Z","shell.execute_reply.started":"2022-12-28T14:42:27.379555Z","shell.execute_reply":"2022-12-28T14:42:27.379583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.381784Z","iopub.status.idle":"2022-12-28T14:42:27.382353Z","shell.execute_reply.started":"2022-12-28T14:42:27.382071Z","shell.execute_reply":"2022-12-28T14:42:27.382099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn = learn.load(model_path)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.384104Z","iopub.status.idle":"2022-12-28T14:42:27.385211Z","shell.execute_reply.started":"2022-12-28T14:42:27.384892Z","shell.execute_reply":"2022-12-28T14:42:27.384929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tst_files = get_image_files(test_image_path)\ntst_files","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.387988Z","iopub.status.idle":"2022-12-28T14:42:27.388896Z","shell.execute_reply.started":"2022-12-28T14:42:27.388601Z","shell.execute_reply":"2022-12-28T14:42:27.38863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dl = dls_ff.test_dl(tst_files)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.390614Z","iopub.status.idle":"2022-12-28T14:42:27.391551Z","shell.execute_reply.started":"2022-12-28T14:42:27.391263Z","shell.execute_reply":"2022-12-28T14:42:27.391293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs,_,idxs = learn.get_preds(\n                   dl=test_dl,\n                   with_decoded=True\n                   )","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.393177Z","iopub.status.idle":"2022-12-28T14:42:27.394092Z","shell.execute_reply.started":"2022-12-28T14:42:27.393762Z","shell.execute_reply":"2022-12-28T14:42:27.393791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probs[:,1].numpy()","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.395727Z","iopub.status.idle":"2022-12-28T14:42:27.39662Z","shell.execute_reply.started":"2022-12-28T14:42:27.396329Z","shell.execute_reply":"2022-12-28T14:42:27.396359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tst = pd.read_csv(csv_path/'test.csv')\ndf_tst=(\n    df_tst\n    .assign(image_id = lambda df: df['image_id'].astype(str))\n)\ndf_tst","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.398253Z","iopub.status.idle":"2022-12-28T14:42:27.399078Z","shell.execute_reply.started":"2022-12-28T14:42:27.398756Z","shell.execute_reply":"2022-12-28T14:42:27.398783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_test_image(\n                       tst_files:List[Path],\n                       df_tst:pd.DataFrame,\n                        probs:np.ndarray):\n\n    prediction_ids = []\n    for i in tst_files:\n        prediction_ids.append(df_tst.loc[df_tst['image_id'] == i.stem.split('_')[1],'prediction_id'].values[0] )\n    return (\n        pd.DataFrame(prediction_ids)\n        .assign(cancer = lambda df:probs[:, 1].numpy())\n        .rename(columns={0:'prediction_id'})\n        .groupby('prediction_id', as_index=False).mean())","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.400562Z","iopub.status.idle":"2022-12-28T14:42:27.401382Z","shell.execute_reply.started":"2022-12-28T14:42:27.401094Z","shell.execute_reply":"2022-12-28T14:42:27.401122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = process_test_image(\n                                   tst_files,\n                                  df_tst,probs)\n","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.402843Z","iopub.status.idle":"2022-12-28T14:42:27.403671Z","shell.execute_reply.started":"2022-12-28T14:42:27.403384Z","shell.execute_reply":"2022-12-28T14:42:27.403412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss.to_csv(\n         'submission.csv', \n          index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-28T14:42:27.404949Z","iopub.status.idle":"2022-12-28T14:42:27.4062Z","shell.execute_reply.started":"2022-12-28T14:42:27.405864Z","shell.execute_reply":"2022-12-28T14:42:27.405894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}