{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-01T06:14:27.105087Z","iopub.execute_input":"2023-01-01T06:14:27.105395Z","iopub.status.idle":"2023-01-01T06:14:27.124684Z","shell.execute_reply.started":"2023-01-01T06:14:27.105328Z","shell.execute_reply":"2023-01-01T06:14:27.123799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install timm==0.6.12","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:14:27.12642Z","iopub.execute_input":"2023-01-01T06:14:27.126847Z","iopub.status.idle":"2023-01-01T06:14:40.404067Z","shell.execute_reply.started":"2023-01-01T06:14:27.126813Z","shell.execute_reply":"2023-01-01T06:14:40.402679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import classification_report\nimport timm.optim.optim_factory as optim_factory\nfrom timm.data import create_transform\nfrom torch.utils.data import Dataset\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom sklearn import metrics\nimport glob\n%matplotlib inline\nimport torch.nn as nn\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nimport torch\nimport timm\nimport cv2","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:14:40.405818Z","iopub.execute_input":"2023-01-01T06:14:40.406205Z","iopub.status.idle":"2023-01-01T06:14:43.567044Z","shell.execute_reply.started":"2023-01-01T06:14:40.406167Z","shell.execute_reply":"2023-01-01T06:14:43.565895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_EPOCHS = 5\nNUM_SPLITS = 4\n\nRESIZE_TO = (512, 512)\n\nDATA_PATH = '/kaggle/input/rsna-breast-cancer-detection/'\nTRAIN_IMAGE_DIR = '/kaggle/input/rsnabcd-512-png-v2-dataset/train_images/'\nMODEL_PATH = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:12.763488Z","iopub.execute_input":"2023-01-01T06:16:12.764251Z","iopub.status.idle":"2023-01-01T06:16:12.771611Z","shell.execute_reply.started":"2023-01-01T06:16:12.764212Z","shell.execute_reply":"2023-01-01T06:16:12.768535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = pd.read_csv(f'{DATA_PATH}/train.csv')\ntrain_csv['path'] = TRAIN_IMAGE_DIR + train_csv[\"patient_id\"].astype(str) + \"/\" + train_csv[\"image_id\"].astype(str) + \".png\"\n\nskf = StratifiedKFold(NUM_SPLITS, shuffle=True, random_state=7)\n\ndata_train=train_csv['path'].values\nlabels_train=train_csv['cancer'].values","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:13.079049Z","iopub.execute_input":"2023-01-01T06:16:13.079347Z","iopub.status.idle":"2023-01-01T06:16:13.214289Z","shell.execute_reply.started":"2023-01-01T06:16:13.07932Z","shell.execute_reply":"2023-01-01T06:16:13.213301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta_torch(preds, labels, beta=1):\n    preds = preds.clip(0, 1)\n\n    y_true_count = labels.sum()\n    ctp = preds[labels == 1].sum()\n    cfp = preds[labels == 0].sum()\n\n    beta_squared = beta * beta\n\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n\n    if c_precision > 0 and c_recall > 0:\n        return ((1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)).item()\n    else:\n        return 0.0\n\ndef pfbeta_thresh(preds, labels):\n    optimized_preds = optimize_preds(preds, labels)\n    return pfbeta_torch(optimized_preds, labels)\n\n\ndef optimize_preds(preds, labels, return_thresh=False, print_results=False):\n    preds = preds.clone()\n\n    without_thresh = pfbeta_torch(preds, labels)\n\n    threshs = np.linspace(0, 1, 101)\n    f1s = [pfbeta_torch((preds > thr).float(), labels) for thr in threshs]\n    idx = np.argmax(f1s)\n    thresh, best_pfbeta = threshs[idx], f1s[idx]\n\n    preds = (preds > thresh).float()\n\n    if print_results:\n        print(f\"without optimization: {without_thresh:.3f}\")\n        pfbeta = pfbeta_torch(preds, labels)\n        print(f\"with optimization: {pfbeta:.3f}\")\n        print(f\"best_thresh: {thresh}\")\n\n    if return_thresh:\n        return thresh\n\n    return preds","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:14.439257Z","iopub.execute_input":"2023-01-01T06:16:14.439631Z","iopub.status.idle":"2023-01-01T06:16:14.450575Z","shell.execute_reply.started":"2023-01-01T06:16:14.439601Z","shell.execute_reply":"2023-01-01T06:16:14.449513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BalanceSampler(torch.utils.data.sampler.Sampler):\n    def __init__(self, dataset, ratio=4):\n        self.r = ratio-1\n        self.dataset = dataset\n        self.pos_index = np.where(dataset.label>0)[0]\n        self.neg_index = np.where(dataset.label==0)[0]\n\n        self.length = self.r*int(np.floor(len(self.neg_index)/self.r))\n\n    def __iter__(self):\n        pos_index = self.pos_index.copy()\n        neg_index = self.neg_index.copy()\n        np.random.shuffle(pos_index)\n        np.random.shuffle(neg_index)\n\n        neg_index = neg_index[:self.length].reshape(-1,self.r)\n        pos_index = np.random.choice(pos_index, self.length//self.r).reshape(-1,1)\n\n        index = np.concatenate([pos_index,neg_index],-1).reshape(-1)\n        return iter(index)\n\n    def __len__(self):\n        return self.length","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:14.653292Z","iopub.execute_input":"2023-01-01T06:16:14.653602Z","iopub.status.idle":"2023-01-01T06:16:14.664586Z","shell.execute_reply.started":"2023-01-01T06:16:14.653575Z","shell.execute_reply":"2023-01-01T06:16:14.663587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class GetLoader(Dataset):\n    def __init__(self, transform, data_train=train_csv['path'].values, labels_train=train_csv['cancer'].values, is_train=True):\n        self.data = data_train\n        self.label = labels_train\n        self.trans = transform\n\n    def __getitem__(self, index):\n        data = Image.open(self.data[index]).convert('RGB')\n        data = torch.as_tensor(self.trans(data),dtype=torch.float32).cuda()\n        labels = torch.as_tensor(self.label[index],dtype=torch.float32).cuda()\n        return data, labels\n\n    def __len__(self):\n        return len(self.data)\n\n\ndef build_transform(is_train):\n    if is_train:\n        transform = create_transform(\n            input_size=RESIZE_TO,\n            is_training=True,\n            scale=(0.75, 1.33),\n            ratio=(0.08, 1.0),\n            hflip=0.5,\n            vflip=0.5,\n            color_jitter=0.4,\n            interpolation=\"random\",\n        )\n    else:\n        transform = create_transform(\n            input_size=RESIZE_TO,\n            is_training=False,\n            interpolation=\"bilinear\",\n        )\n    return transform\n\n\ndef build_dataset(data_train=train_csv['path'].values, labels_train=train_csv['cancer'].values, is_train=True):\n    transform = build_transform(is_train)\n    dataset = GetLoader(transform,data_train, labels_train, is_train=True)\n    return dataset","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:14.901275Z","iopub.execute_input":"2023-01-01T06:16:14.901587Z","iopub.status.idle":"2023-01-01T06:16:14.911972Z","shell.execute_reply.started":"2023-01-01T06:16:14.90156Z","shell.execute_reply":"2023-01-01T06:16:14.91077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tf_efficientnetv2_s():\n    model = timm.create_model(\n        'tf_efficientnetv2_s', pretrained=True, in_chans=3, num_classes=1)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:15.924443Z","iopub.execute_input":"2023-01-01T06:16:15.925134Z","iopub.status.idle":"2023-01-01T06:16:15.930072Z","shell.execute_reply.started":"2023-01-01T06:16:15.9251Z","shell.execute_reply":"2023-01-01T06:16:15.928855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size_train = 8\nbatch_size_test = 4\n# THRESHOLD = 0.7","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:16.11881Z","iopub.execute_input":"2023-01-01T06:16:16.119187Z","iopub.status.idle":"2023-01-01T06:16:16.124155Z","shell.execute_reply.started":"2023-01-01T06:16:16.119157Z","shell.execute_reply":"2023-01-01T06:16:16.12284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num=0\nfor train_index, test_index in skf.split(data_train,labels_train):\n        X_train, X_test = np.array(data_train)[train_index], np.array(data_train)[test_index]\n        y_train, y_test = np.array(labels_train)[train_index], np.array(labels_train)[test_index]\n        train_dataset = build_dataset(X_train, y_train,is_train=True)\n        train_dataloader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=batch_size_train,\n        sampler=BalanceSampler(train_dataset),\n        drop_last=True,\n    )\n        test_dataset = build_dataset(X_test, y_test,is_train=False)\n        test_dataloader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=batch_size_test,\n        drop_last=False,\n    )\n        model = tf_efficientnetv2_s().cuda()\n        model_without_ddp = model\n        param_groups = optim_factory.param_groups_weight_decay(model_without_ddp,0.05)\n        optimizer = torch.optim.AdamW(param_groups, lr=2.25e-4, betas=(0.9, 0.999))\n        loss = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([3])).cuda()\n        model.train()\n        num+=1\n        for epoch in range(NUM_EPOCHS):\n            model.train()\n            for img, label in tqdm(train_dataloader):\n                label = label.unsqueeze(1)\n                l = loss(model(img),label.float())\n                optimizer.zero_grad()\n                l.backward()\n                optimizer.step()\n            prediction = []\n            labels = []\n            model.eval()\n            for img, label in tqdm(test_dataloader):\n                output = torch.sigmoid(model(img)).detach().cpu()\n                # output = model(img)\n                prediction.append(output)\n                labels.append(label.cpu())                 \n            preds = torch.cat(prediction)\n            labels = torch.cat(labels)\n            print('{}efficientnet_b4_epoch{}.pth'.format(num,epoch))\n            threshold = optimize_preds(preds.float(), labels.float(), return_thresh=True, print_results=True)\n            print(threshold)\n            torch.save(model.state_dict(),'{}{}efficientnet_b4_epoch{}.pth'.format(MODEL_PATH,num,epoch))","metadata":{"execution":{"iopub.status.busy":"2023-01-01T06:16:16.445313Z","iopub.execute_input":"2023-01-01T06:16:16.445762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1115it [16:38,  1.12it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.46it/s]\n1efficientnet_b4_epoch0.pth\nwithout optimization: 0.074\nwith optimization: 0.114\nbest_thresh: 0.96\n0.96\n1115it [16:33,  1.12it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.45it/s]\n1efficientnet_b4_epoch1.pth\nwithout optimization: 0.128\nwith optimization: 0.189\nbest_thresh: 0.76\n0.76\n1115it [16:36,  1.12it/s]                        \n100%|██████████| 855/855 [02:35<00:00,  5.51it/s]\n1efficientnet_b4_epoch2.pth\nwithout optimization: 0.100\nwith optimization: 0.139\nbest_thresh: 0.6900000000000001\n0.6900000000000001\n1115it [16:33,  1.12it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.45it/s]\n1efficientnet_b4_epoch3.pth\nwithout optimization: 0.113\nwith optimization: 0.169\nbest_thresh: 0.86\n0.86\n1115it [16:33,  1.12it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.38it/s]\n1efficientnet_b4_epoch4.pth\nwithout optimization: 0.119\nwith optimization: 0.145\nbest_thresh: 0.47000000000000003\n0.47000000000000003\n1115it [16:42,  1.11it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.38it/s]\n2efficientnet_b4_epoch0.pth\nwithout optimization: 0.089\nwith optimization: 0.139\nbest_thresh: 0.8200000000000001\n0.8200000000000001\n1115it [16:42,  1.11it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.40it/s]\n2efficientnet_b4_epoch1.pth\nwithout optimization: 0.109\nwith optimization: 0.192\nbest_thresh: 0.96\n0.96\n1115it [16:41,  1.11it/s]                        \n100%|██████████| 855/855 [02:37<00:00,  5.42it/s]\n2efficientnet_b4_epoch2.pth\nwithout optimization: 0.112\nwith optimization: 0.154\nbest_thresh: 0.8300000000000001\n0.8300000000000001\n1115it [16:42,  1.11it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.39it/s]\n2efficientnet_b4_epoch3.pth\nwithout optimization: 0.113\nwith optimization: 0.137\nbest_thresh: 0.48\n0.48\n1115it [16:42,  1.11it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.39it/s]\n2efficientnet_b4_epoch4.pth\nwithout optimization: 0.121\nwith optimization: 0.146\nbest_thresh: 0.49\n0.49\n1115it [16:44,  1.11it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.46it/s]\n3efficientnet_b4_epoch0.pth\nwithout optimization: 0.075\nwith optimization: 0.128\nbest_thresh: 0.9\n0.9\n1115it [16:51,  1.10it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.48it/s]\n3efficientnet_b4_epoch1.pth\nwithout optimization: 0.111\nwith optimization: 0.178\nbest_thresh: 0.96\n0.96\n1115it [16:43,  1.11it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.45it/s]\n3efficientnet_b4_epoch2.pth\nwithout optimization: 0.103\nwith optimization: 0.141\nbest_thresh: 0.88\n0.88\n1115it [16:46,  1.11it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.46it/s]\n3efficientnet_b4_epoch3.pth\nwithout optimization: 0.117\nwith optimization: 0.145\nbest_thresh: 0.91\n0.91\n1115it [16:44,  1.11it/s]                        \n100%|██████████| 855/855 [02:36<00:00,  5.46it/s]\n3efficientnet_b4_epoch4.pth\nwithout optimization: 0.100\nwith optimization: 0.151\nbest_thresh: 0.97\n0.97\n1115it [16:56,  1.10it/s]                        \n100%|██████████| 855/855 [02:39<00:00,  5.35it/s]\n4efficientnet_b4_epoch0.pth\nwithout optimization: 0.090\nwith optimization: 0.192\nbest_thresh: 0.9\n0.9\n1115it [16:54,  1.10it/s]                        \n100%|██████████| 855/855 [02:38<00:00,  5.39it/s]\n4efficientnet_b4_epoch1.pth\nwithout optimization: 0.110\nwith optimization: 0.175\nbest_thresh: 0.93\n0.93\n1115it [17:01,  1.09it/s]                        \n100%|██████████| 855/855 [02:43<00:00,  5.24it/s]\n4efficientnet_b4_epoch2.pth\nwithout optimization: 0.127\nwith optimization: 0.176\nbest_thresh: 0.91\n0.91\n1115it [16:54,  1.10it/s]                        \n100%|██████████| 855/855 [02:41<00:00,  5.28it/s]\n4efficientnet_b4_epoch3.pth\nwithout optimization: 0.143\nwith optimization: 0.186\nbest_thresh: 0.72\n0.72\n1115it [17:02,  1.09it/s]                        \n100%|██████████| 855/855 [02:39<00:00,  5.37it/s]\n4efficientnet_b4_epoch4.pth\nwithout optimization: 0.132\nwith optimization: 0.187\nbest_thresh: 0.81\n0.81","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}