{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm torchinfo","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:22:54.992426Z","iopub.execute_input":"2022-12-03T02:22:54.993519Z","iopub.status.idle":"2022-12-03T02:23:08.244427Z","shell.execute_reply.started":"2022-12-03T02:22:54.9934Z","shell.execute_reply":"2022-12-03T02:23:08.242935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"####################\n### Import Package\n####################\nimport os \nimport gc\nimport cv2\nimport timm\nimport torch\nimport pydicom\nimport torchvision\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt \n\n\n\nfrom torch import nn \nfrom torchinfo import summary\nfrom tqdm.autonotebook import tqdm\nfrom torchvision import transforms\nfrom torch.optim import lr_scheduler\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\nfrom timm.scheduler import CosineLRScheduler\nfrom torch.utils.data import DataLoader, Dataset\n\n\nbar = '=='\ndevice = torch.device('cuda') if torch.cuda.is_available() else 'cpu'\nprint(bar*20)\nprint(f'PyTorch Version :{torch.__version__}')\nprint(f'Device :{device}')\n","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:08.247173Z","iopub.execute_input":"2022-12-03T02:23:08.247526Z","iopub.status.idle":"2022-12-03T02:23:12.019318Z","shell.execute_reply.started":"2022-12-03T02:23:08.247478Z","shell.execute_reply":"2022-12-03T02:23:12.018127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG :\n    \n    # =================\n    # About Model\n    # =================\n    model = 'tf_efficientnetv2_s'\n    pretrained = True\n    in_chans = 3\n    \n    # ================\n    # About Training\n    # ================\n    nfold = 5\n    epochs = 2#30\n    batch_size = 36\n    num_workers = 2\n    weight_decay = 1e-6\n    max_grad_norm = 1000\n    lr_max = 4e-4\n    epochs_warmup = 1.0\n    \n    # ===============\n    # Normal vision\n    # ===============\n    DEBUG = False\n    beta = 0.5\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:12.021849Z","iopub.execute_input":"2022-12-03T02:23:12.022878Z","iopub.status.idle":"2022-12-03T02:23:12.030002Z","shell.execute_reply.started":"2022-12-03T02:23:12.02283Z","shell.execute_reply":"2022-12-03T02:23:12.028744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################\n### Load the CSV file\n##########################\ntrain_csv = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\ntrain_csv.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:12.035238Z","iopub.execute_input":"2022-12-03T02:23:12.035598Z","iopub.status.idle":"2022-12-03T02:23:12.171341Z","shell.execute_reply.started":"2022-12-03T02:23:12.035568Z","shell.execute_reply":"2022-12-03T02:23:12.170229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_ids = [] \npatient_ids = []\n\nfor i in tqdm(range(len(train_csv))) :\n    \n    image_id = train_csv['image_id'].iloc[i]\n    image_ids.append(image_id)\n    \n    patient_id = train_csv['patient_id'].iloc[i]\n    patient_ids.append(patient_id)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:12.173001Z","iopub.execute_input":"2022-12-03T02:23:12.173395Z","iopub.status.idle":"2022-12-03T02:23:13.364213Z","shell.execute_reply.started":"2022-12-03T02:23:12.173364Z","shell.execute_reply":"2022-12-03T02:23:13.363074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nums = 3 # example No.x nums\nprint(f'Image ID: {image_ids[nums]}')\nprint(f'Image patient ID: {patient_ids[nums]}')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:13.365906Z","iopub.execute_input":"2022-12-03T02:23:13.366311Z","iopub.status.idle":"2022-12-03T02:23:13.37303Z","shell.execute_reply.started":"2022-12-03T02:23:13.366264Z","shell.execute_reply":"2022-12-03T02:23:13.371795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'CSV Nums: {len(train_csv)}')\nprint(f'Imgae ID Nums: {len(image_ids)}')\nprint(f'patient ID Nums: {len(patient_ids)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:13.374758Z","iopub.execute_input":"2022-12-03T02:23:13.37548Z","iopub.status.idle":"2022-12-03T02:23:13.38438Z","shell.execute_reply.started":"2022-12-03T02:23:13.375443Z","shell.execute_reply":"2022-12-03T02:23:13.383087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eg_ima = image_ids[nums]\neg_pat = patient_ids[nums]\ndataset_path = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed/'\n\nexample_image = cv2.imread(f'{dataset_path}{eg_pat}/{eg_ima}.png')\nprint(f'Image Shape: {example_image.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:13.386257Z","iopub.execute_input":"2022-12-03T02:23:13.387156Z","iopub.status.idle":"2022-12-03T02:23:13.421191Z","shell.execute_reply.started":"2022-12-03T02:23:13.387109Z","shell.execute_reply":"2022-12-03T02:23:13.420297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(example_image)\ndel image_ids, patient_ids \ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:13.422442Z","iopub.execute_input":"2022-12-03T02:23:13.423337Z","iopub.status.idle":"2022-12-03T02:23:13.813544Z","shell.execute_reply.started":"2022-12-03T02:23:13.4233Z","shell.execute_reply":"2022-12-03T02:23:13.812541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################\n### Load Path\n##########################\nimage_paths = []\nimage_Labels = []\n\nfor i in tqdm(range(len(train_csv))) :\n    \n    c = train_csv['cancer'].iloc[i]\n    b = train_csv['image_id'].iloc[i]\n    a = train_csv['patient_id'].iloc[i]\n    \n    p = f'/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed/{a}/{b}.png'\n    image_paths.append(p)\n    image_Labels.append(c)","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:13.817787Z","iopub.execute_input":"2022-12-03T02:23:13.818165Z","iopub.status.idle":"2022-12-03T02:23:16.527757Z","shell.execute_reply.started":"2022-12-03T02:23:13.818127Z","shell.execute_reply":"2022-12-03T02:23:16.526742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(image_paths) == len(train_csv)","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:16.532747Z","iopub.execute_input":"2022-12-03T02:23:16.533499Z","iopub.status.idle":"2022-12-03T02:23:16.545866Z","shell.execute_reply.started":"2022-12-03T02:23:16.533444Z","shell.execute_reply":"2022-12-03T02:23:16.544801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"###########################\n### Load the Images to List\n##########################\nImages = []\nlabel_name = ['Without Cancer','Cancer'] # 0 and 1\n\nif CFG.DEBUG :\n    for i in tqdm(range(len(image_paths))) :\n        ima = cv2.imread(image_paths[i])\n        Images.append(ima)\nelse :\n    for i in tqdm(range(40)) :\n        ima = cv2.imread(image_paths[i])\n        Images.append(ima)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:16.547833Z","iopub.execute_input":"2022-12-03T02:23:16.549246Z","iopub.status.idle":"2022-12-03T02:23:16.919996Z","shell.execute_reply.started":"2022-12-03T02:23:16.549206Z","shell.execute_reply":"2022-12-03T02:23:16.918658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labs = []\nshow_num = 5\nshow_ims_nums = 25\n\nfor i in tqdm(range(show_ims_nums)) :\n    lables = label_name[image_Labels[i]]\n    labs.append(lables)\n\nplt.figure(figsize=(60, 60))\nfor i in tqdm(range(show_ims_nums)):\n    ax = plt.subplot(show_num, show_num, i + 1)\n    ax.set_title(labs[i], fontproperties='SimHei', fontsize=40)\n    plt.imshow(Images[i])\n    plt.axis(\"off\")\n        \ndel labs, Images, image_paths, image_Labels     \ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:16.92189Z","iopub.execute_input":"2022-12-03T02:23:16.922862Z","iopub.status.idle":"2022-12-03T02:23:23.980185Z","shell.execute_reply.started":"2022-12-03T02:23:16.922819Z","shell.execute_reply":"2022-12-03T02:23:23.979015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## F1 Score","metadata":{}},{"cell_type":"code","source":"def probabilistic_f1(labels, predictions, beta=CFG.beta):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n            cfp += 1 - prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:23.981653Z","iopub.execute_input":"2022-12-03T02:23:23.981953Z","iopub.status.idle":"2022-12-03T02:23:23.989336Z","shell.execute_reply.started":"2022-12-03T02:23:23.981926Z","shell.execute_reply":"2022-12-03T02:23:23.988401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RSNA_Dataset(Dataset) :\n    def __init__(self, csv_file, root, train) :\n        \n        self.data = csv_file\n        self.train = train\n        self.root_path = root\n        \n        \n    def __len__(self) :\n        return len(self.data)\n\n        \n    def __getitem__(self, index) :\n        \n        \n        image_id = self.data['image_id'].iloc[index]\n        pat_id = self.data['patient_id'].iloc[index]\n        \n        if self.train :\n            \n            image_label = self.data['cancer'].iloc[index]\n            image_label = np.array(image_label).astype(np.float32)\n            image_label = torch.tensor(image_label)\n            \n            ima_p = f'{self.root_path}{pat_id}/{image_id}.png'\n            image_inputs = cv2.imread(ima_p) / 255.0\n            image_inputs = image_inputs.transpose(2,1,0).astype(np.float32)\n            image_inputs = torch.tensor(image_inputs)\n            \n        else :\n            image_label = None\n            ima_p = f'{self.root_path}{pat_id}/{image_id}.png'\n            image_inputs = cv2.imread(ima_p) / 255.0\n            image_inputs = image_inputs.transpose(2,1,0).astype(np.float32)\n            image_inputs = torch.tensor(image_inputs)\n            \n            \n            \n        return image_inputs, image_label","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:23.990586Z","iopub.execute_input":"2022-12-03T02:23:23.991679Z","iopub.status.idle":"2022-12-03T02:23:24.005374Z","shell.execute_reply.started":"2022-12-03T02:23:23.991589Z","shell.execute_reply":"2022-12-03T02:23:24.003895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = RSNA_Dataset(train_csv, '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed/', True)\na,b = dataset[0]\nprint(a.shape)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:24.007011Z","iopub.execute_input":"2022-12-03T02:23:24.008051Z","iopub.status.idle":"2022-12-03T02:23:24.194396Z","shell.execute_reply.started":"2022-12-03T02:23:24.007989Z","shell.execute_reply":"2022-12-03T02:23:24.193289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model \n\nclass RSNANN(nn.Module):\n    def __init__(self, name, in_chans, pretrained=False):\n        \"\"\"\n        name (str): timm model name, e.g. tf_efficientnet_b2_ns\n        \"\"\"\n        super(RSNANN, self).__init__()\n\n        # Use timm\n        model = timm.create_model(name, pretrained=pretrained, in_chans=in_chans)\n\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity() # <-------我就是占位的\n\n        self.fc = nn.Linear(n_features, 1)\n        self.model = model\n\n    def forward(self, x):\n        x = self.model(x)\n        x = self.fc(x)\n        return x\n\n\nmodel = RSNANN(name=CFG.model,in_chans=CFG.in_chans, pretrained=CFG.pretrained)\nloss_fn = nn.BCEWithLogitsLoss()\n\nsummary(model)\n#model","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:23:24.196055Z","iopub.execute_input":"2022-12-03T02:23:24.197122Z","iopub.status.idle":"2022-12-03T02:23:29.332697Z","shell.execute_reply.started":"2022-12-03T02:23:24.197077Z","shell.execute_reply":"2022-12-03T02:23:29.331503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\n\ndef evaluate(model, loader_val, *, compute_score=True, pbar=None):\n    \"\"\"\n    Predict and compute loss and score\n    \"\"\"\n    tb = time.time()\n    model.eval()\n\n    loss_sum = 0.0\n    n_sum = 0\n    y_all = []\n    y_pred_all = []\n\n    if pbar is not None:\n        pbar = tqdm(desc='Predict', nrows=78, total=pbar)\n\n    for img, y in loader_val:\n        \n        n = img.size(0)\n        img =  img.to(device)\n        y = y.to(device)\n\n        with torch.no_grad():\n            y_pred = model(img)\n            \n        loss = loss_fn(y_pred.view(-1), y)\n\n        n_sum += n\n        loss_sum += n * loss.item()\n\n        y_all.append(y.cpu().detach().numpy())\n        y_pred_all.append(torch.sigmoid(y_pred).squeeze().cpu().detach().numpy())\n\n        if pbar is not None :\n            pbar.update(len(img))\n        \n        del loss, y_pred, img, y\n\n    loss_val = loss_sum / n_sum\n\n    y = np.concatenate(y_all)\n    y_pred = np.concatenate(y_pred_all)\n\n    score = roc_auc_score(y, y_pred) if compute_score else None\n    \n    \n    \n    \n    ret = {'loss': loss_val,\n           'score': score,\n           'y': y,\n           'y_pred': y_pred,\n           'time': time.time() - tb}\n    \n    \n\n    return ret","metadata":{"execution":{"iopub.status.busy":"2022-12-03T03:29:34.420613Z","iopub.execute_input":"2022-12-03T03:29:34.421061Z","iopub.status.idle":"2022-12-03T03:29:34.434011Z","shell.execute_reply.started":"2022-12-03T03:29:34.421018Z","shell.execute_reply":"2022-12-03T03:29:34.432736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kfold = KFold(n_splits=CFG.nfold, random_state=42, shuffle=True)\n\n\nfor ifold, (idx_train, idx_test) in enumerate(kfold.split(train_csv)):\n    print('Fold %d/%d' % (ifold, CFG.nfold))\n    torch.manual_seed(42 + ifold + 1)\n\n    # Train - val split\n    dataset_train = RSNA_Dataset(train_csv.iloc[idx_train], '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed/', True)\n    dataset_val = RSNA_Dataset(train_csv.iloc[idx_train], '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs/train_images_processed/', True)\n\n    loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=CFG.batch_size,\n                     num_workers=CFG.num_workers, pin_memory=True, shuffle=True, drop_last=True)\n    loader_val = torch.utils.data.DataLoader(dataset_val, batch_size=CFG.batch_size,\n                     num_workers=CFG.num_workers, pin_memory=True)\n\n    # Model and optimizer\n    model.to(device)\n    model.train()\n\n    optimizer = torch.optim.AdamW(model.parameters(), lr=CFG.lr_max, weight_decay=CFG.weight_decay)\n\n    # Learning-rate schedule\n    nbatch = len(loader_train)\n    warmup = CFG.epochs_warmup * nbatch  # number of warmup steps\n    nsteps = CFG.epochs * nbatch        # number of total steps\n\n    scheduler = CosineLRScheduler(optimizer,\n                                  warmup_t=warmup,\n                                  warmup_lr_init=0.0,\n                                  warmup_prefix=True, # 1 epoch of warmup\n                                  t_initial=(nsteps - warmup), lr_min=1e-6)                # 3 epochs of cosine\n    \n    time_val = 0.0\n    lrs = []\n\n    tb = time.time()\n    print('Epoch   loss          score   lr')\n    for iepoch in range(CFG.epochs):\n        loss_sum = 0.0\n        n_sum = 0\n\n        # Train\n        for ibatch, (img, y) in enumerate(loader_train):\n            n = img.size(0)\n            img = img.to(device)\n            y = y.to(device)\n\n            optimizer.zero_grad()\n\n            y_pred = model(img)\n            loss = loss_fn(y_pred.view(-1), y)\n\n            loss_train = loss.item()\n            loss_sum += n * loss_train\n            n_sum += n\n\n            loss.backward()\n\n            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(),\n                                                       CFG.max_grad_norm)\n            optimizer.step()\n            \n            scheduler.step(iepoch * nbatch + ibatch + 1)\n            lrs.append(optimizer.param_groups[0]['lr'])            \n\n        # Evaluate\n        val = evaluate(model, loader_val)\n        time_val += val['time']\n        loss_train = loss_sum / n_sum\n        lr_now = optimizer.param_groups[0]['lr']\n        dt = (time.time() - tb) / 60\n        print('Epoch %d %.4f %.4f %.4f  %.2e  %.2f min' %\n              (iepoch + 1, loss_train, val['loss'], val['score'], lr_now, dt))\n\n    dt = time.time() - tb\n    print('Training done %.2f min total, %.2f min val' % (dt / 60, time_val / 60))\n\n    \n    \n    # Save Model\n    torch.save(model.state_dict(), f'saved_model{ifold}.bin')\n    \n    saved_model = torch.jit.script(model)\n    saved_model.save(f'saved_model{ifold}_JIT.bin')\n    print('==>> Mdoel Saved!!') \n    gc.collect()\n\n    break  # 1 fold only","metadata":{"execution":{"iopub.status.busy":"2022-12-03T03:29:42.638074Z","iopub.execute_input":"2022-12-03T03:29:42.638517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('LR Schedule: Cosine with linear warmup')\nplt.xlabel('steps')\nplt.ylabel('learning rate')\nplt.plot(lrs)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-12-03T02:44:14.400126Z","iopub.execute_input":"2022-12-03T02:44:14.401022Z","iopub.status.idle":"2022-12-03T02:44:14.639718Z","shell.execute_reply.started":"2022-12-03T02:44:14.400968Z","shell.execute_reply":"2022-12-03T02:44:14.638446Z"},"trusted":true},"execution_count":null,"outputs":[]}]}