{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# !pip install timm -q","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:13.859575Z","iopub.execute_input":"2023-07-28T08:07:13.860097Z","iopub.status.idle":"2023-07-28T08:07:26.91931Z","shell.execute_reply.started":"2023-07-28T08:07:13.860053Z","shell.execute_reply":"2023-07-28T08:07:26.917557Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\n\nimport gc\n\n# image manipulation\nimport cv2\nimport PIL\nfrom PIL import Image\n\n# visualisation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# helpers\nfrom tqdm import tqdm\nimport time\nimport copy\nimport gc\nfrom enum import Enum\nfrom sklearn import model_selection\nfrom sklearn.metrics import roc_auc_score, precision_recall_curve\n\nimport timm\n\n\n# for cnn\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, AdamW, SGD\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader, random_split, TensorDataset, Dataset, WeightedRandomSampler\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau, StepLR\nfrom torchvision import models\nfrom torchmetrics.classification import BinaryF1Score, BinaryPrecision, BinaryRecall, BinaryAccuracy, BinaryROC, BinaryAUROC\nfrom torchvision import transforms\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\nimport albumentations as A","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:26.922052Z","iopub.execute_input":"2023-07-28T08:07:26.922459Z","iopub.status.idle":"2023-07-28T08:07:44.556178Z","shell.execute_reply.started":"2023-07-28T08:07:26.922424Z","shell.execute_reply":"2023-07-28T08:07:44.554843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed = 42):\n    '''Sets the seed of the entire notebook so results are the same every time we run.\n    This is for REPRODUCIBILITY.'''\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # When running on the CuDNN backend, two further options must be set\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n    # Set a fixed value for the hash seed\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    print('> SEEDING DONE')\n    \nset_seed(42)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.557647Z","iopub.execute_input":"2023-07-28T08:07:44.558736Z","iopub.status.idle":"2023-07-28T08:07:44.572476Z","shell.execute_reply.started":"2023-07-28T08:07:44.558702Z","shell.execute_reply":"2023-07-28T08:07:44.571613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config :\n    epochs_warmup = 0\n    epochs = 5\n    num_cycles = 0.5\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Dataset:\n    def __init__(self, df, transform):\n        self.df = df.copy()\n        self.transform = transform\n     \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        \n        patient_id = self.df.loc[idx, 'patient_id']\n        image_id = self.df.loc[idx, 'image_id']\n        # Target\n        target = self.df.loc[idx, 'cancer'] \n        \n        # Get and preprocess images\n        #dcm_path = '/kaggle/input/rsna-breast-cancer-detection/train_images/'\n        #image = dicom.dcmread(image_path)\n        \n        #png_path = '/kaggle/input/rsna-png-images-same-format-as-original/output/rsna_pngs/train_images'\n        #png_path = '/kaggle/input/png-cutted-images/output/png_cutted_images'\n        png_path = '/kaggle/input/preprocessed-images-rsna/output/preprocessed_images_RSNA'\n        \n        # Image path\n        #image_dcm_path =  os.path.join(png_path, patient_id.astype(str), image_id.astype(str)+'.dcm')\n        image_png_path =  os.path.join(png_path, patient_id.astype(str), image_id.astype(str)+'.png')\n\n        # convert image to RGB\n        image = Image.open(image_png_path).convert('RGB')\n        \n        # Check if image is RGB \n        #print('image shape = ', np.array(image).shape)\n        \n        image = image.resize((1024,912))\n        \n        \n        # Apply transformers on images\n        #if (target == 1) and self.transform:\n        if self.transform:    \n            image = self.transform(image).to(torch.float32)\n        else :\n            default_transform = transforms.Compose([transforms.ToTensor()])\n            image = default_transform(image).to(torch.float32)\n            #image = image.to(torch.float32)\n            \n                \n        \n        # Convert to tensors\n        #image = torch.tensor(image, dtype=torch.float32)\n        \n        target = torch.tensor(target, dtype=torch.float) # long\n        \n        return image, target","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.575904Z","iopub.execute_input":"2023-07-28T08:07:44.576365Z","iopub.status.idle":"2023-07-28T08:07:44.601589Z","shell.execute_reply.started":"2023-07-28T08:07:44.576331Z","shell.execute_reply":"2023-07-28T08:07:44.600242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Efficientnetv2_s\ndef gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (\n            self.__class__.__name__\n            + \"(\"\n            + \"p=\"\n            + \"{:.4f}\".format(self.p.data.tolist()[0])\n            + \", \"\n            + \"eps=\"\n            + str(self.eps)\n            + \")\"\n        )\n    \nclass Efficientnetv2_s(nn.Module):\n    def __init__(self, p=3, p_trainable=False, eps=1e-6):\n        super(Efficientnetv2_s, self).__init__()\n        \n        # tf_efficientnetv2_s\n        self.efficientnetv2_s = timm.create_model('tf_efficientnetv2_s', pretrained=True, in_chans=3)\n        model = self.efficientnetv2_s\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n        self.fc = nn.Linear(n_features, 1) # cancer\n        self.pool = nn.Sequential(\n            GeM(p=p, eps=eps, p_trainable=p_trainable),\n            nn.Flatten())\n    \n    # if tf_efficientnetv2_s\n    def forward(self, x):\n        x = self.efficientnetv2_s(x) \n        #x = self.efficientnetv2_s.forward_features(x)\n        #x = self.pool(x)\n        logits = self.fc(x)\n        return logits   \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.603671Z","iopub.execute_input":"2023-07-28T08:07:44.604422Z","iopub.status.idle":"2023-07-28T08:07:44.62045Z","shell.execute_reply.started":"2023-07-28T08:07:44.60438Z","shell.execute_reply":"2023-07-28T08:07:44.619487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        #self.network = models.resnet18(pretrained=True)\n        #n_features = self.network.fc.out_features\n        \n        self.network = timm.create_model('resnet18', pretrained=True, in_chans=3)\n        n_features = self.network.fc.out_features\n        \n        # add additional layer that maps 2048 extracted features from resnet to 1 feature \n        #determining the class\n        self.classifier_layer = nn.Sequential(\n            nn.Linear(n_features , 256),\n            nn.Dropout(0.5),\n            nn.Linear(256 , 1)\n        )\n    \n    def forward(self, xb):        \n        xb = self.network(xb)\n        xb = self.classifier_layer(xb)\n        return torch.sigmoid(xb)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.621927Z","iopub.execute_input":"2023-07-28T08:07:44.622644Z","iopub.status.idle":"2023-07-28T08:07:44.640792Z","shell.execute_reply.started":"2023-07-28T08:07:44.622604Z","shell.execute_reply":"2023-07-28T08:07:44.639569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Only resnet\nclass CNN_RESNET(nn.Module):\n    def __init__(self):\n        super(CNN_RESNET, self).__init__()\n        #self.network = models.resnet18(pretrained=True)\n        #n_features = self.network.fc.out_features\n        \n        self.resnet = timm.create_model('resnet18', pretrained=True, in_chans=3) #resnet18\n        n_features = self.resnet.fc.out_features\n        \n        # add additional layer that maps 2048 extracted features from resnet to 1 feature \n        #determining the class\n        self.classifier_layer = nn.Sequential(\n            nn.Linear(n_features , 256),\n            nn.Dropout(0.8), #0.3\n            nn.Linear(256 , 1)\n        )\n    \n    def forward(self, xb):        \n        xb = self.resnet(xb)\n        xb = self.classifier_layer(xb)\n        return xb\n        #return torch.sigmoid(xb)\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.642058Z","iopub.execute_input":"2023-07-28T08:07:44.642415Z","iopub.status.idle":"2023-07-28T08:07:44.659462Z","shell.execute_reply.started":"2023-07-28T08:07:44.642385Z","shell.execute_reply":"2023-07-28T08:07:44.658499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\nclass GeM(nn.Module):\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n    def __repr__(self):\n        return (\n            self.__class__.__name__\n            + \"(\"\n            + \"p=\"\n            + \"{:.4f}\".format(self.p.data.tolist()[0])\n            + \", \"\n            + \"eps=\"\n            + str(self.eps)\n            + \")\"\n        )\n    \nclass MammoModel(nn.Module):\n    def __init__(self, name, *, pretrained=False, in_chans=3, p=3, p_trainable=False, eps=1e-6):\n        super().__init__()\n        model = timm.create_model(name, pretrained=pretrained, in_chans=in_chans)\n        clsf = model.default_cfg['classifier']\n        n_features = model._modules[clsf].in_features\n        model._modules[clsf] = nn.Identity()\n        \n        self.fc = nn.Linear(n_features, 1) # cancer\n        self.model = model\n\n        self.pool = nn.Sequential(\n            GeM(p=p, eps=eps, p_trainable=p_trainable),\n            nn.Flatten())\n    \n    def forward(self, x):\n        x = self.model(x)\n        x = self.model.forward_features(x)\n        x = self.pool(x)\n        logits = self.fc(x)\n        return logits","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.661126Z","iopub.execute_input":"2023-07-28T08:07:44.661698Z","iopub.status.idle":"2023-07-28T08:07:44.673265Z","shell.execute_reply.started":"2023-07-28T08:07:44.661665Z","shell.execute_reply":"2023-07-28T08:07:44.672299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Model (we need to improve our model''' \nclass BreastCancerModel(nn.Module):\n    def __init__(self, Config):\n        super().__init__()\n        # efficientnet\n        self.efficientnet = timm.create_model('efficientnet_b4', pretrained=True,\n                                             in_chans=3)\n        in_features = self.efficientnet.classifier.in_features\n        self.efficientnet.classifier = nn.Linear(in_features, Config.NUM_CLASSES)  \n        \n        \n        # Resnet\n        self.resnet = timm.create_model('resnet50', pretrained=True, in_chans=3)\n        self.resnet.fc = nn.Linear(2048, Config.NUM_CLASSES)\n    \n        \n    def forward(self, image):\n        output = self.efficientnet(image)\n        #output = self.resnet(image)\n    \n        return output\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.674603Z","iopub.execute_input":"2023-07-28T08:07:44.675112Z","iopub.status.idle":"2023-07-28T08:07:44.690597Z","shell.execute_reply.started":"2023-07-28T08:07:44.675084Z","shell.execute_reply":"2023-07-28T08:07:44.689486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def BCELoss_class_weighted(weights):\n    \"\"\"\n    weights[0] is weight for class 0 (negative class)\n    weights[1] is weight for class 1 (positive class)\n    \"\"\"\n    def loss(y_pred, target):\n        y_pred = torch.clamp(y_pred,min=1e-7,max=1-1e-7) # for numerical stability\n        bce = - weights[1] * target * torch.log(y_pred) - (1 - target) * weights[0] * torch.log(1 - y_pred)\n        return torch.mean(bce)\n\n    return loss","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.696641Z","iopub.execute_input":"2023-07-28T08:07:44.696998Z","iopub.status.idle":"2023-07-28T08:07:44.708431Z","shell.execute_reply.started":"2023-07-28T08:07:44.696967Z","shell.execute_reply":"2023-07-28T08:07:44.707413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create class for earlystopping\nclass EarlyStopper:\n    def __init__(self, patience=1, min_delta=0):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.min_loss = np.inf\n\n    def early_stop(self, loss):\n        if loss <= self.min_loss:\n            self.min_loss = loss\n            self.counter = 0\n        elif loss > (self.min_loss + self.min_delta):\n            self.counter += 1\n            if self.counter >= self.patience:\n                return True\n        return False","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.709946Z","iopub.execute_input":"2023-07-28T08:07:44.710654Z","iopub.status.idle":"2023-07-28T08:07:44.722789Z","shell.execute_reply.started":"2023-07-28T08:07:44.710619Z","shell.execute_reply":"2023-07-28T08:07:44.721735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_optimal_threshold(fpr, tpr, thresholds): # based on ROC curve\n    J = tpr - fpr\n    index = np.argmax(J)\n    OptThreshold = thresholds[index]\n    return OptThreshold\n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.724254Z","iopub.execute_input":"2023-07-28T08:07:44.725326Z","iopub.status.idle":"2023-07-28T08:07:44.737024Z","shell.execute_reply.started":"2023-07-28T08:07:44.725292Z","shell.execute_reply":"2023-07-28T08:07:44.736133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### '''Training function'''\ndef train(data_loader, model, optimizer, device, criterion, scheduler=None):\n    # Set model in training mode\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled=True)\n    \n    # Metrics\n    metricf1 = BinaryF1Score()\n    precision = BinaryPrecision()\n    recall = BinaryRecall()\n    accuracy = BinaryAccuracy()\n    roc = BinaryROC()\n    auc = BinaryAUROC()\n    \n    train_metrics = {'loss' : [], 'acc' : [], 'f1': [], 'precision': [], 'recall': [], 'auc': []}\n    \n    # Initial threshold\n    threshold = 0.5\n        \n\n    # DataLoader\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    running_loss = 0\n    correct = 0\n    total = 0\n    all_outputs = torch.Tensor([])\n    all_labels = torch.Tensor([]) # needed to calculate auc score ??\n   \n    for batch_idx, data in enumerate(tk0):\n        images = data[0]\n        targets = data[1]\n        \n        targets = torch.unsqueeze(targets, 1)\n        \n        images = images.to(device).float() \n        targets = targets.to(device).float() # long\n        \n        #print('targets shape = ', targets.shape)\n        \n        optimizer.zero_grad()\n        #with torch.set_grad_enabled(True):\n        with torch.cuda.amp.autocast(enabled=True):    \n        \n            outputs = model(images)#.squeeze()\n            #print('outputs shape = ', outputs.shape)\n\n        loss = criterion(outputs, targets)\n\n        #loss.backward()\n        #optimizer.step()\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        # all outputs\n        all_outputs = torch.cat((all_outputs, outputs.to('cpu')))\n\n        # all labels\n        all_labels = torch.cat((all_labels, targets.to('cpu')))\n\n\n        # Correct classifications\n        probs = outputs.sigmoid() # we don't need sigmoid if we have it in the \n        # model class \n        predicted_vals = probs > threshold\n        correct += torch.sum(predicted_vals == targets.data)\n\n\n        # Total classifications\n        total += targets.size(0)\n\n        #auc score\n        #auc_score = roc_auc_score(all_labels.detach().numpy(), all_outputs .detach().numpy())\n        \n        # running loss    \n        running_loss += loss.item()\n\n        # collect any unused memmory\n        #gc.collect()\n        #torch.cuda.empty_cache()\n        \n        del outputs\n        torch.cuda.empty_cache()\n\n    #Accuracy\n    accu = correct/total\n    \n    # Average training loss (epoch loss)\n    train_loss = running_loss / len(data_loader)\n    \n    #auc score\n    auc_score = roc_auc_score(all_labels.detach().numpy(), all_outputs.detach().numpy())\n    \n    \n    # ROC\n    print('all_labels type', all_labels.dtype)\n    fpr, tpr, thresholds = roc(all_outputs, all_labels.long())\n    \n    \n    # Find optimal threshold\n    OptThreshold = find_optimal_threshold(fpr, tpr, thresholds)\n    \n    \n    print(f'New threshold is {OptThreshold}')\n    \n    # Calculate metrics using optimized threshold\n    # f1 score\n    f1_measure = metricf1(all_outputs > OptThreshold, all_labels)\n\n    ","metadata":{"execution":{"iopub.status.busy":"2023-08-01T08:59:15.83748Z","iopub.execute_input":"2023-08-01T08:59:15.837873Z","iopub.status.idle":"2023-08-01T08:59:15.8531Z","shell.execute_reply.started":"2023-08-01T08:59:15.837843Z","shell.execute_reply":"2023-08-01T08:59:15.851653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Training function by work of the 7th place'''\n\ndef train(data_loader, model, optimizer, device, criterion, scheduler=None):\n    # Set model in training mode\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled=True)\n    \n    # Metrics\n    metricf1 = BinaryF1Score()\n    precision = BinaryPrecision()\n    recall = BinaryRecall()\n    accuracy = BinaryAccuracy()\n    roc = BinaryROC()\n    auc = BinaryAUROC()\n    \n    train_metrics = {'loss' : [], 'acc' : [], 'f1': [], 'precision': [], 'recall': [], 'auc': []}\n    \n    # Initial threshold\n    threshold = 0.5\n        \n\n    # DataLoader\n    tk0 = tqdm(data_loader, total=len(data_loader))\n    \n    running_loss = 0\n    correct = 0\n    total = 0\n    all_outputs = torch.Tensor([])\n    all_labels = torch.Tensor([]) # needed to calculate auc score ??\n   \n    for batch_idx, data in enumerate(tk0):\n        images = data[0]\n        targets = data[1]\n        \n        targets = torch.unsqueeze(targets, 1)\n        \n        images = images.to(device).float() \n        targets = targets.to(device).float() # long\n        \n        #print('targets shape = ', targets.shape)\n        \n        optimizer.zero_grad()\n        \n        #with torch.set_grad_enabled(True):\n        with torch.cuda.amp.autocast(enabled=True):    \n        \n            outputs = model(images)#.squeeze()\n            #print('outputs shape = ', outputs.shape)\n\n        loss = criterion(outputs, targets)\n\n        #loss.backward()\n        #optimizer.step()\n\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        # all outputs\n        all_outputs = torch.cat((all_outputs, outputs.to('cpu')))\n\n        # all labels\n        all_labels = torch.cat((all_labels, targets.to('cpu')))\n\n\n        # Correct classifications\n        probs = outputs.sigmoid() \n        \n        predicted_vals = probs > threshold\n        correct += torch.sum(predicted_vals == targets.data)\n\n\n        # Total classifications\n        total += targets.size(0)\n\n        #auc score\n        #auc_score = roc_auc_score(all_labels.detach().numpy(), all_outputs .detach().numpy())\n        \n        # running loss    \n        running_loss += loss.item()\n\n        # collect any unused memmory\n        #gc.collect()\n        #torch.cuda.empty_cache()\n\n    #Accuracy\n    accu = correct/total\n    \n    # Average training loss (epoch loss)\n    train_loss = running_loss / len(data_loader)\n    \n    #auc score\n    auc_score = roc_auc_score(all_labels.detach().numpy(), all_outputs.detach().numpy())\n\n    \n    # Calculate metrics using optimized threshold\n    # f1 score\n    f1_measure = metricf1(all_outputs > threshold, all_labels)\n   \n    print('Train Loss: %.3f | Accuracy: %.3f'%(train_loss,accu))\n    print('auc_score : %.3f'%(auc_score))\n    print('f1_measure : %.3f'%(f1_measure))\n  \n    \n   \n","metadata":{"execution":{"iopub.status.busy":"2023-07-10T10:29:31.078544Z","iopub.execute_input":"2023-07-10T10:29:31.078977Z","iopub.status.idle":"2023-07-10T10:29:31.094956Z","shell.execute_reply.started":"2023-07-10T10:29:31.078945Z","shell.execute_reply":"2023-07-10T10:29:31.094055Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Evaluation function'''\n\ndef evaluation(data_loader, model, device, criterion):\n    model.eval()\n    \n    # Metrics\n    metricf1 = BinaryF1Score()\n    precision = BinaryPrecision()\n    recall = BinaryRecall()\n    accuracy = BinaryAccuracy()\n    roc = BinaryROC()\n    auc = BinaryAUROC()\n    \n    \n    predictions = []\n    tar = []\n    running_loss = 0\n    correct = 0\n    total = 0\n    \n    threshold = 0.5\n    \n    all_outputs = torch.Tensor([])\n    all_labels = torch.Tensor([]) # needed to calculate auc score ??\n    \n    for batch_idx, data in enumerate(data_loader):\n            images = data[0]\n            targets = data[1]\n            \n            #targets = torch.unsqueeze(targets.to(torch.float), 1) #float32\n            targets = torch.unsqueeze(targets, 1)\n        \n            images = images.to(device, dtype=torch.float) \n            targets = targets.to(device, dtype=torch.float) # long ??\n       \n            with torch.no_grad():\n                outputs = model(images)#.squeeze()\n            loss = criterion(outputs, targets) \n                \n            # all outputs\n            all_outputs = torch.cat((all_outputs, outputs.to('cpu')))\n            \n            # all labels\n            all_labels = torch.cat((all_labels, targets.to('cpu')))    \n            \n            running_loss += loss.item()\n            \n            # Correct classifications (for accuracy calculation)\n            probs = outputs.sigmoid()\n    \n            predicted_vals = probs > threshold\n            correct += torch.sum(predicted_vals == targets.data)\n            # Total classifications\n            total += targets.size(0)\n            \n    \n            # Predictions\n            #predictions.append(predicted_vals.cpu().numpy())\n            predictions.append(predicted_vals.cpu().numpy())\n            \n            # Targets\n            tar.append(targets.cpu().numpy())\n            \n            # collect any unused memmory\n            #gc.collect()\n            #torch.cuda.empty_cache()\n            \n            del outputs\n            torch.cuda.empty_cache()\n     \n    # Loss \n    eval_loss = running_loss / len(data_loader)\n    # Accuracy\n    eval_accu = correct/total\n    print('Eval Loss: %.3f | Accuracy: %.3f'%(eval_loss,eval_accu))\n    \n    #auc score\n    eval_auc_score = roc_auc_score(all_labels.detach().numpy(), all_outputs .detach().numpy())\n    \n    # ROC\n    print('type of labels', all_labels.dtype)\n    fpr, tpr, thresholds = roc(all_outputs, all_labels.long())\n    \n    # Find optimal threshold\n    OptThreshold = find_optimal_threshold(fpr, tpr, thresholds)\n    \n    # Calculate metrics using optimized threshold\n    # f1 score\n    eval_f1_measure = metricf1(all_outputs > OptThreshold, all_labels)\n    \n    \n    # Predictions\n    predictions = np.concatenate(predictions) # this line convert the list \n      # of lists into a 1d array.\n        \n    # Targets    \n    tar = np.concatenate(tar)\n    \n    print('Eval Loss: %.3f | Eval Accuracy: %.3f'%(eval_loss,eval_accu))\n    print('Eval auc_score : %.3f'%(eval_auc_score))\n    print('Eval_f1_measure : %.3f'%(eval_f1_measure))\n\n    return predictions, tar, OptThreshold     \n    ","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.75916Z","iopub.execute_input":"2023-07-28T08:07:44.759862Z","iopub.status.idle":"2023-07-28T08:07:44.777576Z","shell.execute_reply.started":"2023-07-28T08:07:44.759822Z","shell.execute_reply":"2023-07-28T08:07:44.776474Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Probabilistic F1 score'''\ndef pfbeta(labels, predictions, beta):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n    \n    \n    for idx in range(len(labels)):  \n        #print('predictions[idx] = ', predictions[idx])\n        prediction = min(max(predictions[idx], 0), 1)\n        #print('prediction = ', prediction)\n        #print('labels[idx] = ', labels[idx])\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n            #print('ctp = ', ctp)\n        else:\n            cfp += prediction\n            #print('cfp = ', cfp)\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.77866Z","iopub.execute_input":"2023-07-28T08:07:44.77967Z","iopub.status.idle":"2023-07-28T08:07:44.798637Z","shell.execute_reply.started":"2023-07-28T08:07:44.779639Z","shell.execute_reply":"2023-07-28T08:07:44.797427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_csv = '/kaggle/input/rsna-breast-cancer-detection/train.csv'\ndfx = pd.read_csv(train_csv)\n\n'''\naugmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomVerticalFlip(0.5),\n    transforms.RandomRotation(5),\n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n])\n'''\ntransform = transforms.Compose([\n    transforms.ToTensor()])\n\n'''\naugmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomVerticalFlip(),\n    transforms.RandomAffine(degrees=(0, 180), scale=(0.8, 1.2)),\n    transforms.ElasticTransform(),\n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n])\n\n'''\n\naugmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    transforms.RandomHorizontalFlip(0.5),\n    transforms.RandomVerticalFlip(0.5),\n    #transforms.RandomPerspective(),\n    #transforms.RandomRotation((0, 90)),\n    #transforms.RandomAutocontrast(),\n    #transforms.RandomAffine(degrees=(0, 180), scale=(0.8, 1.2)),\n    #transforms.ElasticTransform(),\n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n    transforms.Normalize(mean = [0.1338, 0.1338, 0.1338],\n                         std = [0.2068, 0.2068, 0.2068])    \n])\n\nvalid_augmentator = transforms.Compose([\n    # input for augmentator is always PIL image\n    # transforms.ToPILImage(),\n    #transforms.RandomHorizontalFlip(0.5),\n    #transforms.RandomVerticalFlip(0.5),\n    #transforms.RandomPerspective(),\n    #transforms.RandomRotation((0, 90)),\n    #transforms.RandomAutocontrast(),\n    #transforms.RandomAffine(degrees=(0, 180), scale=(0.8, 1.2)),\n    #transforms.ElasticTransform(),\n    transforms.ToTensor(), # return it as a tensor and transforms it to [0, 1]\n    transforms.Normalize(mean = [0.1338, 0.1338, 0.1338],\n                         std = [0.2068, 0.2068, 0.2068])    \n])","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.800121Z","iopub.execute_input":"2023-07-28T08:07:44.800969Z","iopub.status.idle":"2023-07-28T08:07:44.930643Z","shell.execute_reply.started":"2023-07-28T08:07:44.800929Z","shell.execute_reply":"2023-07-28T08:07:44.92965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 8  #32, 16","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.932361Z","iopub.execute_input":"2023-07-28T08:07:44.932771Z","iopub.status.idle":"2023-07-28T08:07:44.937088Z","shell.execute_reply.started":"2023-07-28T08:07:44.932732Z","shell.execute_reply":"2023-07-28T08:07:44.936386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Split data before oversampling using train_test_split from sklearn'''\ndf_train, df_valid = model_selection.train_test_split(dfx, test_size=0.15, stratify=dfx['cancer'], random_state=42)\ndf_train = df_train.reset_index(drop=True)\ndf_valid = df_valid.reset_index(drop=True)\n\n# Instantiate Dataset with training data\ntrain_dataset = Dataset(df_train, transform=augmentator)\n\n# Instantiate Dataloader with training dataset\ntrain_data_loader = torch.utils.data.DataLoader(train_dataset, \n                                                batch_size=batch_size, \n                                                num_workers=2,\n                                               drop_last=True)\n\n# Instantiate Dataset with validation data\nvalid_data =  Dataset(df_valid, transform=valid_augmentator)\n\n\n# Instantiate Dataloader with valiation dataset\nvalid_data_loader = torch.utils.data.DataLoader(valid_data, \n                                                batch_size=batch_size,\n                                                num_workers=2, \n                                                drop_last=True)\n# The advantage of train_test_split is that, the splitting of data into train and validation datasets is \n# done before oversampling data. In this notebook, we use augmentations to deal with imbalanced data, so\n# we are oversampling our data.This operation should be applied only on the train dataset. Otherwise, \n# the validation dataset will de very similar to the train data and we will obtain an unrealistic \n# measure of performance.","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:44.938065Z","iopub.execute_input":"2023-07-28T08:07:44.938913Z","iopub.status.idle":"2023-07-28T08:07:44.999098Z","shell.execute_reply.started":"2023-07-28T08:07:44.938883Z","shell.execute_reply":"2023-07-28T08:07:44.997869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' Split data after oversampling using random_split'''\n'''\ndataset = Dataset(dfx, transform=augmentator)\nval_pct = 0.1\nval_size = int(val_pct * len(dataset))\ntrain_size = len(dataset) - val_size\ntrain_dataset, val_dataset = random_split(dataset, [train_size, val_size])\n\n\n\n# Applying sampler just to train dataset, not for validation, since the validation dataset \n# should be an imitation of real Dataset\n\n#train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle = False, num_workers = 2, \n#                              pin_memory = True, sampler = weighted_random_sampler) # using our sampler\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle = True, num_workers = 2, \n                              pin_memory = True) # random sampler\n\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle = False, pin_memory = True)\n# If you use shuffle = True the DataLoader will initialize a RandomSampler for you, \n# otherwise it’ll use SequentialSampler.\n\ndataloaders = {'train' : train_dataloader, 'val' : val_dataloader}\ndataset_sizes = {'train': train_size, 'val' : val_size}\n    \n'''\n","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:45.000939Z","iopub.execute_input":"2023-07-28T08:07:45.001543Z","iopub.status.idle":"2023-07-28T08:07:45.009703Z","shell.execute_reply.started":"2023-07-28T08:07:45.00151Z","shell.execute_reply":"2023-07-28T08:07:45.008559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta_binarized(labels, predictions):\n    positives = predictions[labels == 1]\n    scores = []\n    for th in positives:\n        binarized = (predictions >= th).astype('int')\n        score = pfbeta(labels, binarized, 1)\n        scores.append(score)\n    return np.max(scores)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:45.010982Z","iopub.execute_input":"2023-07-28T08:07:45.011298Z","iopub.status.idle":"2023-07-28T08:07:45.027601Z","shell.execute_reply.started":"2023-07-28T08:07:45.011271Z","shell.execute_reply":"2023-07-28T08:07:45.026337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# Define the model\n#model = MammoModel('tf_efficientnetv2_s', pretrained=True) #efficientnet_b5\n#model = CNN()\nmodel = Efficientnetv2_s()\n#model = BreastCancerModel(Config=Config)\nmodel.to(device)\n\n# Define the criterion (loss)\n#w_pos = 3\n#w_neg = 1\n#criterion = BCELoss_class_weighted(weights = [w_neg, w_pos])\n\n# or\ncriterion = nn.BCEWithLogitsLoss()\n\n# Define the optimizer\n#optimizer = AdamW(model.parameters(), lr=2.56e-05)\noptimizer = Adam(model.parameters(), lr=2.56e-05)\n\n\n# Define the metric\nmetric = BinaryF1Score().to(device)\n\n\nearlystoper = EarlyStopper(patience = 3)\n\n# Scheduler\n#scheduler = StepLR(optimizer, step_size=5, gamma=0.1)\nscheduler = get_cosine_schedule_with_warmup(\n                optimizer, num_warmup_steps=Config.epochs_warmup, num_training_steps=Config.epochs, \n        num_cycles=Config.num_cycles\n            )\n\n\n# Model path\n#model_path = f\"model_{fold}.bin\"\nmodel_path = \"model.bin\"\n    \n# Run\nbest_score = 0\nfor epoch in range(Config.epochs): \n    train(train_data_loader, model, optimizer, device, criterion, scheduler)\n    scheduler.step()\n    predictions, targets, OptThreshold = evaluation(valid_data_loader, model, device, criterion)\n\n    print('epoch = ', epoch)\n\n    score = pfbeta(targets, predictions, beta=1)\n    #score = pfbeta_binarized(targets, predictions)\n    \n    #if F1_score > best_score:\n    if score > best_score:\n        #best_score = F1_score\n        best_score = score\n        \n        PATH = \"model.pt\"\n        \n        checkpoint = torch.save({\n            'model_state_dict': model.state_dict(),\n            'threshold' : OptThreshold\n            }, PATH)\n            \n        \n    #print('f1_score = ', best_score)\n    print('score = ', best_score) \n\nprint('best_score_ever = ', best_score) \n\ntorch.cuda.empty_cache()\ngc.collect()\n","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:07:45.028959Z","iopub.execute_input":"2023-07-28T08:07:45.029437Z","iopub.status.idle":"2023-07-28T08:37:49.691436Z","shell.execute_reply.started":"2023-07-28T08:07:45.029407Z","shell.execute_reply":"2023-07-28T08:37:49.689979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In this version, we used only augmentations. \n# In this notebook, we use train_test_split from sklearn in order to split data before oversampling the\n# train dataset.\n# In this notebook, I use preprocessed_images_rnsa dataset created by the notebook \n# preprocess_images_RSNA.","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:37:49.69286Z","iopub.status.idle":"2023-07-28T08:37:49.693578Z","shell.execute_reply.started":"2023-07-28T08:37:49.693363Z","shell.execute_reply":"2023-07-28T08:37:49.693387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# In this version, I have added normalization to the augmentations part. \n# In this version we used binarized f1 score.\n# In this version, we apply augmentations without condition target =1.\n\n# In this version, we use tf_efficientnetv2_s model, without pooling_layer.\n\n# In this version, we have a training and an evaluation functions, by moving threshold. In this case \n# we should use binarized score function.\n# => score =  0.19 in the fisrt epoch.\n\n\n# To resume, in this version, we have efficient-net without pooling layer, our training and evaluation \n# functions, with threshold and pfbeta.\n# With 3 epochs, we obtained a score=0.16.\n\n","metadata":{"execution":{"iopub.status.busy":"2023-07-28T08:37:49.694825Z","iopub.status.idle":"2023-07-28T08:37:49.695473Z","shell.execute_reply.started":"2023-07-28T08:37:49.695264Z","shell.execute_reply":"2023-07-28T08:37:49.695291Z"},"trusted":true},"execution_count":null,"outputs":[]}]}