{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from IPython.display import clear_output, display_html\nimport os\nimport sys\nimport warnings\nfrom pathlib import Path\n\n# Basic libraries\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport scipy as sc\nfrom scipy import stats\n\n# Train Test Split\nfrom sklearn.model_selection import train_test_split\n\n# Cross Validation\nfrom sklearn.model_selection import KFold, cross_val_score, StratifiedKFold, learning_curve, train_test_split\n\n# Tensorflow\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n# PyTorch \nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import DataLoader\nimport torchvision\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nsys.path.append('../input/timm-pytorch-image-models/pytorch-image-models-master')\nimport timm \n\nfrom PIL import Image\n\n# Plotly\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.figure_factory as ff\nimport plotly.offline as offline\nimport plotly.graph_objs as go\n\nwarnings.filterwarnings('ignore')\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nclear_output()","metadata":{"_uuid":"829765ab-1e9f-49d1-8260-04a690b63b18","_cell_guid":"e4d6d4b8-0ad5-4114-bf04-09bb8181451e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:09.405931Z","iopub.execute_input":"2023-02-09T10:46:09.406476Z","iopub.status.idle":"2023-02-09T10:46:20.349801Z","shell.execute_reply.started":"2023-02-09T10:46:09.406364Z","shell.execute_reply":"2023-02-09T10:46:20.348794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Weights and Biases","metadata":{"_uuid":"8f3b9c03-1f69-4a34-a10f-7df5aaa8f882","_cell_guid":"45b47672-491a-48ca-aa99-82e46689cb51","trusted":true}},{"cell_type":"code","source":"!pip install wandb\nclear_output()\nimport wandb\nfrom wandb.keras import WandbCallback\n\ntry:\n    from kaggle_secrets import UserSecretsClient\n    user_secrets = UserSecretsClient()\n    api_key = user_secrets.get_secret(\"wandb_api\")\n    wandb.login(key=api_key)\n    anony = None\nexcept:\n    anony = \"must\"\n    print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')","metadata":{"_uuid":"e8012c62-10f6-499b-952c-27f7eb99a9a7","_cell_guid":"ad15aad0-6748-4cf2-9774-cdf6539ac3df","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:20.351716Z","iopub.execute_input":"2023-02-09T10:46:20.35199Z","iopub.status.idle":"2023-02-09T10:46:37.157959Z","shell.execute_reply.started":"2023-02-09T10:46:20.351964Z","shell.execute_reply":"2023-02-09T10:46:37.156437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data():\n    '''Load each of the datasets we are given.'''\n    \n    data_dir = Path(\"../input/rsna-splits\")\n    train = pd.read_csv(data_dir / \"train.csv\")\n    data_dir = Path(\"../input/rsna-breast-cancer-detection\")\n    test = pd.read_csv(data_dir / \"test.csv\")\n    sample_submission = pd.read_csv(data_dir / 'sample_submission.csv')\n    return train, test, sample_submission\n\ntrain, test, sample_submission = load_data()\nclear_output()","metadata":{"_uuid":"95ba8797-dbb4-4f0c-b04c-3aaf88810531","_cell_guid":"3f657b21-fd63-44cb-961a-ed235fe3859b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.159625Z","iopub.execute_input":"2023-02-09T10:46:37.160289Z","iopub.status.idle":"2023-02-09T10:46:37.365198Z","shell.execute_reply.started":"2023-02-09T10:46:37.160242Z","shell.execute_reply":"2023-02-09T10:46:37.363994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n#IMAGES_PATH = '../input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512'\nTRAIN_IMAGES_PATH = f'/kaggle/input/rsna-breast-cancer-1024-pngs/output'","metadata":{"_uuid":"4d045c30-6c98-427a-a672-f6917525c50c","_cell_guid":"bfec5330-0bdd-4153-a215-cf83bdda9f42","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.370895Z","iopub.execute_input":"2023-02-09T10:46:37.378534Z","iopub.status.idle":"2023-02-09T10:46:37.384694Z","shell.execute_reply.started":"2023-02-09T10:46:37.37849Z","shell.execute_reply":"2023-02-09T10:46:37.383545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Competition's Metric","metadata":{"_uuid":"1d2778e7-1277-4177-8379-a26c009050d1","_cell_guid":"a33aebfd-7bc2-4ca4-b7ca-f4903cbc014b","trusted":true}},{"cell_type":"code","source":"def pfbeta(labels, predictions, beta=1.):\n    y_true_count = 0\n    ctp = 0\n    cfp = 0\n\n    for idx in range(len(labels)):\n        prediction = min(max(predictions[idx], 0), 1)\n        if (labels[idx]):\n            y_true_count += 1\n            ctp += prediction\n        else:\n            cfp += prediction\n\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / max(y_true_count, 1)  # avoid / 0\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        #print(type(np.asarray(result.cpu())[0]))\n        return np.asarray(result.cpu())[0]\n    else:\n        return 0","metadata":{"_uuid":"4a929886-eb77-400c-9ab4-2a8291d2fb38","_cell_guid":"c598c042-96c5-49cd-9458-03961245ff65","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.389218Z","iopub.execute_input":"2023-02-09T10:46:37.389882Z","iopub.status.idle":"2023-02-09T10:46:37.405253Z","shell.execute_reply.started":"2023-02-09T10:46:37.389845Z","shell.execute_reply":"2023-02-09T10:46:37.40434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def pfbeta_torch(labels, preds, beta=1):\n    preds = preds.clip(0, 1)\n    y_true_count = labels.sum()\n    ctp = preds[labels==1].sum()\n    cfp = preds[labels==0].sum()\n    beta_squared = beta * beta\n    c_precision = ctp / (ctp + cfp)\n    c_recall = ctp / y_true_count\n    if (c_precision > 0 and c_recall > 0):\n        result = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall)\n        return result\n    else:\n        return 0.0","metadata":{"_uuid":"97ba0ff1-94b5-428a-b456-8c98a711d445","_cell_guid":"5be21bc1-dae1-4add-908a-c6f5c36811e5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.407319Z","iopub.execute_input":"2023-02-09T10:46:37.408094Z","iopub.status.idle":"2023-02-09T10:46:37.415531Z","shell.execute_reply.started":"2023-02-09T10:46:37.408058Z","shell.execute_reply":"2023-02-09T10:46:37.414493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimal_f1(labels, predictions):\n    '''With this function we obtain the optimal threshold, given our model predictions. '''\n    \n    thres = np.linspace(0, 1, 201)\n    f1s = [pfbeta(labels, predictions > thr) for thr in thres]\n    idx = np.argmax(f1s)\n    return f1s[idx], thres[idx]","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:46:37.417245Z","iopub.execute_input":"2023-02-09T10:46:37.418181Z","iopub.status.idle":"2023-02-09T10:46:37.425481Z","shell.execute_reply.started":"2023-02-09T10:46:37.418145Z","shell.execute_reply":"2023-02-09T10:46:37.424362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Strategy","metadata":{"_uuid":"38569869-0c1f-42aa-8a5f-18cb4fa58eab","_cell_guid":"b3bea00e-6496-4302-8cb3-007a6ba4485e","trusted":true}},{"cell_type":"markdown","source":"from sklearn.model_selection import StratifiedKFold\n\nNUM_SPLITS = 4\n\npatient_id_any_cancer = train.groupby('patient_id').cancer.max().reset_index()\nskf = StratifiedKFold(NUM_SPLITS, shuffle=True, random_state=42)\nsplits = list(skf.split(patient_id_any_cancer.patient_id, patient_id_any_cancer.cancer))","metadata":{"_uuid":"02b8fcad-48db-40ec-bbcb-3fe23354bc18","_cell_guid":"5ca9005c-7b85-45d5-ba57-c9431a5f6074","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-01-08T15:11:04.824633Z","iopub.execute_input":"2023-01-08T15:11:04.826723Z","iopub.status.idle":"2023-01-08T15:11:04.855485Z","shell.execute_reply.started":"2023-01-08T15:11:04.826684Z","shell.execute_reply":"2023-01-08T15:11:04.85463Z"}}},{"cell_type":"markdown","source":"# Dataset","metadata":{"_uuid":"fdd7fe82-485a-4e1f-b62d-391ff9616a3b","_cell_guid":"5124be85-39c0-4992-98f5-21ecd3d8b8ed","trusted":true}},{"cell_type":"code","source":"import cv2\n\ndef horizontal_flip(img):\n    '''Flips horizontally the image given as parameter. '''\n    return cv2.flip(img, 1)","metadata":{"_uuid":"3dfc6416-2f9c-4292-a9c2-312ff9102c80","_cell_guid":"4a4cedf3-a244-4410-8c72-d68e6e18c14f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.427333Z","iopub.execute_input":"2023-02-09T10:46:37.428013Z","iopub.status.idle":"2023-02-09T10:46:37.571261Z","shell.execute_reply.started":"2023-02-09T10:46:37.427979Z","shell.execute_reply":"2023-02-09T10:46:37.569446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision\nimport albumentations\nfrom albumentations.augmentations.crops.transforms import RandomResizedCrop\n\ndef get_transforms(crop=False):\n    \"\"\"\n    # old transforms\n    create_transform(\n        (1024, 512), \n        mean=0.53, #(0.53, 0.53, 0.53),\n        std=0.23, #(0.23, 0.23, 0.23),\n        is_training=is_training, \n        auto_augment=f'rand-m{config.AUTO_AUG_M}-n{config.AUTO_AUG_N}'\n    )\n    \"\"\"\n    def transforms(img, laterality):\n        \n        if laterality == 'R':\n            img = horizontal_flip(img)\n            \n            #tfm = [\n            #    torchvision.transforms.RandomHorizontalFlip(0.5),\n            #    torchvision.transforms.RandomRotation(degrees=(-5, 5)), \n            #    torchvision.transforms.RandomResizedCrop((1024, 512), scale=(0.8, 1), ratio=(0.45, 0.55)) \n            #]\n        #img = torchvision.transforms.RandomResizedCrop((1024, 512), scale=(0.8, 1), ratio=(0.45, 0.55))(torch.tensor(img))\n        #print(img.shape)\n        #img = cv2.resize(img, (512, 1024))  \n        if crop == True: \n            random_crop = RandomResizedCrop(1024, 512, scale=(0.8, 1), ratio=(0.45, 0.55))\n            img = random_crop(image=img)['image']\n        #img = torchvision.transforms.RandomResizedCrop((1024, 512), scale=(0.8, 1), ratio=(0.45, 0.55))(torch.tensor(img))\n        #print(img)\n        #img = img.resize(img.shape[1], img.shape[2], img.shape[0])\n        \n        return img\n\n    return lambda img, laterality: transforms(img, laterality)\n\nif DEBUG:\n    tfm = get_transforms(crop=True)\n    img = cv2.imread(f\"{TRAIN_IMAGES_PATH}/10006/1459541791.png\")\n    plt.imshow(img)\n    plt.show()\n\n    v = tfm(img, \"R\")\n    plt.imshow(v)\n    \n    del tfm, img","metadata":{"_uuid":"b43091f0-de37-4f68-a554-1fc4185e172e","_cell_guid":"83345b2c-74dc-47c6-b90b-7b92adb95767","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:37.575703Z","iopub.execute_input":"2023-02-09T10:46:37.57625Z","iopub.status.idle":"2023-02-09T10:46:38.020465Z","shell.execute_reply.started":"2023-02-09T10:46:37.57621Z","shell.execute_reply":"2023-02-09T10:46:38.019487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.transforms import *\nimport kornia\nfrom kornia.augmentation import *\n\nclass DataAugmentation1(nn.Module):\n    def __init__(self,):\n        super().__init__()\n        \n        self.flip = nn.Sequential(\n            RandomVerticalFlip(p=0.5),\n        )\n\n        p=0.8\n        self.transform_geometry = ImageSequential(\n            RandomAffine(degrees=5, scale=[0.8,1.2], shear=20, p=p),\n            #RandomThinPlateSpline(scale=0.05, p=p),\n            random_apply=1, #choose 1\n        )\n\n    @torch.no_grad()  # disable gradients for effiency\n    def forward(self, x):\n        x = self.flip(x)  # BxCxHxW\n        x = self.transform_geometry(x)\n        #x = self.transform_intensity(x)\n        #x = self.transform_other(x)\n        return x\n\ndata_augmentation = DataAugmentation1()\n\nclass BreastCancerDataSet(torch.utils.data.Dataset):\n    def __init__(self, df, path):\n        super().__init__()\n        self.df = df # Dataframe containing information about the images\n        self.path = path # Root directory of the images\n\n    def __getitem__(self, i):\n        # Construct the filepath of the image using its patient_id and image_id\n        path = f'{self.path}/{self.df.iloc[i].patient_id}_{self.df.iloc[i].image_id}.png'\n        y_cancer = torch.as_tensor(self.df.iloc[i].cancer) # Cancer label of the image\n        \n        img = Image.open(path).convert('RGB') \n        img = torchvision.transforms.ToTensor()(img) # Convert PIL Image to Pytorch Tensor\n        if self.df.iloc[i].laterality == 'R': \n            img = RandomHorizontalFlip(p=1)(img) # All images with same laterality\n\n        if y_cancer == 1: img = data_augmentation(img)\n        \n        # 1536 / 768 = 2 => H = 2*W\n        img = torchvision.transforms.RandomResizedCrop((2048, 1024), scale=(0.8, 1), ratio=(0.45, 0.55))(img)\n        #img = torchvision.transforms.Resize((1024, 512))(img) # All images must have the same size \n        \n        return img.squeeze(), y_cancer\n\n    def __len__(self):\n        return len(self.df)\n\nif DEBUG:\n    training_data = BreastCancerDataSet(train, TRAIN_IMAGES_PATH)\n    img, l = training_data[1]\n    plt.imshow(img.squeeze().permute(1,2,0))","metadata":{"_uuid":"9771946a-a675-42cd-8559-429c727f1034","_cell_guid":"7b760bf9-7b6b-4877-ab6c-ff1f4862426d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.024454Z","iopub.execute_input":"2023-02-09T10:46:38.024744Z","iopub.status.idle":"2023-02-09T10:46:38.264666Z","shell.execute_reply.started":"2023-02-09T10:46:38.024718Z","shell.execute_reply":"2023-02-09T10:46:38.26366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"_uuid":"aadd0837-d2f3-4fd3-9d77-9a27a62c115e","_cell_guid":"ec5c21f9-7d2d-4ef8-af39-129167cd52f7","trusted":true}},{"cell_type":"markdown","source":"training_data = BreastCancerDataSet(train, TRAIN_IMAGES_PATH, get_transforms(crop=True))\ndata_loader = torch.utils.data.DataLoader(\n                      training_data, \n                      batch_size = 4, shuffle=True)\n\nX, y = next(iter(data_loader))\n\nprint(X.permute(0,3,1,2).shape)\n\naux_model(X.permute(0,3,1,2).to(torch.float))","metadata":{"_uuid":"a2508369-ac61-40b0-ae9a-44e79184a5cf","_cell_guid":"27791526-c54d-49f6-80fa-c4d332135113","execution":{"iopub.status.busy":"2023-01-02T00:37:04.220691Z","iopub.execute_input":"2023-01-02T00:37:04.221183Z","iopub.status.idle":"2023-01-02T00:37:10.009979Z","shell.execute_reply.started":"2023-01-02T00:37:04.221139Z","shell.execute_reply":"2023-01-02T00:37:10.008873Z"},"trusted":true}},{"cell_type":"code","source":"#timm.list_models()\n#'inception_resnet_v2',\n# 'inception_v3',\n# 'inception_v4',","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:46:38.26711Z","iopub.execute_input":"2023-02-09T10:46:38.267842Z","iopub.status.idle":"2023-02-09T10:46:38.272417Z","shell.execute_reply.started":"2023-02-09T10:46:38.267803Z","shell.execute_reply":"2023-02-09T10:46:38.271359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchvision.models import efficientnet\nimport cv2\n\nclass BreastCancerModel(nn.Module):\n    # pasar s como parametro para inicializar los pesos\n    def __init__(self, model_type = 'inception_resnet_v2'):\n        super().__init__()\n        self.backbone = timm.create_model(model_type, pretrained=True)\n        self.backbone_dim = self.backbone(torch.randn(1, 3, 2048, 1024)).shape[-1]\n        self.head = nn.Sequential(\n            nn.Linear(self.backbone_dim, 1)\n        )\n        \n    #def _init_weights(self, s):\n    #    nn.init.normal_(self.head.weight, mean = 0.0, std = s)\n                \n    def forward(self, x):\n        x = self.backbone(x)\n        x = self.head(x)\n        return x\n    \n    def predict(self, x):\n        cancer = self.forward(x)\n        return cancer\n    \nif DEBUG:\n    with torch.no_grad():\n        model = BreastCancerModel()\n        #model = model.to(device)\n        pred= model.predict(torch.randn(2, 3, 1024, 512))\n        print('seresnext', pred.shape)\n        print(pred)\n\n    del model","metadata":{"_uuid":"c5cf113f-e4a2-4447-a21b-14088b6f8b23","_cell_guid":"63495281-d487-4090-b25a-37186bd87c0d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.27399Z","iopub.execute_input":"2023-02-09T10:46:38.274461Z","iopub.status.idle":"2023-02-09T10:46:38.286105Z","shell.execute_reply.started":"2023-02-09T10:46:38.274425Z","shell.execute_reply":"2023-02-09T10:46:38.285196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{"_uuid":"aff4b0a2-7f05-4c13-ae78-a0fbce19012d","_cell_guid":"354efd6c-ac48-4f4a-a1f6-83e0181d0f25","trusted":true}},{"cell_type":"code","source":"# Config device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"_uuid":"174d2317-2bde-4bda-ba17-ba75856668ac","_cell_guid":"852c4bf0-23a7-4598-b8d7-d6cc3b626af5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.289668Z","iopub.execute_input":"2023-02-09T10:46:38.289927Z","iopub.status.idle":"2023-02-09T10:46:38.366736Z","shell.execute_reply.started":"2023-02-09T10:46:38.289903Z","shell.execute_reply":"2023-02-09T10:46:38.365174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{"_uuid":"b1c78183-68a3-41e5-ad4e-11cc6e5c041f","_cell_guid":"7a659538-4242-42ae-8750-f7ab7b5d65b9","trusted":true}},{"cell_type":"markdown","source":"Aquí vamos a hacer oversampling de las imagenes con cancer. Esto, para conseguir más casos positivos para el entrenamiento.","metadata":{"_uuid":"8ab3f40b-763f-4082-a65a-065400f8671c","_cell_guid":"7fc188ea-ebac-45db-9a22-f27eb3065c01","trusted":true}},{"cell_type":"code","source":"class EarlyStopping():\n    \"\"\"\n    Early stopping to stop the training when the loss does not improve after\n    certain epochs.\n    \"\"\"\n    def __init__(self, patience=5, min_delta=0):\n        \"\"\"\n        :param patience: how many epochs to wait before stopping when loss is\n               not improving\n        :param min_delta: minimum difference between new loss and old loss for\n               new loss to be considered as an improvement\n        \"\"\"\n        self.patience = patience\n        self.min_delta = min_delta\n        self.counter = 0\n        self.best_loss = None\n        self.early_stop = False\n        \n    def __call__(self, val_loss):\n        if self.best_loss == None:\n            self.best_loss = val_loss\n            \n        elif self.best_loss - val_loss > self.min_delta:\n            self.best_loss = val_loss\n            # reset counter if validation loss improves\n            self.counter = 0\n            \n        elif self.best_loss - val_loss < self.min_delta:\n            self.counter += 1\n            print(f\"INFO: Early stopping counter {self.counter} of {self.patience}\")\n            if self.counter >= self.patience:\n                print('INFO: Early stopping')\n                self.early_stop = True","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:46:38.367929Z","iopub.execute_input":"2023-02-09T10:46:38.368949Z","iopub.status.idle":"2023-02-09T10:46:38.389252Z","shell.execute_reply.started":"2023-02-09T10:46:38.368903Z","shell.execute_reply":"2023-02-09T10:46:38.388341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SequentialSampler(torch.utils.data.Sampler):\n\n    def __init__(self, idx):\n        self.idx = idx\n\n    def __iter__(self):\n        return iter(self.idx)\n\n    def __len__(self):\n        return len(self.idx)\n    \nclass BalanceSampler(torch.utils.data.Sampler):\n\n    def __init__(self, dataset, idx, ratio = 3):\n        # Ratio = 3: [1,0,0,1,0,0,1, ...] \n        # Ratio = 8: [1,0,0,0,0,0,0,0,1, ...]\n        self.r = ratio-1\n        self.dataset = dataset\n        self.df = dataset.df.iloc[idx,:]\n        self.pos_index = self.df.index.values[np.where(self.df.cancer>0)[0]]\n        self.neg_index = self.df.index.values[np.where(self.df.cancer==0)[0]]\n\n        self.length = self.r * int(np.floor(len(self.neg_index)/self.r)) \n        self.ds_len =  self.length + (self.length // self.r) \n\n    def __iter__(self):\n        pos_index = self.pos_index.copy()\n        neg_index = self.neg_index.copy()\n        np.random.shuffle(pos_index)\n        np.random.shuffle(neg_index)\n\n        neg_index = neg_index[:self.length].reshape(-1,self.r)\n        #pos_index = np.random.choice(pos_index, self.length//self.r).reshape(-1,1)\n        pos_index = np.tile(pos_index, (len(neg_index) // len(pos_index)) + 1)[:len(neg_index)].reshape(-1,1)\n\n        index = np.concatenate([pos_index,neg_index],-1).reshape(-1)\n        return iter(index)\n\n    def __len__(self):\n        return self.ds_len\n    \nif DEBUG: \n    training_data = BreastCancerDataSet(train, TRAIN_IMAGES_PATH, get_transforms(crop=True))\n    data_loader = torch.utils.data.DataLoader(\n                          training_data,\n                          batch_size = 16,\n                          sampler = BalanceSampler(training_data, train, range(100)))\n    X, y = next(iter(data_loader))\n    \n    print(type(y))","metadata":{"execution":{"iopub.status.busy":"2023-02-09T10:46:38.391706Z","iopub.execute_input":"2023-02-09T10:46:38.393606Z","iopub.status.idle":"2023-02-09T10:46:38.406287Z","shell.execute_reply.started":"2023-02-09T10:46:38.393569Z","shell.execute_reply":"2023-02-09T10:46:38.405497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#https://www.reddit.com/r/MachineLearning/comments/kvs1ex/d_here_are_17_ways_of_making_pytorch_training/\nimport gc\n\ntorch.backends.cudnn.benchmark = True\n\ndef gc_collect():\n    gc.collect()\n    torch.cuda.empty_cache()\n\ndef train_one_epoch(train_loader, model, loss_function, optimizer, scaler):\n    # Train mode\n    model.train()\n    \n    # Track metrics\n    loss_epoch = 0\n    pfbeta_epoch = 0\n    thresholds = []\n    \n    # Loop over minibatches\n    for inputs, labels in tqdm(train_loader):\n        # Send to device\n        inputs = inputs.to(device, dtype = torch.float32, non_blocking=True)\n        labels = labels.to(device, dtype = torch.float32, non_blocking=True)\n                \n        # Forward pass\n        with torch.cuda.amp.autocast():\n            outputs = model(inputs)\n            loss = loss_function(outputs.flatten(), labels)\n            #loss = torch.nn.functional.binary_cross_entropy_with_logits(outputs.flatten(), labels).item()\n        \n        # Scales the loss, and calls backward() to create scaled gradients\n        scaler.scale(loss).backward()\n\n        # Unscales gradients and calls or skips optimizer.step()\n        scaler.step(optimizer)\n\n        # Updates the scale for next iteration\n        scaler.update()\n            \n        # Backprop\n        #loss.backward()\n\n        # Update parameters\n        #optimizer.step()\n\n        # Zero gradients\n        optimizer.zero_grad(set_to_none=True)\n\n        # Track loss\n        loss_epoch += loss.detach().item()\n        \n        # Competition's Metric\n        outputs = torch.sigmoid(outputs)\n\n        pf1, threshold = optimal_f1(labels, outputs)\n        pfbeta_epoch += pf1\n        thresholds.append(threshold)\n    \n    # We're gonna average all the thresholds obtained\n        #r = pfbeta_torch(labels, outputs)\n        #pfbeta_epoch += r\n        \n        del inputs, labels, pf1, threshold\n        gc_collect()\n        \n    # Update learning rate\n    #scheduler.step()\n        \n    threshold = np.mean(thresholds)        \n    return loss_epoch/len(train_loader), pfbeta_epoch/len(train_loader), threshold, model","metadata":{"_uuid":"cec87f08-65c4-4578-88c4-0a6d76117d1e","_cell_guid":"717e90de-4dbe-404a-9c1c-509d4613ada3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.407911Z","iopub.execute_input":"2023-02-09T10:46:38.408374Z","iopub.status.idle":"2023-02-09T10:46:38.422272Z","shell.execute_reply.started":"2023-02-09T10:46:38.408339Z","shell.execute_reply":"2023-02-09T10:46:38.42124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef validate_one_epoch(validation_loader, model, loss_function):\n    # Eval mode\n    model.eval()\n    \n    # Track metrics\n    loss_epoch = 0\n    pfbeta_epoch = 0\n    thresholds = []\n    val_predictions = []\n    val_labels = []\n    \n    # Don't update weights\n    with torch.no_grad():\n        # Loop over minibatches\n        for inputs, labels in tqdm(validation_loader):\n            val_labels.append(labels)\n            # Send to device\n            inputs = inputs.to(device, dtype = torch.float32, non_blocking=True)\n            labels = labels.to(device, dtype = torch.float32, non_blocking=True)\n            #labels = labels.to()\n            \n            # Forward pass\n            predictions = model.predict(inputs)            \n            loss = loss_function(predictions.flatten(), labels)\n            predictions = torch.sigmoid(predictions)\n            val_predictions.append(predictions)\n            \n            # Track loss\n            loss_epoch += loss.detach().item()\n            \n            # Competition's Metric\n            pf1, threshold = optimal_f1(labels, predictions)\n            pfbeta_epoch += pf1\n            thresholds.append(threshold)\n    \n    # We're gonna average all the thresholds obtained\n    threshold = np.mean(thresholds)\n    \n    val_predictions = torch.cat(val_predictions).flatten().cpu().numpy()\n    val_labels = torch.cat(val_labels).flatten().cpu().numpy()\n    \n    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(12,8))\n    sns.distplot(val_predictions, hist=True, color='orange', ax = axes)\n    sns.distplot(val_labels, hist=True, color='blue', ax = axes)\n    plt.show()\n    #wandb.log({\"Predictions vs Labels\": fig})\n    \n    # Calculate ROC\n    valid_roc = roc_auc_score(val_labels, val_predictions)\n    \n    return loss_epoch/len(validation_loader), pfbeta_epoch/len(validation_loader), threshold, valid_roc\n    #return loss_epoch/len(validation_loader), pfbeta_epoch/len(validation_loader), threshold","metadata":{"_uuid":"36778696-101b-4ff4-8399-9c6b2ec84c39","_cell_guid":"89b414e6-1aa4-4c75-b314-d94a26a6fd4b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.425556Z","iopub.execute_input":"2023-02-09T10:46:38.425833Z","iopub.status.idle":"2023-02-09T10:46:38.437583Z","shell.execute_reply.started":"2023-02-09T10:46:38.425808Z","shell.execute_reply":"2023-02-09T10:46:38.436487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor\n    \n#def train_model(model, loss_function, optimizer, scheduler, verbose=True):\ndef train_model(verbose=True):\n    torch.manual_seed(42)\n    # Init Weights and Biases \n    run = wandb.init(project=\"RSNA - Breast Cancer Detection\", entity=\"javigallego4\", group = 'Tuning')\n    \n    # Model and Dataset\n    model = BreastCancerModel(model_type=wandb.config.backbone)\n    model = model.to(device)\n    \n    # Freezing initial layers\n    NUM_FROZEN_LAYERS = int(len(list(model.named_parameters())) * wandb.config.freezing) # how many layers you want to freeze\n    for name, param in list(model.named_parameters())[0:NUM_FROZEN_LAYERS]:\n        param.requires_grad = False\n        \n    # Dataset    \n    training_data = BreastCancerDataSet(train, TRAIN_IMAGES_PATH)\n\n    # Loss function, optimizer and scheduler\n    loss_function = torch.nn.BCEWithLogitsLoss(pos_weight=torch.tensor([wandb.config.ratio]).to(device))\n    params = [p for p in model.parameters() if p.requires_grad]\n    optimizer = optim.AdamW(params, lr = wandb.config.lr)\n    scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.1, patience=2, threshold = 0.005, verbose = True)\n    scaler = torch.cuda.amp.GradScaler()\n\n    cv_val_pf1 = []\n    cv_val_roc = []\n    cv_val_loss = []\n    cv_val_threshold = []\n    \n    # Site 1 and 2 have different biases !!\n    for i in range(2): \n        train_idx = train[(train.VAL_FOLD != 0) & (train.site_id == i+1)].index\n        val_idx = train[(train.VAL_FOLD == 0) & (train.site_id == i+1)].index\n        \n        best_pf1 = 0\n        early_stopping = EarlyStopping(5, 0.005)\n        \n        print('====== Fold: {} ======='.format(i))\n        \n        # Loop over epochs\n        for epoch in range(wandb.config.epochs):\n            \n            trainloader = torch.utils.data.DataLoader(\n                          training_data, sampler = BalanceSampler(training_data, train_idx, wandb.config.ratio),  \n                          batch_size = 16, num_workers = 4, pin_memory=True)\n        \n            validationloader = torch.utils.data.DataLoader(\n                              training_data, sampler = SequentialSampler(val_idx), batch_size = 16, num_workers = 4, pin_memory=True)\n            \n            # Train\n            train_loss, train_pfbeta, train_threshold, model = train_one_epoch(trainloader, model, loss_function, optimizer, scaler)\n            \n            del trainloader\n            gc_collect()\n    \n            # Evaluate\n            val_loss, val_pf1, val_threshold, val_roc = validate_one_epoch(validationloader, model, loss_function)\n            \n            del validationloader\n            gc_collect()\n            \n            # Track metrics for site 1\n            wandb.log({\n                'epoch': epoch,\n                'train_loss': train_loss,\n                'val_loss': val_loss,\n                'train_metric': train_pfbeta,\n                'val_metric': val_pf1, \n                'val_threshold': val_threshold,\n                'val_roc_auc': val_roc\n            })\n            \n            # Apply scheduler\n            scheduler.step(val_loss)\n\n            # Print results\n            if verbose:\n                if (epoch+1)%1==0:\n                    print(f'\\nEpoch {epoch+1}/{wandb.config.epochs}')\n                    #print(f'Site 1: loss {train_loss_site1:.5f}, pfbeta {train_pfbeta_site1}, threshold {train_threshold_site1:.5f}, val_loss {val_loss_site1:.5f}, val_pfbeta {val_pf1_site1},  val_threshold {val_threshold_site1}, val_roc_auc {val_roc_site1}')\n                    #print(f'Site 2: loss {train_loss_site2:.5f}, pfbeta {train_pfbeta_site2}, threshold {train_threshold_site2:.5f}, val_loss {val_loss_site2:.5f}, val_pfbeta {val_pf1_site2},  val_threshold {val_threshold_site2}, val_roc_auc {val_roc_site2}')\n                    print(f': loss {train_loss:.5f}, pfbeta {train_pfbeta}, val_loss {val_loss:.5f}, val_pfbeta {val_pf1},  threshold {val_threshold}, val_roc_auc {val_roc}\\n')\n    \n            # Early Stopping\n            early_stopping(val_loss)\n            if early_stopping.early_stop:\n                break\n            else:\n                if val_pf1 > best_pf1: \n                    best_pf1 = val_pf1\n                    c_roc = val_roc\n                    c_threshold = val_threshold\n                    c_loss = val_loss\n                    \n                    val_loss, val_pf1, val_threshold, val_roc\n                    #best_pf1_site1 = val_pf1_site1\n                    best_model_state_dic = model.state_dict()\n        \n        cv_val_threshold.append(c_threshold)\n        cv_val_loss.append(c_loss)\n        cv_val_roc.append(c_roc)\n        cv_val_pf1.append(best_pf1)\n        \n        wandb.log({\n            'fold': i,\n            'best_pf1': best_pf1,\n            'best_threshold': c_threshold,\n            'best_roc_auc': c_roc,\n        })\n        \n        PATH = \"{}_fold_{}.pt\".format(wandb.run.name, i)\n        torch.save(best_model_state_dic, PATH)\n    \n    averaged_pf1 = cv_val_pf1[0] * 0.57 + cv_val_pf1[1] * 0.43\n    averaged_loss = cv_val_loss[0] * 0.57 + cv_val_loss[1] * 0.43\n    averaged_roc = cv_val_roc[0] * 0.57 + cv_val_roc[1] * 0.43\n    averaged_thr = cv_val_threshold[0] * 0.57 + cv_val_threshold[1] * 0.43\n    \n    wandb.log({\n        'cv_averaged_pf1': averaged_pf1,\n        'cv_averaged_loss': averaged_loss,\n        'cv_averaged_roc': averaged_roc, \n        'cv_averaged_threshold': averaged_thr\n    })\n    \n    PATH = \"{}.pt\".format(wandb.run.name)\n    torch.save(best_model_state_dic, PATH)\n    \n    run.finish()","metadata":{"_uuid":"05222b40-8f0e-4f17-ade2-7e3d1246054b","_cell_guid":"80edc1ad-4ea8-4880-9bd9-d8dcd09a3344","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.439168Z","iopub.execute_input":"2023-02-09T10:46:38.43967Z","iopub.status.idle":"2023-02-09T10:46:38.461938Z","shell.execute_reply.started":"2023-02-09T10:46:38.439635Z","shell.execute_reply":"2023-02-09T10:46:38.460959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Wandb Sweeps","metadata":{"_uuid":"07c25ac3-9f48-4cde-87f6-c09d26ca6567","_cell_guid":"4bbb30d4-24d0-4b21-bc6e-dff61e8a0ceb","trusted":true}},{"cell_type":"code","source":"%env SWEEP_ID=zhl1n4jx\n#os.environ.__delitem__('SWEEP_ID')","metadata":{"_uuid":"390ce307-5f3b-41ca-9270-739d6634fa5f","_cell_guid":"a64d5484-2348-4d9a-8bd7-02cf47c92254","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.463381Z","iopub.execute_input":"2023-02-09T10:46:38.463851Z","iopub.status.idle":"2023-02-09T10:46:38.474869Z","shell.execute_reply.started":"2023-02-09T10:46:38.463812Z","shell.execute_reply":"2023-02-09T10:46:38.473319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sweep_id = os.environ.get('SWEEP_ID')\nprint('wandb sweep ', sweep_id)\n\nif sweep_id is None:\n    # Define the sweep configuration\n    sweep_id = wandb.sweep(sweep={\n            'method': 'bayes',\n            'name': 'EfficientNet + Linear',\n            'metric': {'goal': 'maximize', 'name': 'cv_averaged_pf1'},\n            'parameters':\n                {\n                    'lr': {'values': [1e-03, 1e-04]},  # Learning rate\n                    #'pos_weight': {'values': [46]},  # Positive weight -> No hace falta, nos sirve con el mismo ratio\n                    'ratio': {'distribution': 'int_uniform', \"min\": 3, \"max\": 10},  # Sampler's ratio\n                    #'ratio': {'values': [50]},\n                    'batch_size' : {'values': [32]}, # Batch size\n                    'epochs': {\"min\": 10, \"max\": 20},\n                    'backbone': {'values': ['efficientnet_b4', 'efficientnet_b3', 'efficientnet_b2', 'efficientnet_b0']},\n                    #'backbone': {'values': ['inception_resnet_v2', 'inception_v3','inception_v4']},\n                    'image_size': {'values': [(2048, 1024)]},\n                    'grouping_by': {'values': ['single_image']},\n                    'folds': {'values': [2]},\n                    'freezing': {'values': [0.75]}\n                }\n        }, project=\"RSNA - Breast Cancer Detection\")\n\n    print('Generated sweep id', sweep_id)\n    \n    # Create the sweep using the wandb.sweep function\n    #sweep_id = wandb.sweep(sweep = sweep_config, project=\"RSNA - Breast Cancer Detection\")\n\nelse:\n    \"\"\"\n    Agent run. Use sweep_id generated above to produce (semi)-random hyperparameters run.config\n    \"\"\"\n    wandb.agent(sweep_id, function=train_model, entity=\"javigallego4\", project=\"RSNA - Breast Cancer Detection\", count = 1)","metadata":{"_uuid":"aaf78eb9-b3bb-4f01-8603-b9b1098c5347","_cell_guid":"db5824b0-e7d7-4090-bebf-584e3eda5b16","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-02-09T10:46:38.477367Z","iopub.execute_input":"2023-02-09T10:46:38.478422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}