{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"script is from: https://www.kaggle.com/code/abhimanyukarshni/rsna-training/notebook","metadata":{"papermill":{"duration":0.025318,"end_time":"2021-10-22T13:34:33.370467","exception":false,"start_time":"2021-10-22T13:34:33.345149","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Data Loading","metadata":{"papermill":{"duration":0.022883,"end_time":"2021-10-22T13:34:33.416571","exception":false,"start_time":"2021-10-22T13:34:33.393688","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nos.environ['CUDA_VISIBLE_DEVICES'] = '0' # specify GPUs locally\n\nimport pandas as pd\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.768873,"end_time":"2021-10-22T13:34:34.208523","exception":false,"start_time":"2021-10-22T13:34:33.43965","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:52.417106Z","iopub.execute_input":"2022-12-03T07:51:52.418123Z","iopub.status.idle":"2022-12-03T07:51:53.315111Z","shell.execute_reply.started":"2022-12-03T07:51:52.418021Z","shell.execute_reply":"2022-12-03T07:51:53.31414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/rsna-breast-cancer-detection/train.csv')\n\n#train['image_name'] = train.apply(lambda x: f'{x[\"patient_id\"]}_{x[\"image_id\"]}.png', axis=1)\ntrain['image_name'] = train.apply(lambda x: f'{x[\"patient_id\"]}/{x[\"image_id\"]}.png', axis=1)\nprint(train.shape)\ntrain.head()","metadata":{"papermill":{"duration":0.052886,"end_time":"2021-10-22T13:34:34.284998","exception":false,"start_time":"2021-10-22T13:34:34.232112","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:53.319133Z","iopub.execute_input":"2022-12-03T07:51:53.319435Z","iopub.status.idle":"2022-12-03T07:51:54.438342Z","shell.execute_reply.started":"2022-12-03T07:51:53.319409Z","shell.execute_reply":"2022-12-03T07:51:54.43741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Directory settings","metadata":{"papermill":{"duration":0.023245,"end_time":"2021-10-22T13:34:34.332157","exception":false,"start_time":"2021-10-22T13:34:34.308912","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Directory settings\n# ====================================================\nimport os\n\nOUTPUT_DIR = './'\nif not os.path.exists(OUTPUT_DIR):\n    os.makedirs(OUTPUT_DIR)\n\nTRAIN_PATH = '/kaggle/input/rsna-mammography-images-as-pngs/images_as_pngs_512/train_images_processed_512'\nTEST_PATH = '/kaggle/input/rsna-breast-cancer-detection/test_images'","metadata":{"papermill":{"duration":0.031437,"end_time":"2021-10-22T13:34:34.387401","exception":false,"start_time":"2021-10-22T13:34:34.355964","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:54.442005Z","iopub.execute_input":"2022-12-03T07:51:54.444467Z","iopub.status.idle":"2022-12-03T07:51:54.452517Z","shell.execute_reply.started":"2022-12-03T07:51:54.444427Z","shell.execute_reply":"2022-12-03T07:51:54.451372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CFG","metadata":{"papermill":{"duration":0.024036,"end_time":"2021-10-22T13:34:34.434931","exception":false,"start_time":"2021-10-22T13:34:34.410895","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# CFG\n# ====================================================\nclass CFG:\n    debug=False\n    apex=False\n    print_freq=500\n    num_workers=4\n    model_name='efficientnet_b3'\n    size=512\n    scheduler='CosineAnnealingWarmRestarts' # ['ReduceLROnPlateau', 'CosineAnnealingLR', 'CosineAnnealingWarmRestarts']\n    criterion='TaylorCrossEntropyLoss' # ['CrossEntropyLoss', LabelSmoothing', 'FocalLoss' 'FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    epochs=10\n    T_0=10 # CosineAnnealingWarmRestarts\n    lr=1e-4\n    min_lr=1e-8\n    batch_size=8\n    weight_decay=1e-6\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=42\n    target_size=2\n    target_col='cancer'\n    n_fold=5\n    trn_fold=[0,1,2, 3, 4]\n    train=True\n    smoothing=0.0\n    t1=0.3 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    t2=1.0 # bi-tempered-loss https://www.kaggle.com/c/cassava-leaf-disease-classification/discussion/202017\n    \nif CFG.debug:\n    CFG.epochs = 1\n    train = train.sample(n=1000, random_state=CFG.seed).reset_index(drop=True)","metadata":{"papermill":{"duration":0.0354,"end_time":"2021-10-22T13:34:34.493795","exception":false,"start_time":"2021-10-22T13:34:34.458395","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:54.459707Z","iopub.execute_input":"2022-12-03T07:51:54.460428Z","iopub.status.idle":"2022-12-03T07:51:54.473926Z","shell.execute_reply.started":"2022-12-03T07:51:54.460389Z","shell.execute_reply":"2022-12-03T07:51:54.472809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{"papermill":{"duration":0.023328,"end_time":"2021-10-22T13:34:34.540531","exception":false,"start_time":"2021-10-22T13:34:34.517203","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Imports\n# ====================================================\nimport sys\nsys.path.append('/kaggle/input/pytorch-image-models-timm')\n\nimport os\nimport math\nimport time\nimport random\nimport shutil\nfrom pathlib import Path\nfrom contextlib import contextmanager\nfrom collections import defaultdict, Counter\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn import preprocessing\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tqdm.auto import tqdm\nfrom functools import partial\n\nimport cv2\nfrom PIL import Image\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD\nimport torchvision.models as models\nfrom torch.nn.parameter import Parameter\nfrom torch.utils.data import DataLoader, Dataset, WeightedRandomSampler\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom albumentations import ImageOnlyTransform\n\nimport timm\n\nimport warnings \nwarnings.filterwarnings('ignore')\n\nif CFG.apex:\n    #from apex import amp\n    from torch.cuda.amp import autocast, GradScaler\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"papermill":{"duration":4.116566,"end_time":"2021-10-22T13:34:38.680625","exception":false,"start_time":"2021-10-22T13:34:34.564059","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:54.479712Z","iopub.execute_input":"2022-12-03T07:51:54.482497Z","iopub.status.idle":"2022-12-03T07:51:59.397216Z","shell.execute_reply.started":"2022-12-03T07:51:54.482462Z","shell.execute_reply":"2022-12-03T07:51:59.39588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utils","metadata":{"papermill":{"duration":0.024132,"end_time":"2021-10-22T13:34:38.728766","exception":false,"start_time":"2021-10-22T13:34:38.704634","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\ndef get_score(y_true, y_pred):\n    return roc_auc_score(y_true, y_pred)\n\n\n@contextmanager\ndef timer(name):\n    t0 = time.time()\n    LOGGER.info(f'[{name}] start')\n    yield\n    LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')\n\n\ndef init_logger(log_file=OUTPUT_DIR+'train.log'):\n    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=log_file)\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = init_logger()\n\n\ndef seed_torch(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\nseed_torch(seed=CFG.seed)","metadata":{"papermill":{"duration":0.041397,"end_time":"2021-10-22T13:34:38.827468","exception":false,"start_time":"2021-10-22T13:34:38.786071","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.399404Z","iopub.execute_input":"2022-12-03T07:51:59.400279Z","iopub.status.idle":"2022-12-03T07:51:59.415567Z","shell.execute_reply.started":"2022-12-03T07:51:59.400238Z","shell.execute_reply":"2022-12-03T07:51:59.414581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV split","metadata":{"papermill":{"duration":0.023301,"end_time":"2021-10-22T13:34:38.874349","exception":false,"start_time":"2021-10-22T13:34:38.851048","status":"completed"},"tags":[]}},{"cell_type":"code","source":"folds = train.copy()\nFold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.seed)\nfor n, (train_index, val_index) in enumerate(Fold.split(folds, folds[CFG.target_col])):\n    folds.loc[val_index, 'fold'] = int(n)\nfolds['fold'] = folds['fold'].astype(int)\nprint(folds.groupby(['fold', CFG.target_col]).size())","metadata":{"papermill":{"duration":0.060588,"end_time":"2021-10-22T13:34:38.974662","exception":false,"start_time":"2021-10-22T13:34:38.914074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.420097Z","iopub.execute_input":"2022-12-03T07:51:59.423405Z","iopub.status.idle":"2022-12-03T07:51:59.484903Z","shell.execute_reply.started":"2022-12-03T07:51:59.422877Z","shell.execute_reply":"2022-12-03T07:51:59.484053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"papermill":{"duration":0.023699,"end_time":"2021-10-22T13:34:39.02232","exception":false,"start_time":"2021-10-22T13:34:38.998621","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, df, transform=None):\n        self.df = df\n        self.file_names = df['image_name'].values\n        self.labels = df['cancer'].values\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        file_path = f'{TRAIN_PATH}/{self.file_names[idx]}' #image path\n        #print(file_path, Path(file_path).exists())\n        image = cv2.imread(file_path)[:,:,0]\n        if self.transform:\n            augmented = self.transform(image=image)\n            image = augmented['image']\n            image = image.float()\n        label = torch.tensor(self.labels[idx]).long()\n        return image, label","metadata":{"papermill":{"duration":0.038404,"end_time":"2021-10-22T13:34:39.084478","exception":false,"start_time":"2021-10-22T13:34:39.046074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.488379Z","iopub.execute_input":"2022-12-03T07:51:59.488714Z","iopub.status.idle":"2022-12-03T07:51:59.497433Z","shell.execute_reply.started":"2022-12-03T07:51:59.488682Z","shell.execute_reply":"2022-12-03T07:51:59.496392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=None)\n\nfor i in range(1):\n    image, label = train_dataset[i]\n    plt.imshow(image, cmap='gray')\n    plt.title(f'label: {label}')\n    plt.show() ","metadata":{"papermill":{"duration":1.281925,"end_time":"2021-10-22T13:34:40.390284","exception":false,"start_time":"2021-10-22T13:34:39.108359","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.500248Z","iopub.execute_input":"2022-12-03T07:51:59.500945Z","iopub.status.idle":"2022-12-03T07:51:59.777502Z","shell.execute_reply.started":"2022-12-03T07:51:59.500909Z","shell.execute_reply":"2022-12-03T07:51:59.776289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforms","metadata":{"papermill":{"duration":0.044185,"end_time":"2021-10-22T13:34:40.483494","exception":false,"start_time":"2021-10-22T13:34:40.439309","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Transforms\n# ====================================================\ndef get_transforms(*, data):\n    \n    if data == 'train':\n        return A.Compose([\n            ToTensorV2(),\n        ])\n\n    elif data == 'valid':\n        return A.Compose([\n            ToTensorV2(),\n        ])","metadata":{"papermill":{"duration":0.052814,"end_time":"2021-10-22T13:34:40.579603","exception":false,"start_time":"2021-10-22T13:34:40.526789","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.781746Z","iopub.execute_input":"2022-12-03T07:51:59.782019Z","iopub.status.idle":"2022-12-03T07:51:59.787466Z","shell.execute_reply.started":"2022-12-03T07:51:59.781993Z","shell.execute_reply":"2022-12-03T07:51:59.786397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = TrainDataset(train, transform=get_transforms(data='train'))\n\nfor i in range(1):\n    image, label = train_dataset[i]\n    print(image.shape)\n    plt.imshow(image[0], cmap='gray')\n    plt.title(f'label: {label}')\n    plt.show() ","metadata":{"papermill":{"duration":0.90408,"end_time":"2021-10-22T13:34:41.524556","exception":false,"start_time":"2021-10-22T13:34:40.620476","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:51:59.789177Z","iopub.execute_input":"2022-12-03T07:51:59.789962Z","iopub.status.idle":"2022-12-03T07:52:00.01327Z","shell.execute_reply.started":"2022-12-03T07:51:59.789925Z","shell.execute_reply":"2022-12-03T07:52:00.012351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# MODEL\n# ====================================================\nclass CustomEfficientNet(nn.Module):\n    def __init__(self, model_name=CFG.model_name, pretrained=False):\n        super().__init__()\n        #self.conv = nn.Conv2d(4,3,1)\n        self.conv = nn.Conv2d(1,3,1)\n        self.model = timm.create_model(CFG.model_name, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, CFG.target_size)\n\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.model(x)\n        return x\n    ","metadata":{"papermill":{"duration":0.035912,"end_time":"2021-10-22T13:34:41.587317","exception":false,"start_time":"2021-10-22T13:34:41.551405","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.014537Z","iopub.execute_input":"2022-12-03T07:52:00.014904Z","iopub.status.idle":"2022-12-03T07:52:00.02193Z","shell.execute_reply.started":"2022-12-03T07:52:00.014867Z","shell.execute_reply":"2022-12-03T07:52:00.020764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Criterion","metadata":{"papermill":{"duration":0.025741,"end_time":"2021-10-22T13:34:41.638819","exception":false,"start_time":"2021-10-22T13:34:41.613078","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Label Smoothing","metadata":{"papermill":{"duration":0.025717,"end_time":"2021-10-22T13:34:41.690664","exception":false,"start_time":"2021-10-22T13:34:41.664947","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Label Smoothing\n# ====================================================\nclass LabelSmoothingLoss(nn.Module): \n    def __init__(self, classes=5, smoothing=0.0, dim=-1): \n        super(LabelSmoothingLoss, self).__init__() \n        self.confidence = 1.0 - smoothing \n        self.smoothing = smoothing \n        self.cls = classes \n        self.dim = dim \n    def forward(self, pred, target): \n        pred = pred.log_softmax(dim=self.dim) \n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred) \n            true_dist.fill_(self.smoothing / (self.cls - 1)) \n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence) \n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))","metadata":{"papermill":{"duration":0.038079,"end_time":"2021-10-22T13:34:41.754981","exception":false,"start_time":"2021-10-22T13:34:41.716902","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.023484Z","iopub.execute_input":"2022-12-03T07:52:00.02413Z","iopub.status.idle":"2022-12-03T07:52:00.034073Z","shell.execute_reply.started":"2022-12-03T07:52:00.024095Z","shell.execute_reply":"2022-12-03T07:52:00.033182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Focal Loss","metadata":{"papermill":{"duration":0.026525,"end_time":"2021-10-22T13:34:41.807867","exception":false,"start_time":"2021-10-22T13:34:41.781342","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduce=True):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduce = reduce\n\n    def forward(self, inputs, targets):\n        BCE_loss = nn.CrossEntropyLoss()(inputs, targets)\n\n        pt = torch.exp(-BCE_loss)\n        F_loss = self.alpha * (1-pt)**self.gamma * BCE_loss\n\n        if self.reduce:\n            return torch.mean(F_loss)\n        else:\n            return F_loss","metadata":{"papermill":{"duration":0.035539,"end_time":"2021-10-22T13:34:41.869161","exception":false,"start_time":"2021-10-22T13:34:41.833622","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.035596Z","iopub.execute_input":"2022-12-03T07:52:00.036017Z","iopub.status.idle":"2022-12-03T07:52:00.043877Z","shell.execute_reply.started":"2022-12-03T07:52:00.035984Z","shell.execute_reply":"2022-12-03T07:52:00.04274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## FocalCosineLoss","metadata":{"papermill":{"duration":0.025717,"end_time":"2021-10-22T13:34:41.920667","exception":false,"start_time":"2021-10-22T13:34:41.89495","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class FocalCosineLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, xent=.1):\n        super(FocalCosineLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n\n        self.xent = xent\n\n        self.y = torch.Tensor([1]).cuda()\n\n    def forward(self, input, target, reduction=\"mean\"):\n        cosine_loss = F.cosine_embedding_loss(input, F.one_hot(target, num_classes=input.size(-1)), self.y, reduction=reduction)\n\n        cent_loss = F.cross_entropy(F.normalize(input), target, reduce=False)\n        pt = torch.exp(-cent_loss)\n        focal_loss = self.alpha * (1-pt)**self.gamma * cent_loss\n\n        if reduction == \"mean\":\n            focal_loss = torch.mean(focal_loss)\n\n        return cosine_loss + self.xent * focal_loss","metadata":{"papermill":{"duration":0.038252,"end_time":"2021-10-22T13:34:41.984812","exception":false,"start_time":"2021-10-22T13:34:41.94656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.045506Z","iopub.execute_input":"2022-12-03T07:52:00.045908Z","iopub.status.idle":"2022-12-03T07:52:00.054792Z","shell.execute_reply.started":"2022-12-03T07:52:00.045812Z","shell.execute_reply":"2022-12-03T07:52:00.053711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## SymmetricCrossEntropy","metadata":{"papermill":{"duration":0.025734,"end_time":"2021-10-22T13:34:42.037022","exception":false,"start_time":"2021-10-22T13:34:42.011288","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class SymmetricCrossEntropy(nn.Module):\n\n    def __init__(self, alpha=0.1, beta=1.0, num_classes=5):\n        super(SymmetricCrossEntropy, self).__init__()\n        self.alpha = alpha\n        self.beta = beta\n        self.num_classes = num_classes\n\n    def forward(self, logits, targets, reduction='mean'):\n        onehot_targets = torch.eye(self.num_classes)[targets].cuda()\n        ce_loss = F.cross_entropy(logits, targets, reduction=reduction)\n        rce_loss = (-onehot_targets*logits.softmax(1).clamp(1e-7, 1.0).log()).sum(1)\n        if reduction == 'mean':\n            rce_loss = rce_loss.mean()\n        elif reduction == 'sum':\n            rce_loss = rce_loss.sum()\n        return self.alpha * ce_loss + self.beta * rce_loss","metadata":{"papermill":{"duration":0.037844,"end_time":"2021-10-22T13:34:42.100761","exception":false,"start_time":"2021-10-22T13:34:42.062917","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.056481Z","iopub.execute_input":"2022-12-03T07:52:00.056886Z","iopub.status.idle":"2022-12-03T07:52:00.06658Z","shell.execute_reply.started":"2022-12-03T07:52:00.056853Z","shell.execute_reply":"2022-12-03T07:52:00.06581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Bi-Tempered-Loss","metadata":{"papermill":{"duration":0.025762,"end_time":"2021-10-22T13:34:42.15263","exception":false,"start_time":"2021-10-22T13:34:42.126868","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def log_t(u, t):\n    \"\"\"Compute log_t for `u'.\"\"\"\n    if t==1.0:\n        return u.log()\n    else:\n        return (u.pow(1.0 - t) - 1.0) / (1.0 - t)\n\ndef exp_t(u, t):\n    \"\"\"Compute exp_t for `u'.\"\"\"\n    if t==1:\n        return u.exp()\n    else:\n        return (1.0 + (1.0-t)*u).relu().pow(1.0 / (1.0 - t))\n\ndef compute_normalization_fixed_point(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t > 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same shape as activation with the last dimension being 1.\n    \"\"\"\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations_step_0 = activations - mu\n\n    normalized_activations = normalized_activations_step_0\n\n    for _ in range(num_iters):\n        logt_partition = torch.sum(\n                exp_t(normalized_activations, t), -1, keepdim=True)\n        normalized_activations = normalized_activations_step_0 * \\\n                logt_partition.pow(1.0-t)\n\n    logt_partition = torch.sum(\n            exp_t(normalized_activations, t), -1, keepdim=True)\n    normalization_constants = - log_t(1.0 / logt_partition, t) + mu\n\n    return normalization_constants\n\ndef compute_normalization_binary_search(activations, t, num_iters):\n\n    \"\"\"Returns the normalization value for each example (t < 1.0).\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (< 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n\n    mu, _ = torch.max(activations, -1, keepdim=True)\n    normalized_activations = activations - mu\n\n    effective_dim = \\\n        torch.sum(\n                (normalized_activations > -1.0 / (1.0-t)).to(torch.int32),\n            dim=-1, keepdim=True).to(activations.dtype)\n\n    shape_partition = activations.shape[:-1] + (1,)\n    lower = torch.zeros(shape_partition, dtype=activations.dtype, device=activations.device)\n    upper = -log_t(1.0/effective_dim, t) * torch.ones_like(lower)\n\n    for _ in range(num_iters):\n        logt_partition = (upper + lower)/2.0\n        sum_probs = torch.sum(\n                exp_t(normalized_activations - logt_partition, t),\n                dim=-1, keepdim=True)\n        update = (sum_probs < 1.0).to(activations.dtype)\n        lower = torch.reshape(\n                lower * update + (1.0-update) * logt_partition,\n                shape_partition)\n        upper = torch.reshape(\n                upper * (1.0 - update) + update * logt_partition,\n                shape_partition)\n\n    logt_partition = (upper + lower)/2.0\n    return logt_partition + mu\n\nclass ComputeNormalization(torch.autograd.Function):\n    \"\"\"\n    Class implementing custom backward pass for compute_normalization. See compute_normalization.\n    \"\"\"\n    @staticmethod\n    def forward(ctx, activations, t, num_iters):\n        if t < 1.0:\n            normalization_constants = compute_normalization_binary_search(activations, t, num_iters)\n        else:\n            normalization_constants = compute_normalization_fixed_point(activations, t, num_iters)\n\n        ctx.save_for_backward(activations, normalization_constants)\n        ctx.t=t\n        return normalization_constants\n\n    @staticmethod\n    def backward(ctx, grad_output):\n        activations, normalization_constants = ctx.saved_tensors\n        t = ctx.t\n        normalized_activations = activations - normalization_constants \n        probabilities = exp_t(normalized_activations, t)\n        escorts = probabilities.pow(t)\n        escorts = escorts / escorts.sum(dim=-1, keepdim=True)\n        grad_input = escorts * grad_output\n        \n        return grad_input, None, None\n\ndef compute_normalization(activations, t, num_iters=5):\n    \"\"\"Returns the normalization value for each example. \n    Backward pass is implemented.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      num_iters: Number of iterations to run the method.\n    Return: A tensor of same rank as activation with the last dimension being 1.\n    \"\"\"\n    return ComputeNormalization.apply(activations, t, num_iters)\n\ndef tempered_sigmoid(activations, t, num_iters = 5):\n    \"\"\"Tempered sigmoid function.\n    Args:\n      activations: Activations for the positive class for binary classification.\n      t: Temperature tensor > 0.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_probabilities = tempered_softmax(internal_activations, t, num_iters)\n    return internal_probabilities[..., 0]\n\n\ndef tempered_softmax(activations, t, num_iters=5):\n    \"\"\"Tempered softmax function.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      t: Temperature > 1.0.\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A probabilities tensor.\n    \"\"\"\n    if t == 1.0:\n        return activations.softmax(dim=-1)\n\n    normalization_constants = compute_normalization(activations, t, num_iters)\n    return exp_t(activations - normalization_constants, t)\n\ndef bi_tempered_binary_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing = 0.0,\n        num_iters=5,\n        reduction='mean'):\n\n    \"\"\"Bi-Tempered binary logistic loss.\n    Args:\n      activations: A tensor containing activations for class 1.\n      labels: A tensor with shape as activations, containing probabilities for class 1\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing\n      num_iters: Number of iterations to run the method.\n    Returns:\n      A loss tensor.\n    \"\"\"\n    internal_activations = torch.stack([activations,\n        torch.zeros_like(activations)],\n        dim=-1)\n    internal_labels = torch.stack([labels.to(activations.dtype),\n        1.0 - labels.to(activations.dtype)],\n        dim=-1)\n    return bi_tempered_logistic_loss(internal_activations, \n            internal_labels,\n            t1,\n            t2,\n            label_smoothing = label_smoothing,\n            num_iters = num_iters,\n            reduction = reduction)\n\ndef bi_tempered_logistic_loss(activations,\n        labels,\n        t1,\n        t2,\n        label_smoothing=0.0,\n        num_iters=5,\n        reduction = 'mean'):\n\n    \"\"\"Bi-Tempered Logistic Loss.\n    Args:\n      activations: A multi-dimensional tensor with last dimension `num_classes`.\n      labels: A tensor with shape and dtype as activations (onehot), \n        or a long tensor of one dimension less than activations (pytorch standard)\n      t1: Temperature 1 (< 1.0 for boundedness).\n      t2: Temperature 2 (> 1.0 for tail heaviness, < 1.0 for finite support).\n      label_smoothing: Label smoothing parameter between [0, 1). Default 0.0.\n      num_iters: Number of iterations to run the method. Default 5.\n      reduction: ``'none'`` | ``'mean'`` | ``'sum'``. Default ``'mean'``.\n        ``'none'``: No reduction is applied, return shape is shape of\n        activations without the last dimension.\n        ``'mean'``: Loss is averaged over minibatch. Return shape (1,)\n        ``'sum'``: Loss is summed over minibatch. Return shape (1,)\n    Returns:\n      A loss tensor.\n    \"\"\"\n\n    if len(labels.shape)<len(activations.shape): #not one-hot\n        labels_onehot = torch.zeros_like(activations)\n        labels_onehot.scatter_(1, labels[..., None], 1)\n    else:\n        labels_onehot = labels\n\n    if label_smoothing > 0:\n        num_classes = labels_onehot.shape[-1]\n        labels_onehot = ( 1 - label_smoothing * num_classes / (num_classes - 1) ) \\\n                * labels_onehot + \\\n                label_smoothing / (num_classes - 1)\n\n    probabilities = tempered_softmax(activations, t2, num_iters)\n\n    loss_values = labels_onehot * log_t(labels_onehot + 1e-10, t1) \\\n            - labels_onehot * log_t(probabilities, t1) \\\n            - labels_onehot.pow(2.0 - t1) / (2.0 - t1) \\\n            + probabilities.pow(2.0 - t1) / (2.0 - t1)\n    loss_values = loss_values.sum(dim = -1) #sum over classes\n\n    if reduction == 'none':\n        return loss_values\n    if reduction == 'sum':\n        return loss_values.sum()\n    if reduction == 'mean':\n        return loss_values.mean()","metadata":{"papermill":{"duration":0.068889,"end_time":"2021-10-22T13:34:42.247428","exception":false,"start_time":"2021-10-22T13:34:42.178539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.06873Z","iopub.execute_input":"2022-12-03T07:52:00.069588Z","iopub.status.idle":"2022-12-03T07:52:00.096906Z","shell.execute_reply.started":"2022-12-03T07:52:00.069553Z","shell.execute_reply":"2022-12-03T07:52:00.096032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BiTemperedLogisticLoss(nn.Module): \n    def __init__(self, t1, t2, smoothing=0.0): \n        super(BiTemperedLogisticLoss, self).__init__() \n        self.t1 = t1\n        self.t2 = t2\n        self.smoothing = smoothing\n    def forward(self, logit_label, truth_label):\n        loss_label = bi_tempered_logistic_loss(\n            logit_label, truth_label,\n            t1=self.t1, t2=self.t2,\n            label_smoothing=self.smoothing,\n            reduction='none'\n        )\n        \n        loss_label = loss_label.mean()\n        return loss_label","metadata":{"papermill":{"duration":0.034966,"end_time":"2021-10-22T13:34:42.308334","exception":false,"start_time":"2021-10-22T13:34:42.273368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.098503Z","iopub.execute_input":"2022-12-03T07:52:00.09892Z","iopub.status.idle":"2022-12-03T07:52:00.111434Z","shell.execute_reply.started":"2022-12-03T07:52:00.098879Z","shell.execute_reply":"2022-12-03T07:52:00.10987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TaylorCrossEntropyLoss","metadata":{"papermill":{"duration":0.025974,"end_time":"2021-10-22T13:34:42.360216","exception":false,"start_time":"2021-10-22T13:34:42.334242","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TaylorSoftmax(nn.Module):\n    '''\n    This is the autograd version\n    '''\n    def __init__(self, dim=1, n=2):\n        super(TaylorSoftmax, self).__init__()\n        assert n % 2 == 0\n        self.dim = dim\n        self.n = n\n\n    def forward(self, x):\n        '''\n        usage similar to nn.Softmax:\n            >>> mod = TaylorSoftmax(dim=1, n=4)\n            >>> inten = torch.randn(1, 32, 64, 64)\n            >>> out = mod(inten)\n        '''\n        fn = torch.ones_like(x)\n        denor = 1.\n        for i in range(1, self.n+1):\n            denor *= i\n            fn = fn + x.pow(i) / denor\n        out = fn / fn.sum(dim=self.dim, keepdims=True)\n        return out\n\n    \nclass TaylorCrossEntropyLoss(nn.Module):\n    def __init__(self, n=2, ignore_index=-1, reduction='mean', smoothing=0.05):\n        super(TaylorCrossEntropyLoss, self).__init__()\n        assert n % 2 == 0\n        self.taylor_softmax = TaylorSoftmax(dim=1, n=n)\n        self.reduction = reduction\n        self.ignore_index = ignore_index\n        self.lab_smooth = LabelSmoothingLoss(CFG.target_size, smoothing=smoothing)\n\n    def forward(self, logits, labels):\n        log_probs = self.taylor_softmax(logits).log()\n        #loss = F.nll_loss(log_probs, labels, reduction=self.reduction,\n        #        ignore_index=self.ignore_index)\n        loss = self.lab_smooth(log_probs, labels)\n        return loss","metadata":{"papermill":{"duration":0.04033,"end_time":"2021-10-22T13:34:42.426839","exception":false,"start_time":"2021-10-22T13:34:42.386509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.113528Z","iopub.execute_input":"2022-12-03T07:52:00.113932Z","iopub.status.idle":"2022-12-03T07:52:00.130384Z","shell.execute_reply.started":"2022-12-03T07:52:00.113894Z","shell.execute_reply":"2022-12-03T07:52:00.129479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper functions","metadata":{"papermill":{"duration":0.025684,"end_time":"2021-10-22T13:34:42.478332","exception":false,"start_time":"2021-10-22T13:34:42.452648","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"papermill":{"duration":0.033083,"end_time":"2021-10-22T13:34:42.537432","exception":false,"start_time":"2021-10-22T13:34:42.504349","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.134996Z","iopub.execute_input":"2022-12-03T07:52:00.137609Z","iopub.status.idle":"2022-12-03T07:52:00.143532Z","shell.execute_reply.started":"2022-12-03T07:52:00.137574Z","shell.execute_reply":"2022-12-03T07:52:00.142607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to train mode\n    model.train()\n    start = end = time.time()\n    global_step = 0\n    for step, (images, labels) in tqdm(enumerate(train_loader), total = len(train_loader)):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        \n        losses.update(loss.item(), batch_size)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        if CFG.apex:\n            with amp.scale_loss(loss, optimizer) as scaled_loss:\n                scaled_loss.backward()\n        else:\n            loss.backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            optimizer.step()\n            optimizer.zero_grad()\n            global_step += 1\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  #'LR: {lr:.6f}  '\n                  .format(\n                   epoch+1, step, len(train_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(train_loader)),\n                   grad_norm=grad_norm,\n                   #lr=scheduler.get_lr()[0],\n                   ))\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    scores = AverageMeter()\n    # switch to evaluation mode\n    model.eval()\n    preds = []\n    start = end = time.time()\n    for step, (images, labels) in enumerate(valid_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n        images = images.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        # compute loss\n        with torch.no_grad():\n            y_preds = model(images)\n        loss = criterion(y_preds, labels)\n        losses.update(loss.item(), batch_size)\n        # record accuracy\n        preds.append(y_preds.softmax(1).to('cpu').numpy())\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Data {data_time.val:.3f} ({data_time.avg:.3f}) '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(\n                   step, len(valid_loader), batch_time=batch_time,\n                   data_time=data_time, loss=losses,\n                   remain=timeSince(start, float(step+1)/len(valid_loader)),\n                   ))\n    predictions = np.concatenate(preds)\n    return losses.avg, predictions\n\n\ndef inference(model, states, test_loader, device):\n    model.to(device)\n    tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n    probs = []\n    for i, (images) in tk0:\n        images = images.to(device)\n        avg_preds = []\n        for state in states:\n            model.load_state_dict(state['model'])\n            model.eval()\n            with torch.no_grad():\n                y_preds = model(images)\n            avg_preds.append(y_preds.softmax(1).to('cpu').numpy())\n        avg_preds = np.mean(avg_preds, axis=0)\n        probs.append(avg_preds)\n    probs = np.concatenate(probs)\n    return probs","metadata":{"papermill":{"duration":0.062746,"end_time":"2021-10-22T13:34:42.626548","exception":false,"start_time":"2021-10-22T13:34:42.563802","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.148339Z","iopub.execute_input":"2022-12-03T07:52:00.150816Z","iopub.status.idle":"2022-12-03T07:52:00.183811Z","shell.execute_reply.started":"2022-12-03T07:52:00.150779Z","shell.execute_reply":"2022-12-03T07:52:00.182714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train loop","metadata":{"papermill":{"duration":0.025842,"end_time":"2021-10-22T13:34:42.678563","exception":false,"start_time":"2021-10-22T13:34:42.652721","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Train loop\n# ====================================================\ndef train_loop(folds, fold):\n\n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    trn_idx = folds[folds['fold'] != fold].index\n    val_idx = folds[folds['fold'] == fold].index\n\n    train_folds = folds.loc[trn_idx].reset_index(drop=True)\n    valid_folds = folds.loc[val_idx].reset_index(drop=True)\n    \n    train_dataset = TrainDataset(train_folds, \n                                 transform=get_transforms(data='train'))\n    valid_dataset = TrainDataset(valid_folds, \n                                 transform=get_transforms(data='valid'))\n\n    train_loader = DataLoader(train_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=True, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset, \n                              batch_size=CFG.batch_size, \n                              shuffle=False, \n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    \n    # ====================================================\n    # scheduler \n    # ====================================================\n    def get_scheduler(optimizer):\n        if CFG.scheduler=='ReduceLROnPlateau':\n            scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=CFG.factor, patience=CFG.patience, verbose=True, eps=CFG.eps)\n        elif CFG.scheduler=='CosineAnnealingLR':\n            scheduler = CosineAnnealingLR(optimizer, T_max=CFG.T_max, eta_min=CFG.min_lr, last_epoch=-1)\n        elif CFG.scheduler=='CosineAnnealingWarmRestarts':\n            scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=CFG.T_0, T_mult=1, eta_min=CFG.min_lr, last_epoch=-1)\n        return scheduler\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    #model = CustomResNext(CFG.model_name, pretrained=True)\n    model = CustomEfficientNet(CFG.model_name, pretrained=True)\n    \n    model.to(device)\n\n    optimizer = Adam(model.parameters(), lr=CFG.lr, weight_decay=CFG.weight_decay, amsgrad=False)\n    scheduler = get_scheduler(optimizer)\n\n    # ====================================================\n    # apex\n    # ====================================================\n    if CFG.apex:\n        model, optimizer = amp.initialize(model, optimizer, opt_level='O1', verbosity=0)\n        \n        \n    \n    # ====================================================\n    # Criterion - ['LabelSmoothing', 'FocalLoss' 'FocalCosineLoss', 'SymmetricCrossEntropyLoss', 'BiTemperedLoss', 'TaylorCrossEntropyLoss']\n    # ====================================================\n    \n    def get_criterion():\n        if CFG.criterion=='CrossEntropyLoss':\n            criterion = nn.CrossEntropyLoss()\n        elif CFG.criterion=='LabelSmoothing':\n            criterion = LabelSmoothingLoss(classes=CFG.target_size, smoothing=CFG.smoothing)\n        elif CFG.criterion=='FocalLoss':\n            criterion = FocalLoss().to(device)\n        elif CFG.criterion=='FocalCosineLoss':\n            criterion = FocalCosineLoss()\n        elif CFG.criterion=='SymmetricCrossEntropyLoss':\n            criterion = SymmetricCrossEntropy().to(device)\n        elif CFG.criterion=='BiTemperedLoss': \n            criterion = BiTemperedLogisticLoss(t1=CFG.t1, t2=CFG.t2, smoothing=CFG.smoothing)\n        elif CFG.criterion=='TaylorCrossEntropyLoss':\n            criterion = TaylorCrossEntropyLoss(smoothing=CFG.smoothing)\n        return criterion\n\n\n    # ====================================================\n    # loop \n    # ====================================================\n    criterion = get_criterion()\n    LOGGER.info(f'Criterion: {criterion}')\n\n    best_score = 0.\n    best_loss = np.inf\n    \n    for epoch in range(CFG.epochs):\n        \n        start_time = time.time()\n        \n        # train\n        avg_loss = train_fn(train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, preds = valid_fn(valid_loader, model, criterion, device)\n        valid_labels = valid_folds[CFG.target_col].values\n        \n        if isinstance(scheduler, ReduceLROnPlateau):\n            scheduler.step(avg_val_loss)\n        elif isinstance(scheduler, CosineAnnealingLR):\n            scheduler.step()\n        elif isinstance(scheduler, CosineAnnealingWarmRestarts):\n            scheduler.step()\n\n        # scoring\n        score = get_score(valid_labels, preds[:,1])\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Accuracy: {score}')\n\n        if score > best_score:\n            print('YES')\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(), \n                        'preds': preds,\n                        'optimizer': optimizer.state_dict(),\n                        'scheduler': scheduler.state_dict()},\n                        OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    \n    check_point = torch.load(OUTPUT_DIR+f'{CFG.model_name}_fold{fold}_best.pth')\n    valid_folds[[str(c) for c in range(CFG.target_size)]] = check_point['preds']\n    valid_folds['preds'] = check_point['preds'][:,1]#.argmax(1)\n\n    return valid_folds","metadata":{"papermill":{"duration":0.057867,"end_time":"2021-10-22T13:34:42.762378","exception":false,"start_time":"2021-10-22T13:34:42.704511","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.188767Z","iopub.execute_input":"2022-12-03T07:52:00.189585Z","iopub.status.idle":"2022-12-03T07:52:00.218208Z","shell.execute_reply.started":"2022-12-03T07:52:00.189549Z","shell.execute_reply":"2022-12-03T07:52:00.217148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# main\n# ====================================================\ndef main():\n    def get_result(result_df):\n        preds = result_df['preds'].values\n        labels = result_df[CFG.target_col].values\n        score = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.5f}')\n    \n    if CFG.train:\n        # train \n        oof_df = pd.DataFrame()\n        for fold in range(CFG.n_fold):\n            if fold in CFG.trn_fold:\n                _oof_df = train_loop(folds, fold)\n                oof_df = pd.concat([oof_df, _oof_df])\n                LOGGER.info(f\"========== fold: {fold} result ==========\")\n                get_result(_oof_df)\n        # CV result\n        LOGGER.info(f\"========== CV ==========\")\n        get_result(oof_df)\n        # save result\n        oof_df.to_csv(OUTPUT_DIR+'oof_df.csv', index=False)","metadata":{"papermill":{"duration":0.037094,"end_time":"2021-10-22T13:34:42.826349","exception":false,"start_time":"2021-10-22T13:34:42.789255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-12-03T07:52:00.219802Z","iopub.execute_input":"2022-12-03T07:52:00.220722Z","iopub.status.idle":"2022-12-03T07:52:00.229492Z","shell.execute_reply.started":"2022-12-03T07:52:00.220687Z","shell.execute_reply":"2022-12-03T07:52:00.228477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    main()","metadata":{"papermill":{"duration":4449.841646,"end_time":"2021-10-22T14:48:52.693942","exception":false,"start_time":"2021-10-22T13:34:42.852296","status":"completed"},"tags":[],"scrolled":true,"execution":{"iopub.status.busy":"2022-12-03T07:52:00.230913Z","iopub.execute_input":"2022-12-03T07:52:00.231512Z"},"trusted":true},"execution_count":null,"outputs":[]}]}