{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# decoding JPEG images and decoding/encoding RLE datasets\n# !pip3 install pylibjpeg==1.4.0\n# https://github.com/pydicom/pylibjpeg\n\n# !pip3 install python-gdcm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip3 -q install timm==0.4.12","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\n\nimport os\nimport sys","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# suitable for kaggle notebook\n# sys.path = ['../ca_2',] + sys.path\n# print(sys.path)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import argparse\nimport warnings","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc, ast, cv2, time, pickle, random\nimport pylibjpeg\nimport gdcm\nimport pydicom\n# pydicom is a pure Python package for working with DICOM files. \n# -It lets you read, modify and write DICOM data in an easy \"pythonic\" way. ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom glob import glob\nfrom PIL import Image\n\n\nimport nibabel as nib\n# read / write access to some common neuroimaging file formats","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold, StratifiedKFold\n\nimport albumentations # python library for pixel-level augmentations ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import timm\n\nimport segmentation_models_pytorch as smp\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.cuda.amp as amp\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import graphviz","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pip3 install torchview\nfrom torchview import draw_graph","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(threshold=sys.maxsize)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_column', None)\npd.set_option('display.max_rows', None)\npd.set_option('display.max_seq_items', None)\npd.set_option('display.max_colwidth', None) # 500\npd.set_option('expand_frame_repr', True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndevice = torch.device('cuda')\n\n# benchmark mode is good whenever your input sizes for your network do not vary. \n# This flag allows you to enable the inbuilt cudnn auto-tuner to find the best algorithm to use for your hardware.\ntorch.backends.cudnn.benchmark = True","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"kernel_type = '0920_1bonev2_effv2s_224_15_6ch_augv2_mixupp5_drl3_rov1p2_bs8_lr23e5_eta23e6_50ep'\nload_kernel = None\nload_last = True\n\nn_folds = 5\nbackbone = 'tf_efficientnetv2_s_in21ft1k'\n\nimage_size = 224\nn_slice_per_c = 15\nin_chans = 6\n\ninit_lr = 23e-5 # 23e-5\neta_min = 23e-6\nbatch_size = 8 # 8\ndrop_rate = 0.\ndrop_rate_last = 0.3 # 0.3\ndrop_path_rate = 0.\np_mixup = 0.5\np_rand_order_v1 = 0.2\n\ndata_dir = '../input/rsna-2022-cervical-spine-fracture-detection/' # ../input/rsna-2022-cervical-spine-fracture-detection\ndata_dir_1 = '../input/narrsna20221npyfiles/numpy_1/numpy_1/'\nuse_amp = True\nnum_workers = 2 # 4\nout_dim = 1\n\nn_epochs = 75\n\nlog_dir = './logs'\nmodel_dir = './models'\nmodel_dir_seg = './kaggle'\nmodel_dir_seg = '../input/narrsna20221npyfiles/'\nos.makedirs(log_dir, exist_ok=True)\nos.makedirs(model_dir, exist_ok=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Albumentations is a computer vision tool that boosts the performance of deep convolutional neural networks.\n# Albumentations is a Python library for image augmentation.\ntransforms_train = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n    albumentations.HorizontalFlip(p=0.5),\n    albumentations.VerticalFlip(p=0.5),\n    albumentations.Transpose(p=0.5),\n    albumentations.RandomBrightnessContrast(brightness_limit=0.1, p=0.7),\n    albumentations.ShiftScaleRotate(shift_limit=0.3, scale_limit=0.3, rotate_limit=45, border_mode=4, p=0.7),\n\n    albumentations.OneOf([\n        albumentations.MotionBlur(blur_limit=3),\n        albumentations.MedianBlur(blur_limit=3),\n        albumentations.GaussianBlur(blur_limit=3),\n        albumentations.GaussNoise(var_limit=(3.0, 9.0)),\n    ], p=0.5),\n    albumentations.OneOf([\n        albumentations.OpticalDistortion(distort_limit=1.),\n        albumentations.GridDistortion(num_steps=5, distort_limit=1.),\n    ], p=0.5),\n\n    albumentations.CoarseDropout(max_height=int(image_size * 0.5), max_width=int(image_size * 0.5), max_holes=1, p=0.5),\n])\n\ntransforms_valid = albumentations.Compose([\n    albumentations.Resize(image_size, image_size),\n])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DataFrame","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(os.path.join(data_dir, 'train_seg.csv'))\n# df_train =>\n#             StudyInstanceUID  patient_overall  C1  C2  C3  C4  C5  C6  C7  \\\n# 0   1.2.826.0.1.3680043.6200                1   1   1   0   0   0   0   0   \n# 1  1.2.826.0.1.3680043.27262                1   0   1   0   0   0   0   0   \n# 2  1.2.826.0.1.3680043.21561                1   0   1   0   0   0   0   0\n# ...\n# len(df_train) => 2018\n\ndf = df_train.sample(16).reset_index(drop=True) if DEBUG else df_train\n\nsid = []\ncs = []\nlabel = []\nfold = []\nfor _, row in df.iterrows():\n    for i in [1,2,3,4,5,6,7]:\n        sid.append(row.StudyInstanceUID)\n        cs.append(i)\n        label.append(row[f'C{i}'])\n        fold.append(row.fold)\n\ndf = pd.DataFrame({\n    'StudyInstanceUID': sid,\n    'Cid': cs,\n    'Cid_label': label,\n    'fold': fold\n})\n\ndf.tail()","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[5:10].head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"class CLSDataset(Dataset):\n    def __init__(self, df, mode, transform):\n\n        self.df = df.reset_index()\n        self.mode = mode\n        self.transform = transform\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        cid = row.Cid\n        \n        images = []\n            \n        filepath = os.path.join(data_dir, f'numpy_1/{row.StudyInstanceUID}_{cid}.npy')\n        images = np.load(filepath)\n            \n            # type(image), image.shape => <class 'numpy.ndarray'> (15, 224, 224, 6)\n\n        images = np.stack([self.transform(image=images[0])['image'] for i in range(n_slice_per_c)], 0)\n\n        images = images.transpose(0,3,1,2)\n            # type(image), image.shape => <class 'numpy.ndarray'> (15, 6, 224, 224)        \n\n        images = images / 255. # trim the 'data values' between 0. and 1. \n            # prior to 255. divide, convert data to float\n\n        if self.mode != 'test':\n            images = torch.tensor(images).float()  \n\n            # images.shape => torch.Size([15, 6, 224, 224])            \n            labels = torch.tensor([row.Cid_label] * n_slice_per_c).float()\n                # labels => tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])\n\n            # randomly shuffling slices of row of 10% train data.\n            if self.mode == 'train' and random.random() < p_rand_order_v1:                \n                indices = torch.randperm(images.size(0))\n                # indices => tensor([ 0,  3, 13, 11, 14,  6,  9,  1, 12,  8,  5,  2,  7, 10,  4])           \n                images = images[indices]\n                    # images.shape => torch.Size([15, 6, 224, 224])\n                \n            return images, labels\n        else:\n            return torch.tensor(images).float()","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # plt.rcParams['figure.figsize'] = 20,8\n\n# df_show = df[28:32]\n# dataset_show = CLSDataset(df_show, 'train', transform=transforms_train)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class TimmModel(nn.Module):\n    def __init__(self, backbone, pretrained=False):\n        super(TimmModel, self).__init__()\n\n        self.encoder = timm.create_model(\n            backbone,\n            in_chans=in_chans,\n            num_classes=out_dim,\n            features_only=False,\n            drop_rate=drop_rate,\n            drop_path_rate=drop_path_rate,\n            pretrained=pretrained\n        )\n        # self.encoder.default_cfg =>\n        # {'url': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-effv2-weights/tf_efficientnetv2_s_21ft1k-d7dafa41.pth', \n        # 'num_classes': 1000, 'input_size': (3, 300, 300), 'pool_size': (10, 10), 'crop_pct': 1.0, 'interpolation': \n        # 'bicubic', 'mean': (0.5, 0.5, 0.5), 'std': (0.5, 0.5, 0.5), 'first_conv': 'conv_stem', 'classifier': 'classifier', \n        # 'test_input_size': (3, 384, 384), 'architecture': 'tf_efficientnetv2_s_in21ft1k'}        \n\n\n        \n        if 'efficient' in backbone:\n            hdim = self.encoder.conv_head.out_channels\n                # (conv_head): Conv2d(256, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False) \n                # self.encoder.conv_head => Conv2d(256, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False)  \n                # self.encoder.conv_head.out_channels => 1280\n                \n                # nn.Identity() => Identity()\n                # self.encoder.classifier => Linear(in_features=1280, out_features=1, bias=True)  \n            # replace the last classifier layer with identity layer.\n            self.encoder.classifier = nn.Identity()\n\n        elif 'convnext' in backbone:\n            hdim = self.encoder.head.fc.in_features\n            self.encoder.head.fc = nn.Identity()\n\n\n        self.lstm = nn.LSTM(hdim, 256, num_layers=2, dropout=drop_rate, bidirectional=True, batch_first=True)\n        self.head = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.Dropout(drop_rate_last),\n            nn.LeakyReLU(0.1),\n            nn.Linear(256, out_dim),\n        )\n\n    def forward(self, x):  # (bs, nslice, ch, sz, sz)\n        # x.shape => torch.Size([2, 15, 6, 224, 224])\n        \n        bs = x.shape[0]\n        # Tensor.view(*shape) => Returns a new tensor with the same data as the self tensor but of a different shape.\n        x = x.view(bs * n_slice_per_c, in_chans, image_size, image_size)\n            # x.shape => torch.Size([30, 6, 224, 224])\n        \n        feat = self.encoder(x)        \n\n            # feat.shape => torch.Size([30, 1280])        \n        feat = feat.view(bs, n_slice_per_c, -1)\n            # feat.shape => torch.Size([2, 15, 1280])\n        \n        feat, _ = self.lstm(feat) # multiple outputs by lstm layer.\n        \n        # tensor.contiguous() will create a copy of the tensor, and the element in the copy will be stored in the memory in a contiguous(ordered) way.\n        # contiguous(ordered) => change the order of data in accordance to indices.\n        # contiguous() function is usually required when we 'changed the shape of a tensor' and further reshaping (view) it. \n        feat = feat.contiguous().view(bs * n_slice_per_c, -1)\n        \n        feat = self.head(feat)\n        feat = feat.view(bs, n_slice_per_c).contiguous()\n\n        return feat\n","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:23.245545Z","iopub.status.busy":"2022-10-29T08:46:23.245146Z","iopub.status.idle":"2022-10-29T08:46:23.26229Z","shell.execute_reply":"2022-10-29T08:46:23.260005Z","shell.execute_reply.started":"2022-10-29T08:46:23.245506Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# m = TimmModel(backbone)\n# m(torch.rand(2, n_slice_per_c, in_chans, image_size, image_size)).shape\n#     # m(torch.rand(2, n_slice_per_c, in_chans, image_size, image_size)).shape => torch.Size([2, 15])","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:23.265243Z","iopub.status.busy":"2022-10-29T08:46:23.26406Z","iopub.status.idle":"2022-10-29T08:46:32.738976Z","shell.execute_reply":"2022-10-29T08:46:32.73783Z","shell.execute_reply.started":"2022-10-29T08:46:23.265207Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# draw_graph(m, input_data = torch.rand(1, 3, 128,128,128), expand_nested=True, save_graph=True).visual_graph","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loss & Metric","metadata":{}},{"cell_type":"code","source":"bce = nn.BCEWithLogitsLoss(reduction='none')\n\n\ndef criterion(logits, targets, activated=False):\n    if activated:\n        losses = nn.BCELoss(reduction='none')(logits.view(-1), targets.view(-1))\n    else:\n        losses = bce(logits.view(-1), targets.view(-1))\n    losses[targets.view(-1) > 0] *= 2.\n    norm = torch.ones(logits.view(-1).shape[0]).to(device)\n    norm[targets.view(-1) > 0] *= 2\n    return losses.sum() / norm.sum()","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:32.741112Z","iopub.status.busy":"2022-10-29T08:46:32.740646Z","iopub.status.idle":"2022-10-29T08:46:32.74936Z","shell.execute_reply":"2022-10-29T08:46:32.747958Z","shell.execute_reply.started":"2022-10-29T08:46:32.741073Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train & Valid func","metadata":{}},{"cell_type":"code","source":"# mixup explained in train_1.ipynb\ndef mixup(input, truth, clip=[0, 1]):\n    indices = torch.randperm(input.size(0))\n    shuffled_input = input[indices]\n    shuffled_labels = truth[indices]\n\n    lam = np.random.uniform(clip[0], clip[1])\n    input = input * lam + shuffled_input * (1 - lam)\n    return input, truth, shuffled_labels, lam\n\n\ndef train_func(model, loader_train, optimizer, scaler=None):\n    model.train()\n    train_loss = []\n    bar = tqdm(loader_train)\n    for images, targets in bar:\n        optimizer.zero_grad()\n        images = images.cuda()\n        targets = targets.cuda()\n        \n        do_mixup = False\n        if random.random() < p_mixup:\n            do_mixup = True\n            images, targets, targets_mix, lam = mixup(images, targets)\n\n        with amp.autocast():\n            logits = model(images)\n            loss = criterion(logits, targets)\n            if do_mixup:\n                loss11 = criterion(logits, targets_mix)\n                loss = loss * lam  + loss11 * (1 - lam)\n        train_loss.append(loss.item())\n        scaler.scale(loss).backward()\n        scaler.step(optimizer)\n        scaler.update()\n\n        bar.set_description(f'smooth loss:{np.mean(train_loss[-30:]):.4f}')\n\n    return np.mean(train_loss)\n\n\ndef valid_func(model, loader_valid):\n    model.eval()\n    valid_loss = []\n    gts = []\n    outputs = []\n    bar = tqdm(loader_valid)\n    with torch.no_grad():\n        for images, targets in bar:\n            images = images.cuda()\n            targets = targets.cuda()\n\n            logits = model(images)\n            loss = criterion(logits, targets)\n            \n            gts.append(targets.cpu())\n            outputs.append(logits.cpu())\n            valid_loss.append(loss.item())\n            \n            bar.set_description(f'smooth loss:{np.mean(valid_loss[-30:]):.4f}')\n\n    outputs = torch.cat(outputs)\n    gts = torch.cat(gts)\n    valid_loss = criterion(outputs, gts).item()\n\n    return valid_loss\n","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:32.75212Z","iopub.status.busy":"2022-10-29T08:46:32.751519Z","iopub.status.idle":"2022-10-29T08:46:32.766761Z","shell.execute_reply":"2022-10-29T08:46:32.765711Z","shell.execute_reply.started":"2022-10-29T08:46:32.752079Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.rcParams['figure.figsize'] = 20, 2\n# optimizer = optim.AdamW(m.parameters(), lr=init_lr)\n# scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs, eta_min = 23e-8)\n\n# lrs = []\n# for epoch in range(1, n_epochs+1):\n#     scheduler_cosine.step(epoch-1)\n#     lrs.append(optimizer.param_groups[0][\"lr\"])\n# plt.plot(range(len(lrs)), lrs)","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:32.770948Z","iopub.status.busy":"2022-10-29T08:46:32.770484Z","iopub.status.idle":"2022-10-29T08:46:33.001268Z","shell.execute_reply":"2022-10-29T08:46:33.000206Z","shell.execute_reply.started":"2022-10-29T08:46:32.77091Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# df_debug = df.copy()\n# df = df[1000:]\n# df = df_debug[1006:].copy()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run(fold):\n\n    log_file = os.path.join(log_dir, f'{kernel_type}.txt')\n    model_file = os.path.join(model_dir, f'{kernel_type}_fold{fold}_best.pth')\n\n    train_ = df[df['fold'] != fold].reset_index(drop=True)\n    valid_ = df[df['fold'] == fold].reset_index(drop=True)\n    dataset_train = CLSDataset(train_, 'train', transform=transforms_train)\n    dataset_valid = CLSDataset(valid_, 'valid', transform=transforms_valid)\n    loader_train = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, shuffle=True, num_workers=num_workers, drop_last=True)\n    loader_valid = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n\n    model = TimmModel(backbone, pretrained=True)\n    model = model.to(device)\n    \n#     # if not first run, load previous model\n#     fold_l = 0\n#     load_model_file = os.path.join(model_dir_seg, f'{kernel_type}_fold{fold_l}_best.pth')\n#     sd = torch.load(load_model_file)\n#     if 'model_state_dict' in sd.keys():\n#         sd = sd['model_state_dict']\n#     sd = {k[7:] if k.startswith('module.') else k: sd[k] for k in sd.keys()}\n#     model.load_state_dict(sd, strict=True)    \n\n    optimizer = optim.AdamW(model.parameters(), lr=init_lr)\n    scaler = torch.cuda.amp.GradScaler() if use_amp else None\n\n    metric_best = np.inf\n    loss_min = np.inf\n\n    scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, n_epochs, eta_min=eta_min)\n\n#     print(len(dataset_train), len(dataset_valid))\n\n    for epoch in range(1, n_epochs+1):\n        scheduler_cosine.step(epoch-1)\n\n        print(time.ctime(), 'Epoch:', epoch)\n\n        train_loss = train_func(model, loader_train, optimizer, scaler)\n        valid_loss = valid_func(model, loader_valid)\n        metric = valid_loss\n\n        content = time.ctime() + ' ' + f'Fold {fold}, Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train_loss: {train_loss:.5f}, valid_loss: {valid_loss:.5f}, metric(valid_loss): {(metric):.6f}.'\n        print(content)\n        with open(log_file, 'a') as appender:\n            appender.write(content + '\\n')\n\n        if metric < metric_best:\n            print(f'metric_best ({metric_best:.6f} --> {metric:.6f}). Saving model ...')\n#             if not DEBUG:\n            torch.save(model.state_dict(), model_file)\n            metric_best = metric\n\n#         # Save Last\n#         if not DEBUG:\n#             torch.save(\n#                 {\n#                     'epoch': epoch,\n#                     'model_state_dict': model.state_dict(),\n#                     'optimizer_state_dict': optimizer.state_dict(),\n#                     'scaler_state_dict': scaler.state_dict() if scaler else None,\n#                     'score_best': metric_best,\n#                 },\n#                 model_file.replace('_best', '_last')\n#             )\n\n    del model\n    torch.cuda.empty_cache()\n    _ = gc.collect()\n","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:33.003917Z","iopub.status.busy":"2022-10-29T08:46:33.003238Z","iopub.status.idle":"2022-10-29T08:46:33.01973Z","shell.execute_reply":"2022-10-29T08:46:33.018392Z","shell.execute_reply.started":"2022-10-29T08:46:33.003873Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run(0)\n# run(1)\n# run(2)\n# run(3)\n# run(4)","metadata":{"execution":{"iopub.execute_input":"2022-10-29T08:46:33.021919Z","iopub.status.busy":"2022-10-29T08:46:33.021481Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}