{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)\n!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nfrom efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom tqdm import tqdm\n\n\nCFG = {\n    'train': False,\n    \n    'train_img_path': '../input/rsna-str-pulmonary-embolism-detection/train',\n    'test_img_path': '../input/rsna-str-pulmonary-embolism-detection/test',\n    'cv_fold_path': '../input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',\n    'train_path': '../input/rsna-str-pulmonary-embolism-detection/train.csv',\n    'test_path': '../input/rsna-str-pulmonary-embolism-detection/test.csv',\n    \n    'image_target_cols': [\n        'pe_present_on_image', # only image level\n    ],\n    \n    'exam_target_cols': [\n        'negative_exam_for_pe', # exam level\n        #'qa_motion',\n        #'qa_contrast',\n        #'flow_artifact',\n        'rv_lv_ratio_gte_1', # exam level\n        'rv_lv_ratio_lt_1', # exam level\n        'leftsided_pe', # exam level\n        'chronic_pe', # exam level\n        #'true_filling_defect_not_pe',\n        'rightsided_pe', # exam level\n        'acute_and_chronic_pe', # exam level\n        'central_pe', # exam level\n        'indeterminate' # exam level\n    ], \n    \n    'img_num': 200,\n    'img_size': 256,\n    'lr': 0.001,\n    'epochs': 4,\n    'device': 'cuda', # cuda, cpu\n    'train_bs': 2,\n    'accum_iter': 8,\n    'verbose_step': 1,\n    'num_workers': 4,\n    'efbnet': 'efficientnet-b0',\n    \n    'train_folds': [np.arange(0,16),\n                    np.concatenate([np.arange(0,12), np.arange(16,20)]),\n                    np.concatenate([np.arange(0,8), np.arange(12,20)]),\n                    np.concatenate([np.arange(0,4), np.arange(8,20)]),\n                    np.arange(4,20),\n                   ],#[np.arange(0,16)],\n    \n    'valid_folds': [np.arange(16,20),\n                    np.arange(12,16),\n                    np.arange(8,12),\n                    np.arange(4,8),\n                    np.arange(0,4)\n                   ],#[np.arange(16,20)],\n    \n    'model_path': '../input/kh-rsna-model',\n    'tag': 'efb0_stage2_multilabel'\n}\n\nSEED = 42321\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\ndef window(img, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    #X = (X*255.0).astype('uint8')\n    return X\n\ndef get_img(path):\n    \n    d = pydicom.read_file(path)\n    '''\n    res = cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (CFG['img_size'], CFG['img_size'])), d.ImagePositionPatient[2]\n    '''\n    \n    '''\n    RED channel / LUNG window / level=-600, width=1500\n    GREEN channel / PE window / level=100, width=700\n    BLUE channel / MEDIASTINAL window / level=40, width=400\n    '''\n    \n    img = (d.pixel_array * d.RescaleSlope) + d.RescaleIntercept\n    \n    r = window(img, -600, 1500)\n    g = window(img, 100, 700)\n    b = window(img, 40, 400)\n    \n    res = np.concatenate([r[:, :, np.newaxis],\n                          g[:, :, np.newaxis],\n                          b[:, :, np.newaxis]], axis=-1)\n    \n    #res = (res*255.0).astype('uint8')\n    res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)\n    #res = cv2.resize(res, (CFG['img_size'], CFG['img_size']))\n    #res = res.astype(np.float32)/255.\n    \n    return res\n\nclass RSNADatasetStage1(Dataset):\n    def __init__(\n        self, df, label_smoothing, data_root, \n        image_subsampling=True, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.label_smoothing = label_smoothing\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        # get labels\n        if self.output_label:\n            target = self.df[CFG['image_target_cols']].values[index]\n            \n        path = \"{}/{}/{}/{}.dcm\".format(self.data_root, \n                                        self.df.iloc[index]['StudyInstanceUID'], \n                                        self.df.iloc[index]['SeriesInstanceUID'], \n                                        self.df.iloc[index]['SOPInstanceUID'])\n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        # do label smoothing\n        if self.output_label == True:\n            target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)\n            \n            return img, target\n        else:\n            return img\n        \nclass RSNADataset(Dataset):\n    def __init__(\n        self, df, label_smoothing, data_root, \n        image_subsampling=True, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df\n        self.patients = self.df['StudyInstanceUID'].unique()\n        self.image_subsampling = image_subsampling\n        self.label_smoothing = label_smoothing\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n        \n    def get_patients(self):\n        return self.patients\n        \n    def __len__(self):\n        return len(self.patients)\n    \n    def __getitem__(self, index: int):\n        \n        patient = self.patients[index]\n        df_ = self.df.loc[self.df.StudyInstanceUID == patient]\n        \n        per_image_feats = get_stage1_columns()\n        #print(per_image_feats)\n        \n        if self.image_subsampling:\n            img_num = min(CFG['img_num'], df_.shape[0])\n            \n            # naive image subsampling\n            img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)\n            \n            # get all images, then slice location and sort according to z values\n            imgs = np.zeros((CFG['img_num'],), np.float32) #np.zeros((CFG['img_num'], CFG['img_size'], CFG['img_size'], 3), np.float32)\n            per_image_preds = np.zeros((CFG['img_num'], len(per_image_feats)), np.float32)\n            locs = np.zeros((CFG['img_num'],), np.float32)\n            image_masks = np.zeros((CFG['img_num'],), np.float32)\n            image_masks[:img_num] = 1.\n            \n            # get labels\n            if self.output_label:\n                exam_label = df_[CFG['exam_target_cols']].values[0]\n                image_labels = np.zeros((CFG['img_num'], len(CFG['image_target_cols'])), np.float32)\n            \n        else:\n            img_num = df_.shape[0]\n            img_ix = np.arange(df_.shape[0])\n            \n            # get all images, then slice location and sort according to z values\n            imgs = np.zeros((img_num, ), np.float32) #np.zeros((img_num, CFG['img_size'], CFG['img_size'], 3), np.float32)\n            per_image_preds = np.zeros((img_num, len(per_image_feats)), np.float32)\n            locs = np.zeros((img_num,), np.float32)\n            image_masks = np.zeros((img_num,), np.float32)\n            image_masks[:img_num] = 1.\n            \n            # get labels\n            if self.output_label:\n                exam_label = df_[CFG['exam_target_cols']].values[0]\n                image_labels = np.zeros((img_num, len(CFG['image_target_cols'])), np.float32)\n                \n        for i, im_ix in enumerate(img_ix):\n            path = \"{}/{}/{}/{}.dcm\".format(self.data_root, \n                                            df_['StudyInstanceUID'].values[im_ix], \n                                            df_['SeriesInstanceUID'].values[im_ix], \n                                            df_['SOPInstanceUID'].values[im_ix])\n            \n            d = pydicom.read_file(path)\n            locs[i] = d.ImagePositionPatient[2]\n            per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]\n            \n            if self.output_label == True:\n                image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]\n\n        #print('get img done')\n        \n        seq_ix = np.argsort(locs)\n        \n        # image features: img_num * img_size * img_size * 1\n        '''\n        imgs = imgs[seq_ix]\n        if self.transforms:\n            imgs = [self.transforms(image=img)['image'] for img in imgs]\n        imgs = torch.stack(imgs)\n        '''\n        \n        # image level features: img_num\n        #locs[:img_num] -= locs[:img_num].min()\n        locs = locs[seq_ix]\n        locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]\n        locs[0] = 0\n        \n        per_image_preds = per_image_preds[seq_ix]\n        \n        # patient level features: 1\n        \n        # train, train-time valid, multiple patients: imgs, locs, image_labels, exam_label, img_num\n        # whole valid-time valid, single patient: imgs, locs, image_labels, exam_label, img_num, sorted id\n        # whole test-time test, single patient: imgs, locs, img_num, sorted_id\n        \n        # do label smoothing\n        if self.output_label == True:\n            image_labels = image_labels[seq_ix]\n            image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)\n            exam_label =  np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)\n            \n            return imgs, per_image_preds, locs, image_labels, exam_label, image_masks\n        else:\n            return imgs, per_image_preds, locs, img_num, index, seq_ix\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            #HorizontalFlip(p=0.5),\n            #VerticalFlip(),\n            #RandomRotate90(p=0.5),\n            #Cutout(p=0.5),\n            #Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''   \n        \ndef get_valid_transforms():\n    return Compose([\n            #Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''  \n\nclass RNSAImageFeatureExtractor(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = EfficientNet.from_name(CFG['efbnet'])\n        #print(self.cnn_model, CFG['efbnet'])\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        \n    def get_dim(self):\n        return self.cnn_model._fc.in_features\n        \n    def forward(self, x):\n        feats = self.cnn_model.extract_features(x)\n        return self.pooling(feats).view(x.shape[0], -1)                         \n\nclass RSNAImgClassifierSingle(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = RNSAImageFeatureExtractor()\n        self.image_predictors = nn.Linear(self.cnn_model.get_dim(), 1)\n        \n    def forward(self, imgs):\n        #print(images.shape)\n        imgs_embdes = self.cnn_model(imgs) # bs * efb_feat_size\n        #print(imgs_embdes.shape)\n        image_preds = self.image_predictors(imgs_embdes)\n        \n        return image_preds\n    \nclass RSNAImgClassifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = RNSAImageFeatureExtractor()\n        self.image_predictors = nn.Linear(self.cnn_model.get_dim(), 9)\n        \n    def forward(self, imgs):\n        #print(images.shape)\n        imgs_embdes = self.cnn_model(imgs) # bs * efb_feat_size\n        #print(imgs_embdes.shape)\n        image_preds = self.image_predictors(imgs_embdes)\n        \n        return image_preds\n\nclass TimeDistributed(nn.Module):\n\n    def __init__(self, module, batch_first=True):\n        super(TimeDistributed, self).__init__()\n        self.module = module\n        self.batch_first = batch_first\n\n    def forward(self, x):\n        ''' x size: (batch_size, time_steps, in_channels, height, width) '''\n        x_size= x.size()\n        c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])\n        \n        c_out = self.module(c_in)\n        r_in = c_out.view(x_size[0], x_size[1], -1)\n        if self.batch_first is False:\n            r_in = r_in.permute(1, 0, 2)\n        return r_in \n    \nclass RSNAClassifier(nn.Module):\n    def __init__(self, hidden_size=64):\n        super().__init__()\n        \n        self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        \n        self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))\n        self.exam_predictor = nn.Linear(hidden_size*2*2, 9)\n        \n    def forward(self, img_preds, locs):\n        \n        embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2) # bs * ts * fs\n        \n        embeds, _ = self.gru(embeds)\n        image_preds = self.image_predictors(embeds)\n        \n        avg_pool = torch.mean(embeds, 1)\n        max_pool, _ = torch.max(embeds, 1)\n        conc = torch.cat([avg_pool, max_pool], 1)\n        \n        exam_pred = self.exam_predictor(conc)\n        return image_preds, exam_pred\n    \n#RSNAClassifier(64)\ndef rsna_wloss_inference(y_true_img, y_true_exam, y_pred_img, y_pred_exam, chunk_sizes):\n    # y_true_img, y_pred_img: (p1*in1 + p2*in2 + ,,,) \n    # y_true_exam, y_pred_exam: (p1*in1 + p2*in2 + ,,,) x 9\n    # chunk_sizes: (patient_num)\n    '''\n    'negative_exam_for_pe', # exam level 0.0736196319\n    'rv_lv_ratio_gte_1', # exam level 0.2346625767\n    'rv_lv_ratio_lt_1', # exam level 0.0782208589\n    'leftsided_pe', # exam level 0.06257668712\n    'chronic_pe', # exam level 0.1042944785\n    'rightsided_pe', # exam level 0.06257668712\n    'acute_and_chronic_pe', # exam level 0.1042944785\n    'central_pe', # exam level 0.1877300613\n    'indeterminate' # exam level 0.09202453988\n    '''\n    \n    # transform into torch tensors\n    y_true_img, y_true_exam, y_pred_img, y_pred_exam = torch.tensor(y_true_img, dtype=torch.float32), torch.tensor(y_true_exam, dtype=torch.float32), torch.tensor(y_pred_img, dtype=torch.float32), torch.tensor(y_pred_exam, dtype=torch.float32)\n    \n    # split into chunks (each chunks is for a single exam)\n    y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks = torch.split(y_true_img, chunk_sizes, dim=0), torch.split(y_true_exam, chunk_sizes, dim=0), torch.split(y_pred_img, chunk_sizes, dim=0), torch.split(y_pred_exam, chunk_sizes, dim=0)\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCELoss(reduction='none')\n    \n    total_loss = torch.tensor(0, dtype=torch.float32)\n    total_weights = torch.tensor(0, dtype=torch.float32)\n    for i, (y_true_img_, y_true_exam_, y_pred_img_, y_pred_exam_) in enumerate(zip(y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks)):\n        exam_loss = bce_func(y_pred_exam_[0, :], y_true_exam_[0, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        image_loss = bce_func(y_pred_img_, y_true_img_)\n        img_num = chunk_sizes[i]\n        qi = torch.sum(y_true_img_)/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss\n\ndef rsna_wloss_train(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):\n    # y_true_img, y_pred_img: patient_numximg_num\n    # y_true_exam, y_pred_exam: patient_num x 9\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1).to(device)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCEWithLogitsLoss(reduction='none').to(device)\n    \n    total_loss = torch.tensor(0, dtype=torch.float32).to(device)\n    total_weights = torch.tensor(0, dtype=torch.float32).to(device)\n    for i in range(y_true_img.shape[0]):\n        exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        img_mask = image_masks[i]\n        #print(torch.sum(y_true_img[i,:]), torch.sum(img_mask))\n        image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:]).flatten()\n        #print(image_loss.shape)\n        #print(img_mask.shape)\n        #print((image_loss*img_mask).shape)\n        #assert False\n        image_loss = image_loss*img_mask # mark 0 loss for padding images\n        img_num = torch.sum(img_mask) #y_true_img.shape[1]\n        qi = torch.sum(y_true_img[i,:])/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss, total_loss, total_weights\n\ndef rsna_wloss_valid(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):\n    # y_true_img, y_pred_img: patient_numximg_num\n    # y_true_exam, y_pred_exam: patient_num x 9\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1).to(device)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCEWithLogitsLoss(reduction='none').to(device)\n    \n    total_loss = torch.tensor(0, dtype=torch.float32).to(device)\n    total_weights = torch.tensor(0, dtype=torch.float32).to(device)\n    for i in range(y_true_img.shape[0]):\n        exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        img_mask = image_masks[i]\n        #print(torch.sum(y_true_img[i,:]), torch.sum(img_mask))\n        image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:]).flatten()\n        #print(image_loss.shape)\n        #print(img_mask.shape)\n        #print((image_loss*img_mask).shape)\n        #assert False\n        image_loss = image_loss*img_mask # mark 0 loss for padding images\n        img_num = torch.sum(img_mask) #y_true_img.shape[1]\n        qi = torch.sum(y_true_img[i,:])/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss, total_loss, total_weights\n\ndef prepare_train_dataloader(train, cv_df, train_fold, valid_fold):\n    train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()\n    valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n\n    train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)\n    valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)\n\n    # train mode to do image-level subsampling\n    train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'],  image_subsampling=True, transforms=get_train_transforms(), output_label=True) \n    valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=True)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=True,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=1,\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    #print(len(train_loader), len(val_loader))\n\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):\n    model.train()\n\n    t = time.time()\n    loss_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, per_image_preds, locs, image_labels, exam_label, image_masks) in enumerate(train_loader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        image_masks = image_masks.to(device).float()\n        image_labels = image_labels.to(device).float()\n        exam_label = exam_label.to(device).float()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss, total_loss, total_weights = rsna_wloss_train(image_labels, exam_label, image_preds, exam_pred, image_masks, device)\n\n            scaler.scale(loss).backward()\n\n            loss_sum += total_loss.detach().item()\n            loss_w_sum += total_weights.detach().item()\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()                \n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                print(\n                    f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \\\n                    f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                    f'time: {(time.time() - t):.4f}', end= '\\r' if (step + 1) != len(train_loader) else '\\n'\n                )\n\ndef post_process(exam_pred, image_pred):\n    \n    rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')\n    rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')\n    central_pe_ix = CFG['exam_target_cols'].index('central_pe')\n    rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')\n    leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')\n    acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')\n    chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')\n    negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')\n    indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')\n    \n    # rule 1 or rule 2 judgement: if any pe image exist\n    has_pe_image = torch.max(image_pred, 1)[0][0] > 0\n    #print(has_pe_image)\n    \n    # rule 1-a: only one >= 0.5, the other < 0.5\n    rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]\n    rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1) # to make one at least > 0.5\n    rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a)) # turn back into logits\n    exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)\n    \n    # rule 1-b-1 or 1-b-2 judgement: at least one > 0.5\n    crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]\n    has_no_pe = torch.max(crl_pe ,1)[0] <= 0 # all <= 0.5\n    #print(has_no_pe)\n    #assert False\n        \n    # rule 1-b\n    max_val = torch.max(crl_pe, 1)[0]\n    crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)\n    exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)\n    \n    # rule 1-c-1 or 1-c-2 judgement: at most one > 0.5\n    ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]\n    both_ac_ch = torch.min(ac_pe ,1)[0] > 0 # all > 0.5\n    \n    # rule 1-c\n    ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1) # to make only one > 0.5\n    ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c)) # turn back into logits\n    exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)\n    \n    # rule 1-d\n    neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]\n    neg_ind_1d = torch.clamp(neg_ind, max=0)\n    exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)\n    \n    # rule 2-a\n    ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]\n    ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1) # to make one at least > 0.5\n    ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a)) # turn back into logits\n    exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)\n    \n    # rule 2-b\n    all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,\n                                          central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,\n                                          acute_and_chronic_pe_ix, chronic_pe_ix]]\n    all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)\n    exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,\n                  central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,\n                  acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)\n    \n    return exam_pred, image_pred\n    \ndef valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, per_image_preds, locs, image_labels, exam_label, image_masks) in enumerate(val_loader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        image_masks = image_masks.to(device).float()\n        image_labels = image_labels.to(device).float()\n        exam_label = exam_label.to(device).float()\n\n        #print(image_labels.shape, exam_label.shape)\n        #with autocast():\n        image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        exam_pred, image_preds= post_process(exam_pred, image_preds)\n\n        loss, total_loss, total_weights = rsna_wloss_valid(image_labels, exam_label, image_preds, exam_pred, image_masks, device)\n\n        loss_sum += total_loss.detach().item()\n        loss_w_sum += total_weights.detach().item()          \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            print(\n                f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \\\n                f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(val_loader) else '\\n'\n            )\n    \n    if schd_loss_update:\n        scheduler.step(loss_sum/loss_w_sum)\n    else:\n        scheduler.step()\n\ndef check_label_consistency(checking_df):\n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    df = checking_df.copy()\n    print(df.shape)\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n\n    rule1d = df_pos.loc[(df_pos.indeterminate        > 0.5) | \n                        (df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    \n    print('label in-consistency counts:', errors.shape)\n        \n    if errors.shape[0] > 0:\n        print(errors.broken_rule.value_counts())\n        print(errors)\n        assert False\n        \ndef inference(model, device, df, root_path):\n    model.eval()\n\n    t = time.time()\n\n    ds = RSNADataset(df, 0.0, root_path,  image_subsampling=False, transforms=get_valid_transforms(), output_label=False)\n    \n    dataloader = torch.utils.data.DataLoader(\n        ds, \n        batch_size=1,\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    \n    patients = ds.get_patients()\n    \n    res_dfs = []\n    \n    for step, (imgs, per_image_preds, locs, img_num, index, seq_ix) in enumerate(dataloader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        \n        index = index.detach().numpy()[0]\n        seq_ix = seq_ix.detach().numpy()[0,:]\n        \n        patient_filt = (df.StudyInstanceUID == patients[index])\n        \n        patient_df = pd.DataFrame()\n        patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]\n        patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values # no need to sort\n        patient_df['StudyInstanceUID'] = patients[index] # single value\n        \n        for c in CFG['image_target_cols']+CFG['exam_target_cols']:\n            patient_df[c] = 0.0\n\n        #with autocast():\n        image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        \n        exam_pred, image_preds = post_process(exam_pred, image_preds)\n        \n        exam_pred = torch.sigmoid(exam_pred).cpu().detach().numpy()\n        image_preds = torch.sigmoid(image_preds).cpu().detach().numpy()\n\n        patient_df[CFG['exam_target_cols']] = exam_pred[0]\n        patient_df[CFG['image_target_cols']] = image_preds[0,:]\n        res_dfs += [patient_df]\n\n        '''\n        res_df = res_df.merge(patient_df, on=['SOPInstanceUID', 'StudyInstanceUID'], how='left')\n        '''\n        # naive slow version\n        '''\n        res_df.loc[patient_filt, CFG['exam_target_cols']] = exam_pred[0]\n        for si, sop_id in enumerate(sop_ids):\n            sop_filt = (patient_filt) & (res_df.SOPInstanceUID == sop_id)\n            res_df.loc[sop_filt, CFG['image_target_cols']] = image_preds[0, si]\n        '''\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(dataloader)):\n            print(\n                f'Inference Step {step+1}/{len(dataloader)}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(dataloader) else '\\n'\n            )\n                \n    res_dfs = pd.concat(res_dfs, axis=0).reset_index(drop=True)\n    res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))\n    assert res_dfs.shape[0] == df.shape[0]\n    check_label_consistency(res_dfs)\n    \n    return res_dfs\n  \nSTAGE1_CFGS = [\n    {\n        'tag': 'efb0_stage1',\n        'model_constructor': RSNAImgClassifierSingle,\n        'dataset_constructor': RSNADatasetStage1,\n        'output_len': 1\n    },\n    {\n        'tag': 'efb0_stage1_multilabel',\n        'model_constructor': RSNAImgClassifier,\n        'dataset_constructor': RSNADatasetStage1,\n        'output_len': 9\n    },\n]\nSTAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'\n\n\ndef get_stage1_columns():\n    \n    new_feats = []\n    for cfg in STAGE1_CFGS:\n        for i in range(cfg['output_len']):\n            f = cfg['tag']+'_'+str(i)\n            new_feats += [f]\n        \n    return new_feats\n\ndef update_stage1_oof_preds(df, cv_df):\n    \n    res_file_name = STAGE1_CFGS_TAG+\"-train.csv\"    \n    \n    new_feats = get_stage1_columns()\n    for f in new_feats:\n        df[f] = 0\n    \n    if os.path.isfile(res_file_name):\n        df = pd.read_csv(res_file_name)\n        print('img acc:', ((df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]]).mean())\n        return df\n    \n    \n    for fold, (train_fold, valid_fold) in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])):\n        if fold < 0:\n            continue\n            \n        valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n        filt = df.StudyInstanceUID.isin(valid_patients)\n        valid_ = df.loc[filt,:].reset_index(drop=True)\n\n        image_preds_all_list = []\n        for cfg in STAGE1_CFGS:\n            valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=True)\n\n            val_loader = torch.utils.data.DataLoader(\n                valid_ds, \n                batch_size=256,\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n                sampler=SequentialSampler(valid_ds)\n            )\n\n            device = torch.device(CFG['device'])\n            model = cfg['model_constructor']().to(device)\n            model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))\n            model.eval()\n\n            image_preds_all = []\n            correct_count = 0\n            count = 0\n            for step, (imgs, target) in enumerate(val_loader):\n                imgs = imgs.to(device).float()\n                target = target.to(device).float()\n\n                image_preds = model(imgs)   #output = model(input)\n                #print(image_preds[:,0], image_preds[:,0].shape)\n                #print(target, target.shape)\n                \n                if len(image_preds.shape) == 1:\n                    image_preds = image_preds.view(-1, 1)\n                \n                correct_count += ((image_preds[:,0]>0) == target[:,0]).sum().detach().item()\n                count += imgs.shape[0]\n                image_preds_all += [image_preds.cpu().detach().numpy()]\n                print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)), end='\\r')\n            print()\n            \n            image_preds_all = np.concatenate(image_preds_all, axis=0)\n            image_preds_all_list += [image_preds_all]\n        \n            del model, val_loader\n            torch.cuda.empty_cache()\n        \n        image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)\n        df.loc[filt, new_feats] = image_preds_all_list\n        \n    df.to_csv(res_file_name, index=False)\n    return df\n            \ndef update_stage1_test_preds(df):\n    \n    new_feats = get_stage1_columns()\n    for f in new_feats:\n        df[f] = 0\n    \n    image_preds_all_list = []\n    for cfg in STAGE1_CFGS:\n        test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=False)\n\n        test_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=256,\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n            sampler=SequentialSampler(test_ds)\n        )\n\n        device = torch.device(CFG['device'])\n        model = cfg['model_constructor']().to(device)\n        model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))\n        model.eval()\n\n        image_preds_all = []\n        for step, imgs in enumerate(tqdm(test_loader)):\n            imgs = imgs.to(device).float()\n\n            image_preds = model(imgs)   #output = model(input)\n            image_preds_all += [image_preds.cpu().detach().numpy()]\n            #print(imgs[0], image_preds[0,:]); break\n        \n        #continue\n        image_preds_all = np.concatenate(image_preds_all, axis=0)\n        image_preds_all_list += [image_preds_all]\n        \n        del model, test_loader\n        torch.cuda.empty_cache()\n        \n    image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)\n    df.loc[:,new_feats] = image_preds_all_list\n    \n    return df\n\nif __name__ == '__main__':\n    if CFG['train']:\n        from  torch.cuda.amp import autocast, GradScaler # for training only, need nightly build pytorch\n\n    seed_everything(SEED)\n    \n    if CFG['train']:\n        # read train file\n        train_df = pd.read_csv(CFG['train_path'])\n\n        # read cv file\n        cv_df = pd.read_csv(CFG['cv_fold_path'])\n\n        with torch.no_grad():\n            train_df = update_stage1_oof_preds(train_df,cv_df)\n        # img must be sorted before feeding into NN for correct orders\n    else:\n        #assert False, \"This kernel is for training only!\"\n        # read test file\n        test_df = pd.read_csv(CFG['test_path'])\n        \n        with torch.no_grad():\n            test_df = update_stage1_test_preds(test_df)\n    \n    if CFG['train']:\n        for fold, (train_fold, valid_fold) in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])):\n\n            train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)\n\n            device = torch.device(CFG['device'])\n            model = RSNAClassifier().to(device)\n            \n            scaler = GradScaler()   \n            optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n            #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n            for epoch in range(CFG['epochs']):\n                train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n                with torch.no_grad():\n                    valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)\n\n            torch.save(model.state_dict(),'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n            \n            model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag'])))\n            \n            # debug\n            #valid_one_epoch(1, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)\n            \n            # prediction for oof\n            valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n            valid_ = train_df.loc[train_df.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)\n\n            with torch.no_grad():\n                val_pred_df = inference(model, device, valid_, CFG['train_img_path'])\n            \n            target = valid_[CFG['image_target_cols']].values\n            pred = (val_pred_df[CFG['image_target_cols']].values > 0.5).astype(int)\n            print('Image PE Accuracy: {:.3f}'.format((target==pred).mean()*100))\n            \n            loss = rsna_wloss_inference(valid_[CFG['image_target_cols']].values, valid_[CFG['exam_target_cols']].values, \n                                        val_pred_df[CFG['image_target_cols']].values, val_pred_df[CFG['exam_target_cols']].values, \n                                        list(valid_.groupby('StudyInstanceUID', sort=False)['SOPInstanceUID'].count()))\n\n            print('Validation loss = {:.4f}'.format(loss.detach().item()))\n            \n            del model, optimizer, train_loader, val_loader, scaler, scheduler\n            torch.cuda.empty_cache()\n            \n        train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))\n        #print(len(train_loader), len(val_loader))\n        device = torch.device(CFG['device'])\n        model = RSNAClassifier().to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n        #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n        torch.save(model.state_dict(),'{}/model_{}'.format(CFG['model_path'], CFG['tag']))\n        \n    else:\n        device = torch.device(CFG['device'])\n        model = RSNAClassifier().to(device)\n        model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))\n        test_pred_df = inference(model, device, test_df, CFG['test_img_path'])       \n        test_pred_df.to_csv('kh_submission_raw.csv')\n        \n        # transform into submission format\n        ids = []\n        labels = []\n        \n        gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False).mean()\n        for col in CFG['exam_target_cols']:\n            ids += [[patient+'_'+col for patient in gp_mean.index]]\n            labels += [gp_mean[col].values]\n            \n        ids += [test_pred_df.SOPInstanceUID.values]\n        labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]\n        ids = np.concatenate(ids)\n        labels = np.concatenate(labels)\n        \n        assert len(ids) == len(labels)\n        \n        submission = pd.DataFrame()\n        submission['id'] = ids\n        submission['label'] = labels\n        print(submission.head(3))\n        print(submission.tail(3))\n        print(submission.shape)\n        submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}