{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi to all! In this notebook, I did not change anything in the original solution (https://www.kaggle.com/khyeh0719/cnn-gru-baseline-stage2-train-inference), but I added a few strips of code according to these recommendations: https: // www.kaggle.com/c/rsna-str-pulmonary-embolism-detection/discussion/186982. As a result, it has greatly reduced the time to submit a notebook on a public test (not privat test)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"package_path = '../input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'\nimport sys; sys.path.append(package_path)\n!cp ../input/gdcm-conda-install/gdcm.tar .\n!tar -xvzf gdcm.tar\n!conda install --offline ./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from glob import glob\nfrom sklearn.model_selection import GroupKFold\nimport cv2\nfrom skimage import io\nimport torch\nfrom torch import nn\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport torchvision\nfrom torchvision import transforms\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\n\nimport sklearn\nimport warnings\nimport joblib\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn import metrics\nimport warnings\nimport cv2\nimport pydicom\nfrom efficientnet_pytorch import EfficientNet\nfrom scipy.ndimage.interpolation import zoom\nfrom tqdm import tqdm\n\n\nCFG = {\n    'train': False,\n    \n    'train_img_path': '../input/rsna-str-pulmonary-embolism-detection/train',\n    'test_img_path': '../input/rsna-str-pulmonary-embolism-detection/test',\n    'cv_fold_path': '../input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',\n    'train_path': '../input/rsna-str-pulmonary-embolism-detection/train.csv',\n    'test_path': '../input/rsna-str-pulmonary-embolism-detection/test.csv',\n    \n    'image_target_cols': [\n        'pe_present_on_image', # only image level\n    ],\n    \n    'exam_target_cols': [\n        'negative_exam_for_pe', # exam level\n        #'qa_motion',\n        #'qa_contrast',\n        #'flow_artifact',\n        'rv_lv_ratio_gte_1', # exam level\n        'rv_lv_ratio_lt_1', # exam level\n        'leftsided_pe', # exam level\n        'chronic_pe', # exam level\n        #'true_filling_defect_not_pe',\n        'rightsided_pe', # exam level\n        'acute_and_chronic_pe', # exam level\n        'central_pe', # exam level\n        'indeterminate' # exam level\n    ], \n    \n    'img_num': 200,\n    'img_size': 256,\n    'lr': 0.0005,\n    'epochs': 2,\n    'device': 'cuda', # cuda, cpu\n    'train_bs': 2,\n    'accum_iter': 8,\n    'verbose_step': 1,\n    'num_workers': 4,\n    'efbnet': 'efficientnet-b0',\n    \n    'train_folds': [np.arange(0,16),\n                    np.concatenate([np.arange(0,12), np.arange(16,20)]),\n                    np.concatenate([np.arange(0,8), np.arange(12,20)]),\n                    np.concatenate([np.arange(0,4), np.arange(8,20)]),\n                    np.arange(4,20),\n                   ],#[np.arange(0,16)],\n    \n    'valid_folds': [np.arange(16,20),\n                    np.arange(12,16),\n                    np.arange(8,12),\n                    np.arange(4,8),\n                    np.arange(0,4)\n                   ],#[np.arange(16,20)],\n    \n    'model_path': '../input/kh-rsna-model',\n    'tag': 'efb0_stage2_multilabel'\n}\n\nSEED = 42321\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\ndef window(img, WL=50, WW=350):\n    upper, lower = WL+WW//2, WL-WW//2\n    X = np.clip(img.copy(), lower, upper)\n    X = X - np.min(X)\n    X = X / np.max(X)\n    #X = (X*255.0).astype('uint8')\n    return X\n\ndef get_img(path):\n    \n    d = pydicom.read_file(path)\n    '''\n    res = cv2.resize((d.pixel_array - d.RescaleIntercept) / (d.RescaleSlope * 1000), (CFG['img_size'], CFG['img_size'])), d.ImagePositionPatient[2]\n    '''\n    \n    '''\n    RED channel / LUNG window / level=-600, width=1500\n    GREEN channel / PE window / level=100, width=700\n    BLUE channel / MEDIASTINAL window / level=40, width=400\n    '''\n    \n    img = (d.pixel_array * d.RescaleSlope) + d.RescaleIntercept\n    \n    r = window(img, -600, 1500)\n    g = window(img, 100, 700)\n    b = window(img, 40, 400)\n    \n    res = np.concatenate([r[:, :, np.newaxis],\n                          g[:, :, np.newaxis],\n                          b[:, :, np.newaxis]], axis=-1)\n    \n    #res = (res*255.0).astype('uint8')\n    res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)\n    #res = cv2.resize(res, (CFG['img_size'], CFG['img_size']))\n    #res = res.astype(np.float32)/255.\n    \n    return res\n\nclass RSNADatasetStage1(Dataset):\n    def __init__(\n        self, df, label_smoothing, data_root, \n        image_subsampling=True, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.label_smoothing = label_smoothing\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n    \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        # get labels\n        if self.output_label:\n            target = self.df[CFG['image_target_cols']].values[index]\n            \n        path = \"{}/{}/{}/{}.dcm\".format(self.data_root, \n                                        self.df.iloc[index]['StudyInstanceUID'], \n                                        self.df.iloc[index]['SeriesInstanceUID'], \n                                        self.df.iloc[index]['SOPInstanceUID'])\n        img  = get_img(path)\n        \n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        # do label smoothing\n        if self.output_label == True:\n            target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)\n            \n            return img, target\n        else:\n            return img\n        \nclass RSNADataset(Dataset):\n    def __init__(\n        self, df, label_smoothing, data_root, \n        image_subsampling=True, transforms=None, output_label=True\n    ):\n        \n        super().__init__()\n        self.df = df\n        self.patients = self.df['StudyInstanceUID'].unique()\n        self.image_subsampling = image_subsampling\n        self.label_smoothing = label_smoothing\n        self.transforms = transforms\n        self.data_root = data_root\n        self.output_label = output_label\n        \n    def get_patients(self):\n        return self.patients\n        \n    def __len__(self):\n        return len(self.patients)\n    \n    def __getitem__(self, index: int):\n        \n        patient = self.patients[index]\n        df_ = self.df.loc[self.df.StudyInstanceUID == patient]\n        \n        per_image_feats = get_stage1_columns()\n        #print(per_image_feats)\n        \n        if self.image_subsampling:\n            img_num = min(CFG['img_num'], df_.shape[0])\n            \n            # naive image subsampling\n            img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)\n            \n            # get all images, then slice location and sort according to z values\n            imgs = np.zeros((CFG['img_num'],), np.float32) #np.zeros((CFG['img_num'], CFG['img_size'], CFG['img_size'], 3), np.float32)\n            per_image_preds = np.zeros((CFG['img_num'], len(per_image_feats)), np.float32)\n            locs = np.zeros((CFG['img_num'],), np.float32)\n            image_masks = np.zeros((CFG['img_num'],), np.float32)\n            image_masks[:img_num] = 1.\n            \n            # get labels\n            if self.output_label:\n                exam_label = df_[CFG['exam_target_cols']].values[0]\n                image_labels = np.zeros((CFG['img_num'], len(CFG['image_target_cols'])), np.float32)\n            \n        else:\n            img_num = df_.shape[0]\n            img_ix = np.arange(df_.shape[0])\n            \n            # get all images, then slice location and sort according to z values\n            imgs = np.zeros((img_num, ), np.float32) #np.zeros((img_num, CFG['img_size'], CFG['img_size'], 3), np.float32)\n            per_image_preds = np.zeros((img_num, len(per_image_feats)), np.float32)\n            locs = np.zeros((img_num,), np.float32)\n            image_masks = np.zeros((img_num,), np.float32)\n            image_masks[:img_num] = 1.\n            \n            # get labels\n            if self.output_label:\n                exam_label = df_[CFG['exam_target_cols']].values[0]\n                image_labels = np.zeros((img_num, len(CFG['image_target_cols'])), np.float32)\n                \n        for i, im_ix in enumerate(img_ix):\n            path = \"{}/{}/{}/{}.dcm\".format(self.data_root, \n                                            df_['StudyInstanceUID'].values[im_ix], \n                                            df_['SeriesInstanceUID'].values[im_ix], \n                                            df_['SOPInstanceUID'].values[im_ix])\n            \n            d = pydicom.read_file(path)\n            locs[i] = d.ImagePositionPatient[2]\n            per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]\n            \n            if self.output_label == True:\n                image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]\n\n        #print('get img done')\n        \n        seq_ix = np.argsort(locs)\n        \n        # image features: img_num * img_size * img_size * 1\n        '''\n        imgs = imgs[seq_ix]\n        if self.transforms:\n            imgs = [self.transforms(image=img)['image'] for img in imgs]\n        imgs = torch.stack(imgs)\n        '''\n        \n        # image level features: img_num\n        #locs[:img_num] -= locs[:img_num].min()\n        locs = locs[seq_ix]\n        locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]\n        locs[0] = 0\n        \n        per_image_preds = per_image_preds[seq_ix]\n        \n        # patient level features: 1\n        \n        # train, train-time valid, multiple patients: imgs, locs, image_labels, exam_label, img_num\n        # whole valid-time valid, single patient: imgs, locs, image_labels, exam_label, img_num, sorted id\n        # whole test-time test, single patient: imgs, locs, img_num, sorted_id\n        \n        # do label smoothing\n        if self.output_label == True:\n            image_labels = image_labels[seq_ix]\n            image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)\n            exam_label =  np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)\n            \n            return imgs, per_image_preds, locs, image_labels, exam_label, image_masks\n        else:\n            return imgs, per_image_preds, locs, img_num, index, seq_ix\n\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,\n    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,\n    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout\n)\n\nfrom albumentations.pytorch import ToTensorV2\n\ndef get_train_transforms():\n    return Compose([\n            #HorizontalFlip(p=0.5),\n            #VerticalFlip(),\n            #RandomRotate90(p=0.5),\n            #Cutout(p=0.5),\n            #Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''   \n        \ndef get_valid_transforms():\n    return Compose([\n            #Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n            ToTensorV2(p=1.0),\n        ], p=1.)\n    '''\n    return transforms.Compose([\n            transforms.Lambda(lambda imgs: torch.stack([transforms.ToTensor()(img) for img in imgs])),\n            transforms.Lambda(lambda imgs: torch.stack([transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                                                             std=[0.229, 0.224, 0.225])(img) for img in imgs])),\n           \n        ])\n    '''  \n\nclass RNSAImageFeatureExtractor(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = EfficientNet.from_name(CFG['efbnet'])\n        #print(self.cnn_model, CFG['efbnet'])\n        self.pooling = nn.AdaptiveAvgPool2d(1)\n        \n    def get_dim(self):\n        return self.cnn_model._fc.in_features\n        \n    def forward(self, x):\n        feats = self.cnn_model.extract_features(x)\n        return self.pooling(feats).view(x.shape[0], -1)                         \n\nclass RSNAImgClassifierSingle(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = RNSAImageFeatureExtractor()\n        self.image_predictors = nn.Linear(self.cnn_model.get_dim(), 1)\n        \n    def forward(self, imgs):\n        #print(images.shape)\n        imgs_embdes = self.cnn_model(imgs) # bs * efb_feat_size\n        #print(imgs_embdes.shape)\n        image_preds = self.image_predictors(imgs_embdes)\n        \n        return image_preds\n    \nclass RSNAImgClassifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.cnn_model = RNSAImageFeatureExtractor()\n        self.image_predictors = nn.Linear(self.cnn_model.get_dim(), 9)\n        \n    def forward(self, imgs):\n        #print(images.shape)\n        imgs_embdes = self.cnn_model(imgs) # bs * efb_feat_size\n        #print(imgs_embdes.shape)\n        image_preds = self.image_predictors(imgs_embdes)\n        \n        return image_preds\n\nclass TimeDistributed(nn.Module):\n\n    def __init__(self, module, batch_first=True):\n        super(TimeDistributed, self).__init__()\n        self.module = module\n        self.batch_first = batch_first\n\n    def forward(self, x):\n        ''' x size: (batch_size, time_steps, in_channels, height, width) '''\n        x_size= x.size()\n        c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])\n        \n        c_out = self.module(c_in)\n        r_in = c_out.view(x_size[0], x_size[1], -1)\n        if self.batch_first is False:\n            r_in = r_in.permute(1, 0, 2)\n        return r_in \n    \nclass RSNAClassifier(nn.Module):\n    def __init__(self, hidden_size=64):\n        super().__init__()\n        \n        self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)\n        \n        self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))\n        self.exam_predictor = nn.Linear(hidden_size*2*2, 9)\n        \n    def forward(self, img_preds, locs):\n        \n        embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2) # bs * ts * fs\n        \n        embeds, _ = self.gru(embeds)\n        image_preds = self.image_predictors(embeds)\n        \n        avg_pool = torch.mean(embeds, 1)\n        max_pool, _ = torch.max(embeds, 1)\n        conc = torch.cat([avg_pool, max_pool], 1)\n        \n        exam_pred = self.exam_predictor(conc)\n        return image_preds, exam_pred\n    \n#RSNAClassifier(64)\ndef rsna_wloss_inference(y_true_img, y_true_exam, y_pred_img, y_pred_exam, chunk_sizes):\n    # y_true_img, y_pred_img: (p1*in1 + p2*in2 + ,,,) \n    # y_true_exam, y_pred_exam: (p1*in1 + p2*in2 + ,,,) x 9\n    # chunk_sizes: (patient_num)\n    '''\n    'negative_exam_for_pe', # exam level 0.0736196319\n    'rv_lv_ratio_gte_1', # exam level 0.2346625767\n    'rv_lv_ratio_lt_1', # exam level 0.0782208589\n    'leftsided_pe', # exam level 0.06257668712\n    'chronic_pe', # exam level 0.1042944785\n    'rightsided_pe', # exam level 0.06257668712\n    'acute_and_chronic_pe', # exam level 0.1042944785\n    'central_pe', # exam level 0.1877300613\n    'indeterminate' # exam level 0.09202453988\n    '''\n    \n    # transform into torch tensors\n    y_true_img, y_true_exam, y_pred_img, y_pred_exam = torch.tensor(y_true_img, dtype=torch.float32), torch.tensor(y_true_exam, dtype=torch.float32), torch.tensor(y_pred_img, dtype=torch.float32), torch.tensor(y_pred_exam, dtype=torch.float32)\n    \n    # split into chunks (each chunks is for a single exam)\n    y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks = torch.split(y_true_img, chunk_sizes, dim=0), torch.split(y_true_exam, chunk_sizes, dim=0), torch.split(y_pred_img, chunk_sizes, dim=0), torch.split(y_pred_exam, chunk_sizes, dim=0)\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCELoss(reduction='none')\n    \n    total_loss = torch.tensor(0, dtype=torch.float32)\n    total_weights = torch.tensor(0, dtype=torch.float32)\n    for i, (y_true_img_, y_true_exam_, y_pred_img_, y_pred_exam_) in enumerate(zip(y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks)):\n        exam_loss = bce_func(y_pred_exam_[0, :], y_true_exam_[0, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        image_loss = bce_func(y_pred_img_, y_true_img_)\n        img_num = chunk_sizes[i]\n        qi = torch.sum(y_true_img_)/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss\n\ndef rsna_wloss_train(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):\n    # y_true_img, y_pred_img: patient_numximg_num\n    # y_true_exam, y_pred_exam: patient_num x 9\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1).to(device)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCEWithLogitsLoss(reduction='none').to(device)\n    \n    total_loss = torch.tensor(0, dtype=torch.float32).to(device)\n    total_weights = torch.tensor(0, dtype=torch.float32).to(device)\n    for i in range(y_true_img.shape[0]):\n        exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        img_mask = image_masks[i]\n        #print(torch.sum(y_true_img[i,:]), torch.sum(img_mask))\n        image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:]).flatten()\n        #print(image_loss.shape)\n        #print(img_mask.shape)\n        #print((image_loss*img_mask).shape)\n        #assert False\n        image_loss = image_loss*img_mask # mark 0 loss for padding images\n        img_num = torch.sum(img_mask) #y_true_img.shape[1]\n        qi = torch.sum(y_true_img[i,:])/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss, total_loss, total_weights\n\ndef rsna_wloss_valid(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):\n    # y_true_img, y_pred_img: patient_numximg_num\n    # y_true_exam, y_pred_exam: patient_num x 9\n    \n    label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988]).view(1, -1).to(device)\n    img_w = 0.07361963\n    bce_func = torch.nn.BCEWithLogitsLoss(reduction='none').to(device)\n    \n    total_loss = torch.tensor(0, dtype=torch.float32).to(device)\n    total_weights = torch.tensor(0, dtype=torch.float32).to(device)\n    for i in range(y_true_img.shape[0]):\n        exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])\n        exam_loss = torch.sum(exam_loss*label_w, 1)[0] # Kaggle uses a binary log loss equation for each label and then takes the mean of the log loss over all labels.\n        #print(exam_loss)\n\n        img_mask = image_masks[i]\n        #print(torch.sum(y_true_img[i,:]), torch.sum(img_mask))\n        image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:]).flatten()\n        #print(image_loss.shape)\n        #print(img_mask.shape)\n        #print((image_loss*img_mask).shape)\n        #assert False\n        image_loss = image_loss*img_mask # mark 0 loss for padding images\n        img_num = torch.sum(img_mask) #y_true_img.shape[1]\n        qi = torch.sum(y_true_img[i,:])/img_num\n        image_loss = torch.sum(img_w*qi*image_loss)\n        #print(image_loss)\n    \n        total_loss += exam_loss+image_loss\n        total_weights += label_w.sum() + img_w*qi*img_num\n        #assert False\n        \n    final_loss = total_loss/total_weights\n    return final_loss, total_loss, total_weights\n\ndef prepare_train_dataloader(train, cv_df, train_fold, valid_fold):\n    train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()\n    valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n\n    train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)\n    valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)\n\n    # train mode to do image-level subsampling\n    train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'],  image_subsampling=True, transforms=get_train_transforms(), output_label=True) \n    valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=True)\n\n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=True,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=1,\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    #print(len(train_loader), len(val_loader))\n\n    return train_loader, val_loader\n\ndef train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):\n    model.train()\n\n    t = time.time()\n    loss_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, per_image_preds, locs, image_labels, exam_label, image_masks) in enumerate(train_loader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        image_masks = image_masks.to(device).float()\n        image_labels = image_labels.to(device).float()\n        exam_label = exam_label.to(device).float()\n\n        #print(image_labels.shape, exam_label.shape)\n        with autocast():\n            image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n            #print(image_preds.shape, exam_pred.shape)\n\n            loss, total_loss, total_weights = rsna_wloss_train(image_labels, exam_label, image_preds, exam_pred, image_masks, device)\n\n            scaler.scale(loss).backward()\n\n            loss_sum += total_loss.detach().item()\n            loss_w_sum += total_weights.detach().item()\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                # may unscale_ here if desired (e.g., to allow clipping unscaled gradients)\n\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad()                \n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                print(\n                    f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \\\n                    f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                    f'time: {(time.time() - t):.4f}', end= '\\r' if (step + 1) != len(train_loader) else '\\n'\n                )\n\ndef post_process(exam_pred, image_pred):\n    \n    rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')\n    rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')\n    central_pe_ix = CFG['exam_target_cols'].index('central_pe')\n    rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')\n    leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')\n    acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')\n    chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')\n    negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')\n    indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')\n    \n    # rule 1 or rule 2 judgement: if any pe image exist\n    has_pe_image = torch.max(image_pred, 1)[0][0] > 0\n    #print(has_pe_image)\n    \n    # rule 1-a: only one >= 0.5, the other < 0.5\n    rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]\n    rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1) # to make one at least > 0.5\n    rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a)) # turn back into logits\n    exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)\n    \n    # rule 1-b-1 or 1-b-2 judgement: at least one > 0.5\n    crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]\n    has_no_pe = torch.max(crl_pe ,1)[0] <= 0 # all <= 0.5\n    #print(has_no_pe)\n    #assert False\n        \n    # rule 1-b\n    max_val = torch.max(crl_pe, 1)[0]\n    crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)\n    exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)\n    \n    # rule 1-c-1 or 1-c-2 judgement: at most one > 0.5\n    ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]\n    both_ac_ch = torch.min(ac_pe ,1)[0] > 0 # all > 0.5\n    \n    # rule 1-c\n    ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1) # to make only one > 0.5\n    ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c)) # turn back into logits\n    exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)\n    \n    # rule 1-d\n    neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]\n    neg_ind_1d = torch.clamp(neg_ind, max=0)\n    exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)\n    \n    # rule 2-a\n    ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]\n    ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1) # to make one at least > 0.5\n    ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a)) # turn back into logits\n    exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)\n    \n    # rule 2-b\n    all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,\n                                          central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,\n                                          acute_and_chronic_pe_ix, chronic_pe_ix]]\n    all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)\n    exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,\n                  central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,\n                  acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)\n    \n    return exam_pred, image_pred\n    \ndef valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):\n    model.eval()\n\n    t = time.time()\n    loss_sum = 0\n    loss_w_sum = 0\n\n    for step, (imgs, per_image_preds, locs, image_labels, exam_label, image_masks) in enumerate(val_loader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        image_masks = image_masks.to(device).float()\n        image_labels = image_labels.to(device).float()\n        exam_label = exam_label.to(device).float()\n\n        #print(image_labels.shape, exam_label.shape)\n        #with autocast():\n        image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        exam_pred, image_preds= post_process(exam_pred, image_preds)\n\n        loss, total_loss, total_weights = rsna_wloss_valid(image_labels, exam_label, image_preds, exam_pred, image_masks, device)\n\n        loss_sum += total_loss.detach().item()\n        loss_w_sum += total_weights.detach().item()          \n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            print(\n                f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \\\n                f'loss: {loss_sum/loss_w_sum:.4f}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(val_loader) else '\\n'\n            )\n    \n    if schd_loss_update:\n        scheduler.step(loss_sum/loss_w_sum)\n    else:\n        scheduler.step()\n\ndef check_label_consistency(checking_df):\n    # CHECKING CONSISTENCY OF POSITIVE EXAM LABELS\n    df = checking_df.copy()\n    print(df.shape)\n    df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID']).pe_present_on_image.max())\n\n    df_pos = df.loc[df.positive_images_in_exam >  0.5]\n    df_neg = df.loc[df.positive_images_in_exam <= 0.5]\n\n    rule1a = df_pos.loc[((df_pos.rv_lv_ratio_lt_1  >  0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 >  0.5)) | \n                        ((df_pos.rv_lv_ratio_lt_1  <= 0.5)  & \n                         (df_pos.rv_lv_ratio_gte_1 <= 0.5))].reset_index(drop = True)\n    rule1a['broken_rule'] = '1a'\n\n    rule1b = df_pos.loc[(df_pos.central_pe    <= 0.5) & \n                        (df_pos.rightsided_pe <= 0.5) & \n                        (df_pos.leftsided_pe  <= 0.5)].reset_index(drop = True)\n    rule1b['broken_rule'] = '1b'\n\n    rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5) & \n                        (df_pos.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule1c['broken_rule'] = '1c'\n    # CHECKING CONSISTENCY OF NEGATIVE EXAM LABELS\n\n    rule1d = df_pos.loc[(df_pos.indeterminate        > 0.5) | \n                        (df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)\n    rule1d['broken_rule'] = '1d'\n\n    rule2a = df_neg.loc[((df_neg.indeterminate        >  0.5)  & \n                         (df_neg.negative_exam_for_pe >  0.5)) | \n                        ((df_neg.indeterminate        <= 0.5)  & \n                         (df_neg.negative_exam_for_pe <= 0.5))].reset_index(drop = True)\n    rule2a['broken_rule'] = '2a'\n\n    rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1     > 0.5) | \n                        (df_neg.rv_lv_ratio_gte_1    > 0.5) |\n                        (df_neg.central_pe           > 0.5) | \n                        (df_neg.rightsided_pe        > 0.5) | \n                        (df_neg.leftsided_pe         > 0.5) |\n                        (df_neg.acute_and_chronic_pe > 0.5) | \n                        (df_neg.chronic_pe           > 0.5)].reset_index(drop = True)\n    rule2b['broken_rule'] = '2b'\n    # MERGING INCONSISTENT PREDICTIONS\n    errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)\n    \n    print('label in-consistency counts:', errors.shape)\n        \n    if errors.shape[0] > 0:\n        print(errors.broken_rule.value_counts())\n        print(errors)\n        assert False\n        \ndef inference(model, device, df, root_path):\n    model.eval()\n\n    t = time.time()\n\n    ds = RSNADataset(df, 0.0, root_path,  image_subsampling=False, transforms=get_valid_transforms(), output_label=False)\n    \n    dataloader = torch.utils.data.DataLoader(\n        ds, \n        batch_size=1,\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    \n    patients = ds.get_patients()\n    \n    res_dfs = []\n    \n    for step, (imgs, per_image_preds, locs, img_num, index, seq_ix) in enumerate(dataloader):\n        imgs = imgs.to(device).float()\n        per_image_preds = per_image_preds.to(device).float()\n        locs = locs.to(device).float()\n        \n        index = index.detach().numpy()[0]\n        seq_ix = seq_ix.detach().numpy()[0,:]\n        \n        patient_filt = (df.StudyInstanceUID == patients[index])\n        \n        patient_df = pd.DataFrame()\n        patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]\n        patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values # no need to sort\n        patient_df['StudyInstanceUID'] = patients[index] # single value\n        \n        for c in CFG['image_target_cols']+CFG['exam_target_cols']:\n            patient_df[c] = 0.0\n\n        #with autocast():\n        image_preds, exam_pred = model(per_image_preds, locs)   #output = model(input)\n        #print(image_preds.shape, exam_pred.shape)\n        \n        exam_pred, image_preds = post_process(exam_pred, image_preds)\n        \n        exam_pred = torch.sigmoid(exam_pred).cpu().detach().numpy()\n        image_preds = torch.sigmoid(image_preds).cpu().detach().numpy()\n\n        patient_df[CFG['exam_target_cols']] = exam_pred[0]\n        patient_df[CFG['image_target_cols']] = image_preds[0,:]\n        res_dfs += [patient_df]\n\n        '''\n        res_df = res_df.merge(patient_df, on=['SOPInstanceUID', 'StudyInstanceUID'], how='left')\n        '''\n        # naive slow version\n        '''\n        res_df.loc[patient_filt, CFG['exam_target_cols']] = exam_pred[0]\n        for si, sop_id in enumerate(sop_ids):\n            sop_filt = (patient_filt) & (res_df.SOPInstanceUID == sop_id)\n            res_df.loc[sop_filt, CFG['image_target_cols']] = image_preds[0, si]\n        '''\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(dataloader)):\n            print(\n                f'Inference Step {step+1}/{len(dataloader)}, ' + \\\n                f'time: {(time.time() - t):.4f}', end='\\r' if (step + 1) != len(dataloader) else '\\n'\n            )\n                \n    res_dfs = pd.concat(res_dfs, axis=0).reset_index(drop=True)\n    res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))\n    print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))\n    assert res_dfs.shape[0] == df.shape[0]\n    check_label_consistency(res_dfs)\n    \n    return res_dfs\n  \nSTAGE1_CFGS = [\n    {\n        'tag': 'efb0_stage1',\n        'model_constructor': RSNAImgClassifierSingle,\n        'dataset_constructor': RSNADatasetStage1,\n        'output_len': 1\n    },\n    {\n        'tag': 'efb0_stage1_multilabel',\n        'model_constructor': RSNAImgClassifier,\n        'dataset_constructor': RSNADatasetStage1,\n        'output_len': 9\n    },\n]\nSTAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'\n\n\ndef get_stage1_columns():\n    \n    new_feats = []\n    for cfg in STAGE1_CFGS:\n        for i in range(cfg['output_len']):\n            f = cfg['tag']+'_'+str(i)\n            new_feats += [f]\n        \n    return new_feats\n\ndef update_stage1_oof_preds(df, cv_df):\n    \n    res_file_name = STAGE1_CFGS_TAG+\"-train.csv\"    \n    \n    new_feats = get_stage1_columns()\n    for f in new_feats:\n        df[f] = 0\n    \n    if os.path.isfile(res_file_name):\n        df = pd.read_csv(res_file_name)\n        print('img acc:', ((df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]]).mean())\n        return df\n    \n    \n    for fold, (train_fold, valid_fold) in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])):\n        if fold < 0:\n            continue\n            \n        valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n        filt = df.StudyInstanceUID.isin(valid_patients)\n        valid_ = df.loc[filt,:].reset_index(drop=True)\n\n        image_preds_all_list = []\n        for cfg in STAGE1_CFGS:\n            valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=True)\n\n            val_loader = torch.utils.data.DataLoader(\n                valid_ds, \n                batch_size=256,\n                num_workers=CFG['num_workers'],\n                shuffle=False,\n                pin_memory=False,\n                sampler=SequentialSampler(valid_ds)\n            )\n\n            device = torch.device(CFG['device'])\n            model = cfg['model_constructor']().to(device)\n            model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))\n            model.eval()\n\n            image_preds_all = []\n            correct_count = 0\n            count = 0\n            for step, (imgs, target) in enumerate(val_loader):\n                imgs = imgs.to(device).float()\n                target = target.to(device).float()\n\n                image_preds = model(imgs)   #output = model(input)\n                #print(image_preds[:,0], image_preds[:,0].shape)\n                #print(target, target.shape)\n                \n                if len(image_preds.shape) == 1:\n                    image_preds = image_preds.view(-1, 1)\n                \n                correct_count += ((image_preds[:,0]>0) == target[:,0]).sum().detach().item()\n                count += imgs.shape[0]\n                image_preds_all += [image_preds.cpu().detach().numpy()]\n                print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)), end='\\r')\n            print()\n            \n            image_preds_all = np.concatenate(image_preds_all, axis=0)\n            image_preds_all_list += [image_preds_all]\n        \n            del model, val_loader\n            torch.cuda.empty_cache()\n        \n        image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)\n        df.loc[filt, new_feats] = image_preds_all_list\n        \n    df.to_csv(res_file_name, index=False)\n    return df\n            \ndef update_stage1_test_preds(df):\n    \n    new_feats = get_stage1_columns()\n    for f in new_feats:\n        df[f] = 0\n    \n    image_preds_all_list = []\n    for cfg in STAGE1_CFGS:\n        test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'],  image_subsampling=False, transforms=get_valid_transforms(), output_label=False)\n\n        test_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=256,\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=False,\n            sampler=SequentialSampler(test_ds)\n        )\n\n        device = torch.device(CFG['device'])\n        model = cfg['model_constructor']().to(device)\n        model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))\n        model.eval()\n\n        image_preds_all = []\n        for step, imgs in enumerate(tqdm(test_loader)):\n            imgs = imgs.to(device).float()\n\n            image_preds = model(imgs)   #output = model(input)\n            image_preds_all += [image_preds.cpu().detach().numpy()]\n            #print(imgs[0], image_preds[0,:]); break\n        \n        #continue\n        image_preds_all = np.concatenate(image_preds_all, axis=0)\n        image_preds_all_list += [image_preds_all]\n        \n        del model, test_loader\n        torch.cuda.empty_cache()\n        \n    image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)\n    df.loc[:,new_feats] = image_preds_all_list\n    \n    return df\n\nif __name__ == '__main__':\n    if CFG['train']:\n        from  torch.cuda.amp import autocast, GradScaler # for training only, need nightly build pytorch\n\n    seed_everything(SEED)\n    \n    if CFG['train']:\n        # read train file\n        train_df = pd.read_csv(CFG['train_path'])\n\n        # read cv file\n        cv_df = pd.read_csv(CFG['cv_fold_path'])\n\n        with torch.no_grad():\n            train_df = update_stage1_oof_preds(train_df,cv_df)\n        # img must be sorted before feeding into NN for correct orders\n    else:\n        #assert False, \"This kernel is for training only!\"\n        # read test file\n        from os import path\n        do_full=False\n        if path.exists('../input/rsna-str-pulmonary-embolism-detection/train') and not do_full:\n            test_df=pd.read_csv(CFG['test_path']).head(50)\n        else:\n            test_df=pd.read_csv(CFG['test_path'])\n        #test_df = pd.read_csv(CFG['test_path'])\n        \n        with torch.no_grad():\n            test_df = update_stage1_test_preds(test_df)\n    \n    if CFG['train']:\n        for fold, (train_fold, valid_fold) in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])):\n\n            train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)\n\n            device = torch.device(CFG['device'])\n            model = RSNAClassifier().to(device)\n            \n            scaler = GradScaler()   \n            optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n            #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n            scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n            for epoch in range(CFG['epochs']):\n                train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n                with torch.no_grad():\n                    valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)\n\n            torch.save(model.state_dict(),'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))\n            \n            model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag'])))\n            \n            # debug\n            #valid_one_epoch(1, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)\n            \n            # prediction for oof\n            valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()\n            valid_ = train_df.loc[train_df.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)\n\n            with torch.no_grad():\n                val_pred_df = inference(model, device, valid_, CFG['train_img_path'])\n            \n            target = valid_[CFG['image_target_cols']].values\n            pred = (val_pred_df[CFG['image_target_cols']].values > 0.5).astype(int)\n            print('Image PE Accuracy: {:.3f}'.format((target==pred).mean()*100))\n            \n            loss = rsna_wloss_inference(valid_[CFG['image_target_cols']].values, valid_[CFG['exam_target_cols']].values, \n                                        val_pred_df[CFG['image_target_cols']].values, val_pred_df[CFG['exam_target_cols']].values, \n                                        list(valid_.groupby('StudyInstanceUID', sort=False)['SOPInstanceUID'].count()))\n\n            print('Validation loss = {:.4f}'.format(loss.detach().item()))\n            \n            del model, optimizer, train_loader, val_loader, scaler, scheduler\n            torch.cuda.empty_cache()\n            \n        train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))\n        #print(len(train_loader), len(val_loader))\n        device = torch.device(CFG['device'])\n        model = RSNAClassifier().to(device)\n        scaler = GradScaler()   \n        optimizer = torch.optim.Adam(model.parameters(), lr=CFG['lr'])\n        #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=1); schd_loss_update=True\n        scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False\n\n        for epoch in range(CFG['epochs']):\n            train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)\n\n        torch.save(model.state_dict(),'{}/model_{}'.format(CFG['model_path'], CFG['tag']))\n        \n    else:\n        device = torch.device(CFG['device'])\n        model = RSNAClassifier().to(device)\n        model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))\n        test_pred_df = inference(model, device, test_df, CFG['test_img_path'])       \n        test_pred_df.to_csv('kh_submission_raw.csv')\n        \n        # transform into submission format\n        ids = []\n        labels = []\n        \n        gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False).mean()\n        for col in CFG['exam_target_cols']:\n            ids += [[patient+'_'+col for patient in gp_mean.index]]\n            labels += [gp_mean[col].values]\n            \n        ids += [test_pred_df.SOPInstanceUID.values]\n        labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]\n        ids = np.concatenate(ids)\n        labels = np.concatenate(labels)\n        \n        assert len(ids) == len(labels)\n        \n        submission = pd.DataFrame()\n        submission['id'] = ids\n        submission['label'] = labels\n        print(submission.head(3))\n        print(submission.tail(3))\n        print(submission.shape)\n        submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}