{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook is based on 2nd place solution of RSNA Intracranial Hemorrhage Detection competetion.  \nhttps://www.kaggle.com/c/rsna-intracranial-hemorrhage-detection/discussion/117228"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import ast\nimport gc\nimport math\nimport os\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.utils.data as data\nimport torch.optim as optim\nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.nn import TransformerEncoder, TransformerEncoderLayer\nimport matplotlib.pyplot as plt\nfrom torchvision import datasets, transforms\nfrom torchvision import models\n\nn_gpu = torch.cuda.device_count()\nprint(\"n_gpus: \", n_gpu)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def set_seeds(SEED):\n    np.random.seed(SEED)\n    torch.manual_seed(SEED)\n    torch.cuda.manual_seed(SEED)\n    if n_gpu > 0:\n        torch.cuda.manual_seed_all(SEED)\n    torch.backends.cudnn.deterministic = True\n\nset_seeds(SEED=2020)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = [\n    \"negative_exam_for_pe\",\n    \"rv_lv_ratio_gte_1\",\n    \"rv_lv_ratio_lt_1\",\n    \"leftsided_pe\",\n    \"chronic_pe\",\n    \"rightsided_pe\",\n    \"acute_and_chronic_pe\",\n    \"central_pe\",\n    \"indeterminate\",\n]\n\nn_classes = len(label_cols)\nn_epochs = 10\nbatch_size = 8\nnum_workers = os.cpu_count()\n\n# LSTM_UNITS = 512\nlr = 1e-5\nlrgamma = 0.95\nDECAY = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trnmdf = pd.read_csv('../input/extract-pe-meta-data/train_metadata.csv')\n# tstmdf = pd.read_csv(os.path.join(path_data, 'test_metadata.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrnmdf['SliceID'] = trnmdf[['SeriesInstanceUID', 'StudyInstanceUID']].apply(\n    lambda x: '{}__{}'.format(*x.tolist()), 1)\n# tstmdf['SliceID'] = tstmdf[['SeriesInstanceUID', 'StudyInstanceUID']].apply(\n#     lambda x: '{}__{}'.format(*x.tolist()), 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trnmdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nposcols = ['ImagePos{}'.format(i) for i in range(1, 4)]\ntrnmdf[poscols] = pd.DataFrame(trnmdf['ImagePositionPatient']\\\n              .apply(lambda x: list(map(float, ast.literal_eval(x)))).tolist())\n# tstmdf[poscols] = pd.DataFrame(tstmdf['ImagePositionPatient']\\\n#               .apply(lambda x: list(map(float, ast.literal_eval(x)))).tolist())\n\ntrnmdf = trnmdf.sort_values(['SliceID']+poscols)\\\n                [['StudyInstanceUID', 'SliceID', 'SOPInstanceUID']+poscols].reset_index(drop=True)\n# tstmdf = tstmdf.sort_values(['SliceID']+poscols)\\\n#                 [['SliceID', 'SOPInstanceUID']+poscols].reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trnmdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trnmdf['seq'] = (trnmdf.groupby(['SliceID']).cumcount() + 1)\n# tstmdf['seq'] = (tstmdf.groupby(['SliceID']).cumcount() + 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"keepcols = ['StudyInstanceUID', 'SliceID', 'SOPInstanceUID', 'seq']\ntrnmdf = trnmdf[keepcols]\n# tstmdf = tstmdf[keepcols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_emb_f = np.load(\"../input/extract-resnet18-features-for-seqeunce-model/emb_train_embdim512.npz\")\nvalid_emb_f = np.load(\"../input/extract-resnet18-features-for-seqeunce-model/emb_valid_embdim512.npz\")\n\ntrain_emb = train_emb_f[\"embddings\"]\ntrndf = pd.DataFrame({\n    \"StudyInstanceUID\": train_emb_f[\"StudyInstanceUID\"], \n    \"SeriesInstanceUID\": train_emb_f[\"SeriesInstanceUID\"],\n    \"SOPInstanceUID\": train_emb_f[\"SOPInstanceUID\"]})\n\nvalid_emb = valid_emb_f[\"embddings\"]\nvaldf = pd.DataFrame({\n    \"StudyInstanceUID\": valid_emb_f[\"StudyInstanceUID\"], \n    \"SeriesInstanceUID\": valid_emb_f[\"SeriesInstanceUID\"],\n    \"SOPInstanceUID\": valid_emb_f[\"SOPInstanceUID\"]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf = trndf[[\"SOPInstanceUID\"]].merge(trnmdf, on=\"SOPInstanceUID\", how=\"left\")\nvaldf = valdf[[\"SOPInstanceUID\"]].merge(trnmdf, on=\"SOPInstanceUID\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(trndf.shape, valdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf['embidx'] = range(trndf.shape[0])\nvaldf['embidx'] = range(valdf.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/train.csv\")\n# tstdf = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf = trndf.merge(train_df.drop(\"StudyInstanceUID\", 1), on=\"SOPInstanceUID\", how=\"left\")\nvaldf = valdf.merge(train_df.drop(\"StudyInstanceUID\", 1), on=\"SOPInstanceUID\", how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(trndf.shape)\nprint(valdf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valdf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del trnmdf, train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PEDataset(data.Dataset):\n    def __init__(self, df, mat, labels=True):\n        self.data = df\n        self.mat = mat\n        self.labels = labels\n        self.patients = df.SliceID.unique()\n        self.data = self.data.set_index('SliceID')\n\n    def __len__(self):\n        return len(self.patients)\n\n    def __getitem__(self, idx):\n        patidx = self.patients[idx]\n        patdf = self.data.loc[patidx].sort_values('seq')\n        patemb = self.mat[patdf['embidx'].values]\n\n        patdeltalag  = np.zeros(patemb.shape)\n        patdeltalead = np.zeros(patemb.shape)\n        patdeltalag[1:] = patemb[1:] - patemb[:-1]\n        patdeltalead[:-1] = patemb[:-1] - patemb[1:]\n\n        patemb = np.concatenate((patemb, patdeltalag, patdeltalead), -1)\n        # print(patemb.shape)\n        \n        ids = torch.tensor(patdf['embidx'].values)\n        \n        assert len(patemb) == len(ids), \"emb size: {} id size: {}\".format(len(patemb), len(ids))\n        \n        if self.labels:\n            labels = torch.tensor(patdf[label_cols].values[0])\n            return {'emb': patemb, 'embidx' : ids, 'labels': labels}    \n        else:      \n            return {'emb': patemb, 'embidx' : ids}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def collatefn(batch):\n    maxlen = max([l['emb'].shape[0] for l in batch])\n    embdim = batch[0]['emb'].shape[1]\n    withlabel = 'labels' in batch[0]\n#     if withlabel:\n#         labdim = batch[0]['labels'].shape[1]\n        \n    for b in batch:\n        masklen = maxlen-len(b['emb'])\n        b['emb'] = np.vstack((np.zeros((masklen, embdim)), b['emb']))\n        b['embidx'] = torch.cat((torch.ones((masklen),dtype=torch.long)*-1, b['embidx']))\n        b['mask'] = np.ones((maxlen))\n        b['mask'][:masklen] = 0.\n#         if withlabel:\n#             b['labels'] = np.vstack((np.zeros((maxlen-len(b['labels']), labdim)), b['labels']))\n            \n    outbatch = {'emb' : torch.tensor(np.vstack([np.expand_dims(b['emb'], 0) \\\n                                                for b in batch])).float()}  \n    outbatch['mask'] = torch.tensor(np.vstack([np.expand_dims(b['mask'], 0) \\\n                                                for b in batch])).float()\n    outbatch['embidx'] = torch.tensor(np.vstack([np.expand_dims(b['embidx'], 0) \\\n                                                for b in batch])).float()\n    if withlabel:\n        # outbatch['labels'] = torch.tensor(np.vstack([np.expand_dims(b['labels'], 0) for b in batch])).float()\n        outbatch[\"labels\"] = torch.tensor(np.vstack([b[\"labels\"] for b in batch])).float()\n    return outbatch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = PEDataset(trndf, train_emb, labels=True)\ntrain_loader = data.DataLoader(train_dataset, \n                               batch_size=batch_size, \n                               shuffle=True, \n                               num_workers=num_workers, \n                               collate_fn=collatefn)\n\nvalid_dataset = PEDataset(valdf, valid_emb, labels=True)\nvalid_loader = data.DataLoader(valid_dataset, \n                               batch_size=batch_size, \n                               shuffle=False, \n                               num_workers=num_workers, \n                               collate_fn=collatefn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PositionalEncoding(nn.Module):\n\n    def __init__(self, d_model, dropout=0.1, max_len=5000):\n        super(PositionalEncoding, self).__init__()\n        self.dropout = nn.Dropout(p=dropout)\n\n        pe = torch.zeros(max_len, d_model)\n        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n        pe[:, 0::2] = torch.sin(position * div_term)\n        pe[:, 1::2] = torch.cos(position * div_term)\n        pe = pe.unsqueeze(0).transpose(0, 1)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        x = x + self.pe[:x.size(0), :]\n        return self.dropout(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PETransformerModel(nn.Module):\n    def __init__(self, n_classes, ninp, nhead, nhid, nlayers, dropout=0.5):\n        super(PETransformerModel, self).__init__()\n        self.pos_encoder = PositionalEncoding(ninp, dropout)\n        encoder_layers = TransformerEncoderLayer(ninp, nhead, nhid, dropout)\n        self.transformer_encoder = TransformerEncoder(encoder_layers, nlayers)\n        self.ninp = ninp\n        self.decoder = nn.Linear(ninp, n_classes)\n        \n        self.init_weigths()\n    \n    def init_weigths(self):\n        initrange = 0.1\n        self.decoder.bias.data.zero_()\n        self.decoder.weight.data.uniform_(-initrange, initrange)\n        \n    def forward(self, x):\n        x = x * math.sqrt(self.ninp)\n        x = self.pos_encoder(x)\n        hidden = self.transformer_encoder(x)\n        print(hidden.size())\n        output = self.decoder(hidden.mean(1))\n        return output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nhid = 768\nnlayers = 2\nnhead = 2\ndropout = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embed_size = train_emb.shape[-1] * 3\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# model = NeuralNet(LSTM_UNITS=LSTM_UNITS, n_classes=n_classes)\nmodel = PETransformerModel(n_classes, embed_size, nhead, nhid, nlayers, dropout=dropout)\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_optimizer = list(model.named_parameters())\nno_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\nplist = [\n    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': DECAY},\n    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n    ]\n\noptimizer = optim.Adam(plist, lr=lr)\nscheduler = StepLR(optimizer, 1, gamma=lrgamma, last_epoch=-1)\ncriterion = torch.nn.BCEWithLogitsLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model, loader):\n    valls = []\n    current_loss_mean = 0.0\n    model.eval()\n    tqdm_loader = tqdm(loader)\n    for step, batch in enumerate(tqdm_loader):\n        mask = batch['mask'].to(device, dtype=torch.bool)\n        inputs = batch[\"emb\"]\n        inputs = inputs.to(device, dtype=torch.float)\n        y = batch['labels'].to(device, dtype=torch.float)\n        \n        logits = model(inputs)\n        \n        # get the mask for masked labels\n        # maskidx = mask.view(-1) == 1\n        \n        # reshape for\n        # logits = logits.view(-1, n_classes)[maskidx]\n        valls.append(torch.sigmoid(logits).detach().cpu().numpy())\n        \n        loss = criterion(logits, y)\n        \n        current_loss_mean = (current_loss_mean * step + loss.item()) / (step + 1)\n        tqdm_loader.set_description(f\"validation loss : {current_loss_mean:.4}\")\n    \n    return np.concatenate(valls, 0), current_loss_mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfor epoch in range(n_epochs):\n    print(\"Epoch: {}\".format(epoch + 1))\n    current_loss_mean = 0.0\n    tr_loss = 0.0\n    model.train()\n    tqdm_loader = tqdm(train_loader)\n    for step, batch in enumerate(tqdm_loader):\n        y = batch['labels'].to(device, dtype=torch.float)\n        mask = batch['mask'].to(device, dtype=torch.bool)\n        x = batch['emb'].to(device, dtype=torch.float)\n        \n        logits = model(x).to(device, dtype=torch.float)\n        \n        # get the mask for masked labels\n        # maskidx = mask.view(-1) == 1\n        # y = y.view(-1, n_classes)[maskidx]\n        # logits = logits.view(-1, n_classes)[maskidx]\n        \n        # Get loss\n        loss = criterion(logits, y)\n        tr_loss += loss.item()\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        \n        if step % 100 == 0:\n            print('Trn step {} of {} trn lossavg {:.5f}'. \\\n                  format(step, len(train_loader), (tr_loss / (1 + step))))\n        \n        current_loss_mean = (current_loss_mean * step + loss.item()) / (step + 1)\n        tqdm_loader.set_description(f\"train loss : {current_loss_mean:.4}\")\n    \n    output_model_file = \"transformer_epoch{}.pth\".format(epoch + 1)\n    torch.save(model.state_dict(), output_model_file)\n\n    scheduler.step()\n    \n    logits, val_loss = validate(model, valid_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}