{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport glob\nimport pandas as pd\nimport numpy as np\nimport pydicom \n\n\nimport torch\nimport torch.optim as optim\nfrom albumentations import Compose, ShiftScaleRotate, Resize, Normalize, HorizontalFlip, RandomBrightnessContrast\nfrom albumentations.pytorch import ToTensorV2\nfrom torch.utils.data import Dataset,Subset\n\nfrom sklearn.metrics import roc_auc_score, f1_score, accuracy_score, jaccard_score, confusion_matrix\nfrom tqdm import notebook as tqdm","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.132775Z","iopub.execute_input":"2022-10-23T13:33:15.133085Z","iopub.status.idle":"2022-10-23T13:33:15.139678Z","shell.execute_reply.started":"2022-10-23T13:33:15.133049Z","shell.execute_reply":"2022-10-23T13:33:15.138966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/'","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.146718Z","iopub.execute_input":"2022-10-23T13:33:15.14762Z","iopub.status.idle":"2022-10-23T13:33:15.151629Z","shell.execute_reply.started":"2022-10-23T13:33:15.147583Z","shell.execute_reply":"2022-10-23T13:33:15.150836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(INPUT_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.153587Z","iopub.execute_input":"2022-10-23T13:33:15.154097Z","iopub.status.idle":"2022-10-23T13:33:15.163649Z","shell.execute_reply.started":"2022-10-23T13:33:15.154062Z","shell.execute_reply":"2022-10-23T13:33:15.162937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Parameters\nn_classes = 6\nn_epochs = 1\nbatch_size = 32","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.165052Z","iopub.execute_input":"2022-10-23T13:33:15.165584Z","iopub.status.idle":"2022-10-23T13:33:15.174586Z","shell.execute_reply.started":"2022-10-23T13:33:15.165549Z","shell.execute_reply":"2022-10-23T13:33:15.173745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"COLS = ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.175847Z","iopub.execute_input":"2022-10-23T13:33:15.176298Z","iopub.status.idle":"2022-10-23T13:33:15.186862Z","shell.execute_reply.started":"2022-10-23T13:33:15.176263Z","shell.execute_reply":"2022-10-23T13:33:15.186141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images_dir = os.path.join(INPUT_PATH, 'stage_2_train/')\ntest_images_dir = os.path.join(INPUT_PATH, 'stage_2_test/')","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.18888Z","iopub.execute_input":"2022-10-23T13:33:15.189336Z","iopub.status.idle":"2022-10-23T13:33:15.196044Z","shell.execute_reply.started":"2022-10-23T13:33:15.189303Z","shell.execute_reply":"2022-10-23T13:33:15.195313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_metadata_csv = '../input/rsna-stage-2-metadata-ihd-2019/stage_2_train_with_metadata.csv'\n# test_metadata_csv = '../input/rsna-stage-2-metadata-ihd-2019/stage_2_test_with_metadata.csv'\ntrain_metadata_csv = '../input/rsna-intracranial-sequence-metadata/train_metadata_noidx.csv'\ntest_metadata_csv = '../input/rsna-intracranial-sequence-metadata/test_metadata_noidx.csv'","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.197339Z","iopub.execute_input":"2022-10-23T13:33:15.197795Z","iopub.status.idle":"2022-10-23T13:33:15.204235Z","shell.execute_reply.started":"2022-10-23T13:33:15.19776Z","shell.execute_reply":"2022-10-23T13:33:15.20353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_csv(train_metadata_csv)\ntrain_metadata.columns","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:15.20547Z","iopub.execute_input":"2022-10-23T13:33:15.20597Z","iopub.status.idle":"2022-10-23T13:33:16.836353Z","shell.execute_reply.started":"2022-10-23T13:33:15.20593Z","shell.execute_reply":"2022-10-23T13:33:16.835482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(os.path.join(INPUT_PATH, 'stage_2_train.csv'))\ntest = pd.read_csv(os.path.join(INPUT_PATH, 'stage_2_sample_submission.csv'))","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:16.837874Z","iopub.execute_input":"2022-10-23T13:33:16.838154Z","iopub.status.idle":"2022-10-23T13:33:19.750204Z","shell.execute_reply.started":"2022-10-23T13:33:16.838118Z","shell.execute_reply":"2022-10-23T13:33:19.749453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocessing : Windowing","metadata":{}},{"cell_type":"code","source":"## A function to correct pixel data and rescale intercercepts ob 12 bit images\ndef dcm_correction(dcm_img):\n        x = dcm_img.pixel_array + 1000\n        px_mode = 4096\n        x[x >= px_mode] = x[x >= px_mode] - px_mode #if there are extra bits in 12-bit grayscale(<=4096)\n        dcm_img.PixelData = x.tobytes()\n        dcm_img.RescaleIntercept = -1000 #setting a common value across all 12-bit US images\n        \n#Systemic/linear windowing\ndef window_image(dcm, window_center, window_width):\n    if (dcm.BitsStored == 12) and (dcm.PixelRepresentation == 0) and (int(dcm.RescaleIntercept) > -100):\n        dcm_correction(dcm)\n\n    img = dcm.pixel_array * dcm.RescaleSlope + dcm.RescaleIntercept #reconstructing the image from pixels\n    img_min = window_center - window_width // 2 #lowest visible value\n    img_max = window_center + window_width // 2 #highest visible value\n    img = np.clip(img, img_min, img_max)\n\n    return img\n\n#Combining all\ndef bsb_window(dcm):\n    brain_img = window_image(dcm, 40, 80)\n    subdural_img = window_image(dcm, 80, 200)\n    soft_img = window_image(dcm, 40, 380)\n\n    brain_img = (brain_img - 0) / 80\n    subdural_img = (subdural_img - (-20)) / 200\n    soft_img = (soft_img - (-150)) / 380\n    bsb_img = np.array([brain_img, subdural_img, soft_img]).transpose(1, 2, 0)\n\n    return bsb_img","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:19.752671Z","iopub.execute_input":"2022-10-23T13:33:19.752956Z","iopub.status.idle":"2022-10-23T13:33:19.762214Z","shell.execute_reply.started":"2022-10-23T13:33:19.752911Z","shell.execute_reply":"2022-10-23T13:33:19.761284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class IntracranialDataset(Dataset):\n\n    def __init__(self, csv_file, path, labels, transform=None):\n        self.path = path\n        self.data = pd.read_csv(csv_file)\n        self.transform = transform\n        self.labels = labels\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n#         img_name = os.path.join(self.path, self.data.loc[idx, 'Image'] + '.png')\n#         img = cv2.imread(img_name)   \n        try:\n            dicom = pydicom.dcmread(self.path, self.data.loc[idx, 'Image'] + '.dcm')\n            img = bsb_window(dicom)\n        except:\n            img = np.zeros((512, 512, 3))\n        \n        if self.transform:       \n            augmented = self.transform(image=img)\n            img = augmented['image']   \n            \n        if self.labels:\n            \n            labels = torch.tensor(\n                self.data.loc[idx, ['epidural', 'intraparenchymal', 'intraventricular', 'subarachnoid', 'subdural', 'any']])\n            return {'image': img, 'labels': labels}    \n        \n        else:      \n            \n            return {'image': img}","metadata":{"execution":{"iopub.status.busy":"2022-10-23T13:33:19.763587Z","iopub.execute_input":"2022-10-23T13:33:19.763909Z","iopub.status.idle":"2022-10-23T13:33:19.77536Z","shell.execute_reply.started":"2022-10-23T13:33:19.763873Z","shell.execute_reply":"2022-10-23T13:33:19.774641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.value_counts()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-10-23T13:33:19.776477Z","iopub.execute_input":"2022-10-23T13:33:19.777316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_csv(train_metadata_csv)\ntest_metadata = pd.read_csv(test_metadata_csv)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata[['ID', 'Image', 'Diagnosis']] = train_metadata['ID'].str.split('_', expand=True)\ntrain_metadata['ImageID'] = 'ID_' + train_metadata['Image']\ntrain_metadata.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata.drop(['ID', 'Diagnosis'], axis=1)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata[:15]","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation","metadata":{}},{"cell_type":"code","source":"# Prepare train table\ntrain[['ID', 'Image', 'Diagnosis']] = train['ID'].str.split('_', expand=True)\ntrain = train[['Image', 'Diagnosis', 'Label']]\ntrain.drop_duplicates(inplace=True)\ntrain = train.pivot(index='Image', columns='Diagnosis', values='Label').reset_index()\ntrain['Image'] = 'ID_' + train['Image']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove invalid instances of images(dcm, PNG)\npng = glob.glob(os.path.join(train_images_dir, '*.dcm'))\npng = [os.path.basename(png)[:-4] for png in png]\npng = np.array(png)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train[train['Image'].isin(png)]\n# train.to_csv('train.csv', index=False)\ntrain.value_counts()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_train = pd.merge(left=train, right=train_metadata, how='left', left_on='Image', right_on='ImageId')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_train","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Train/valid split \ntrain_series = train_metadata['SeriesInstanceUID'].unique() #Identifying unique scans by SeriesID: there are totally \n\nvalid_series = train_series[21000:]\ntrain_series = train_series[:21000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_series))\nprint(len(valid_series))","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = merged_train[merged_train['SeriesInstanceUID'].isin(train_series)]\ntrain_df","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_df = merged_train[merged_train['SeriesInstanceUID'].isin(valid_series)]\nvalid_df","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_df))\nprint(len(valid_df))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So the training set has 728,513 slices and a validation set 24,290 slices","metadata":{}},{"cell_type":"code","source":"os.mkdir('./data')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('data/train.csv', index=False)\nprint(train_df['any'].value_counts())\nvalid_df.to_csv('data/valid.csv', index=False)\nprint(valid_df['any'].value_counts())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare test table\ntest[['ID', 'Image', 'Diagnosis']] = test['ID'].str.split('_', expand=True)\ntest['Image'] = 'ID_' + test['Image']\ntest = test[['Image', 'Label']]\ntest.drop_duplicates(inplace=True)\n\ntest.to_csv('data/test.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data loaders\ntransform_train = Compose([Resize(256, 256),\n                           Normalize(mean=[0.1738, 0.1433, 0.1970], std=[0.3161, 0.2850, 0.3111], max_pixel_value=1.),\n                           HorizontalFlip(),\n                           ShiftScaleRotate(),\n                           RandomBrightnessContrast(),\n                           ToTensorV2()])\n\ntransform_test = Compose([Resize(256, 256),\n                          Normalize(mean=[0.1738, 0.1433, 0.1970], std=[0.3161, 0.2850, 0.3111], max_pixel_value=1.),\n                          ToTensorV2()])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = IntracranialDataset(\n    csv_file='data/train.csv', path=train_images_dir, transform=transform_train, labels=True)\n# print(len(train_dataset))\n\nvalid_dataset = IntracranialDataset(\n    csv_file='data/valid.csv', path=train_images_dir, transform=transform_train, labels=True)\n# print(len(valid_dataset))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = IntracranialDataset(\n    csv_file='data/test.csv', path=test_images_dir, transform=transform_test, labels=False)\n# print(len(test_dataset))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)\nprint(len(data_loader_train))\ndata_loader_valid = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\nprint(len(data_loader_valid))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_loader_test = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\nprint(len(data_loader_test))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"from torch.hub import load_state_dict_from_url\nfrom torchvision.models.resnet import ResNet, Bottleneck","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_urls = {\n    'resnext101_32x8d': 'https://download.pytorch.org/models/ig_resnext101_32x8-c38310e5.pth',\n    'resnext101_32x16d': 'https://download.pytorch.org/models/ig_resnext101_32x16-c6f796b0.pth',\n    'resnext101_32x32d': 'https://download.pytorch.org/models/ig_resnext101_32x32-e4b90b00.pth',\n    'resnext101_32x48d': 'https://download.pytorch.org/models/ig_resnext101_32x48-3e41cc8a.pth',\n}","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _resnext(arch, block, layers, pretrained, progress, **kwargs):\n    model = ResNet(block, layers, **kwargs)\n    state_dict = load_state_dict_from_url(model_urls[arch], progress=progress)\n    model.load_state_dict(state_dict)\n    return model\n\ndef resnext101_32x8d_wsl(progress=True, **kwargs):\n    \"\"\"Constructs a ResNeXt-101 32x8 model pre-trained on weakly-supervised data\n    and finetuned on ImageNet from Figure 5 in\n    `\"Exploring the Limits of Weakly Supervised Pretraining\" <https://arxiv.org/abs/1805.00932>`_\n    Args:\n        progress (bool): If True, displays a progress bar of the download to stderr.\n    \"\"\"\n    kwargs['groups'] = 32\n    kwargs['width_per_group'] = 8\n    return _resnext('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], True, progress, **kwargs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = resnext101_32x8d_wsl()\nprint(model)","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(model.children())","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNeXtModel(torch.nn.Module):\n    def __init__(self):\n        super(ResNeXtModel, self).__init__()\n        resnext = resnext101_32x8d_wsl()\n        self.base = torch.nn.Sequential(*list(resnext.children())[:-1])\n        self.fc = torch.nn.Sequential(torch.nn.Linear(2048,6))\n    \n    def forward(self, input):\n        features = self.base(input).reshape(-1, 2048)\n        out = self.fc(features)\n        return out, features","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training & Validation","metadata":{}},{"cell_type":"code","source":"# Installing useful libraries\n\n!git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#For Mixed precision training\nfrom apex import amp","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ResNeXtModel()\nmodel.to(device)\n\ncriterion = torch.nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=1e-5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\")","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install GPUtil","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from GPUtil import showUtilization as gpu_usage\nfrom numba import cuda","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def free_gpu_cache():\n    print(\"Initial GPU Usage\")\n    gpu_usage()                             \n\n    torch.cuda.empty_cache()\n\n    cuda.select_device(0)\n    cuda.close()\n    cuda.select_device(0)\n\n    print(\"GPU Usage after emptying the cache\")\n    gpu_usage()\n\nfree_gpu_cache()   ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gpu_usage()  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.tensorboard import SummaryWriter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tb  = SummaryWriter('runs/ich_detection_experiment_1')\n# tb  = SummaryWriter()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(n_epochs):\n\n    print('Epoch {}/{}'.format(epoch + 1, n_epochs))\n    print('-' * 10)\n\n    model.train()\n    tr_loss = 0\n    tr_correct = 0\n    \n    tk0 = tqdm.tqdm(data_loader_train, desc=\"Iteration\")\n\n    for step, batch in enumerate(tk0):\n        inputs = batch[\"image\"]\n#         print(inputs.shape)\n        labels = batch[\"labels\"]\n#         print(labels.shape)\n\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n\n        outputs, _ = model(inputs)\n#         print(outputs.shape)\n#         print(outputs)\n        loss = criterion(outputs, labels)\n        preds = (torch.sigmoid(outputs) >=0.5).float()*1\n#         print(preds.shape)\n\n\n        with amp.scale_loss(loss, optimizer) as scaled_loss:\n            scaled_loss.backward()\n            \n        tr_loss += loss.item()\n        tr_correct += torch.sum(preds == labels)\n#         print(tr_correct)\n\n        optimizer.step()\n        optimizer.zero_grad()\n\n        if step % 512 == 0:\n            epoch_loss = tr_loss / (step + 1)\n            print('Training Loss at {}: {:.4f}'.format(step, epoch_loss))\n\n    epoch_loss = tr_loss / len(data_loader_train)\n    print('Training Loss: {:.4f}'.format(epoch_loss))\n    print('-----------------------')\n    #Tensorboard code for visualisations\n    tb.add_scalar(\"Training Loss\", tr_loss, epoch)\n    tb.add_scalar(\"Training Correct preds\", tr_correct, epoch)\n    tb.add_scalar(\"Training Accuracy\", tr_correct/ len(train_dataset), epoch)\n    print('Finished Training!')\n    \n    model.eval()\n    tr_loss = 0\n    tr_correct = 0\n\n    auc_preds = []\n    auc_truths = []\n    \n    print('Validation starts...')\n    for step, batch in enumerate(data_loader_valid):\n        inputs = batch[\"image\"]\n        labels = batch[\"labels\"]\n\n        inputs = inputs.to(device, dtype=torch.float)\n        labels = labels.to(device, dtype=torch.float)\n\n        outputs, _ = model(inputs)\n        loss = criterion(outputs, labels)\n        preds = (torch.sigmoid(outputs) >=0.5).float()*1\n\n        tr_loss += loss.item()\n        tr_correct += torch.sum(preds == labels)\n\n        auc_preds.append(preds.view(-1, 6).detach().cpu().numpy())\n        auc_truths.append(labels.view(-1, 6).detach().cpu().numpy())\n\n    epoch_loss = tr_loss / len(data_loader_valid)\n    print('Validation Loss: {:.4f}'.format(epoch_loss))\n    \n    #Tensorboard code for visualisations\n    tb.add_scalar(\"Validation Loss\", tr_loss, epoch)\n    tb.add_scalar(\"Training Correct preds\", tr_correct, epoch)\n    tb.add_scalar(\"Training Accuracy\", tr_correct/ len(train_dataset), epoch)\n    print('Finished Validation!')\n    \n    roc_preds = np.concatenate(auc_preds)\n\n    roc_truths = np.concatenate(auc_truths)\n\n    for tp in range(0, 6):\n        print('ROC_AUC')\n        print(COLS[tp], roc_auc_score(roc_truths[:, tp], roc_preds[:, tp]), )\n        print('F1 SCORE')\n        print(COLS[tp], f1_score(roc_truths[:, tp], roc_preds[:, tp]), )\n        print('ACCURACY')\n        print(COLS[tp], accuracy_score(roc_truths[:, tp], roc_preds[:, tp]), )\n        tn, fp, fn, tp = confusion_matrix(roc_truths[:, tp], roc_preds[:, tp]).ravel()\n        print('SENSITIVITY')\n        print(COLS[tp], tp/(tp+fn), )\n        print('SPECIFICITY')\n        print(COLS[tp], tn/(tn+fp), )    \n        print('JACCARD SCORE')\n        print(COLS[tp], jaccard_score(roc_truths[:, tp], roc_preds[:, tp]), )\n        \n    print('-----------------------')\n\n# Save checkpoint\ncheckpoint = {\n    'model': model.state_dict(),\n    'optimizer': optimizer.state_dict(),\n    # 'amp': amp.state_dict()\n}\ntorch.save(checkpoint, 'model.pt')\ntb.close()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Points to note -**\n- It is **NOT** multi-class but a **multi-label** classification problem(Refer to the paper)\n- Try using Sigmoid function to get 0or1 across each class, NOT Softmax\n- While finding accuracy, to fin correct predictions use threshold for outputs as 0.5\n\n**Observations and points to noye** -\n- On just 1 epoch, the results are **very poor**\n- Accuracy is high for obvious reasons. It **should not be used** to judge, since it is an **imbalaced problem**\n- F1 should be used when **we care about positive classes**. We should look at improving this\n- ROC-AUC should be used when you care **equally about positive and negative classes**. **Not advised on highly imbalanced** dataset(but let's check what we get and comapre with paper. Look into *average_precision_score* metric instead(when you care more about positive than negative class)\n- What did we get?\n    - Accuracy high - DO NOT go by this\n    - F1 of 0 - very bad, hope it improves when trained on more epochs\n    - ROC-AUC of 0.5 - which means not discriminating between the two claases(0/1); again hope to improve\n    \n**NOTE**\nError fixed. Should work tommorrow(::finger crossed::)\n- Tensorboard visualisation codes are right! But the site to display them doesn't seem to work through kaggle. It should work locally","metadata":{}},{"cell_type":"code","source":"tb.close()","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tensorboard --logdir=runs","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing","metadata":{}},{"cell_type":"code","source":"# Nothing yet","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}