{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Attempt at running Effnet on this massive dataset\n\nuses the Ian Pan Jpeg datset:\n\nhttps://www.kaggle.com/vaillant/rsna-str-pe-detection-jpeg-256\n\nand some tricks from this Pytorch notebook:\n\nhttps://www.kaggle.com/orkatz2/pulmonary-embolism-pytorch-train\n\nas well as some standard Pytorch from the tutorials page:\n\nhttps://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html#load-data\n\n"},{"metadata":{},"cell_type":"markdown","source":"## Results so far\n\nthe CPU is proving to be a bottleneck - during training it hits 150% whilst GPU is only about 25%.  Probably this is in the dataloader.  \n\nChanging pin_memory=True and increasing num_workers to 16 improved time per epoch from 2hr15 to about 1hr30 but this is still too long...\n\nWill try and improve the GPU utilisation a bit more but might have to move to TPUs."},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\nno_epochs = 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -U git+git://github.com/lilohuang/PyTurboJPEG.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from turbojpeg import TurboJPEG, TJPF_GRAY, TJSAMP_GRAY\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!conda install -c conda-forge gdcm -y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install torch_optimizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport cv2\nimport os\nfrom matplotlib import pyplot as plt\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom skimage.color import gray2rgb\nimport functools\nimport torch\nfrom torch import Tensor\nfrom torchvision import transforms\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.parameter import Parameter\nfrom torch.optim import lr_scheduler\nimport torch_optimizer as optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torch.utils.data.sampler import SequentialSampler\nfrom tqdm.auto import tqdm\nfrom matplotlib import animation, rc\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\nimport pydicom as dcm\nimport gdcm\nimport time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.backends.cudnn.benchmark = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/rsna-str-pulmonary-embolism-detection/train.csv'\ntest_path = pd.read_csv(\"../input/rsna-str-pulmonary-embolism-detection/test.csv\")\njpeg_path = '../input/rsna-str-pe-detection-jpeg-256/train-jpegs'\nfiles = glob.glob('../input/rsna-str-pulmonary-embolism-detection/train/*/*/*.dcm')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=pd.read_csv(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_main=train_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_main.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_main.SOPInstanceUID.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_main.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"list_of_jpgs= [i for i in range(1,10)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns_only_for_info = [\"qa_motion\",\"qa_contrast\",\"flow_artifact\",\"true_filling_defect_not_pe\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Can take the onlyforinfo cols out as they are not needed."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.iloc[list_of_jpgs,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_columns = ['StudyInstanceUID','SeriesInstanceUID','SOPInstanceUID','pe_present_on_image', 'negative_exam_for_pe', 'rv_lv_ratio_gte_1', \n                  'rv_lv_ratio_lt_1','leftsided_pe', 'chronic_pe','rightsided_pe', \n                  'acute_and_chronic_pe', 'central_pe', 'indeterminate']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[target_columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class jpg_dataset(Dataset):\n    def __init__(self, root_dir, df,  transforms = None):\n        \n        super().__init__()\n        self.df = df[target_columns]\n        self.root_dir = root_dir\n        self.transforms = transforms\n\n    def __len__(self):\n        return self.df.shape[0]\n\n    def __getitem__(self, ndx):\n        row = self.df.iloc[ndx,:]\n        in_file = open(glob.glob(f\"{root_dir}/{row[0]}/{row[1]}/*{row[2]}.jpg\")[0], 'rb')\n        img = jpeg.decode(in_file.read())\n        in_file.close()\n        label = row[3:].astype(int)\n        label[2:] = label[2:] if label[0]==1 else 0\n        #img /= 255.0\n        \n        #Retriving class label\n\n        \n        #Applying transforms on image\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n\n        return (img,label.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"root_dir=jpeg_path\njpeg = TurboJPEG()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row = train_df.iloc[0,:]\nrow\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trial of dataset class"},{"metadata":{"trusted":true},"cell_type":"code","source":"jpg_data = jpg_dataset(root_dir=jpeg_path,df=train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloader = DataLoader(jpg_data, batch_size=4,\n                        shuffle=True, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i_batch, sample_batched in enumerate(dataloader):\n    image=sample_batched[0]\n    print(i_batch, image.size())\n\n    # observe 4th batch and stop.\n    if i_batch == 3:\n        plt.figure()\n        plt.imshow(image[0])\n        plt.axis('off')\n        plt.ioff()\n        plt.show()\n        break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Real Dataloaders"},{"metadata":{"trusted":true},"cell_type":"code","source":"StudyInstanceUID = list(set(train_df['StudyInstanceUID']))\nprint(len(StudyInstanceUID))\nt_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[0:6500])]\nv_df = train_df[train_df['StudyInstanceUID'].isin(StudyInstanceUID[6500:])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data augmentation and normalization for training\n# Just normalization for validation\ndata_transforms = {\n    'train': A.Compose(\n    [\n        #A.SmallestMaxSize(max_size=160),\n        #A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=15, p=0.5),\n        A.RandomCrop(height=128, width=128),\n        #A.RGBShift(r_shift_limit=15, g_shift_limit=15, b_shift_limit=15, p=0.5),\n        A.RandomBrightnessContrast(p=0.5),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ]\n),\n    'val': A.Compose(\n    [\n        A.SmallestMaxSize(max_size=160),\n        A.CenterCrop(height=128, width=128),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n        ToTensorV2(),\n    ]\n),\n}\n\nprint(\"Initializing Datasets and Dataloaders...\")\n\n# Create training and validation datasets\nimage_datasets = {}\nimage_datasets['train'] = jpg_dataset(root_dir=jpeg_path,df=t_df, transforms = data_transforms['train'])\nimage_datasets['val'] = jpg_dataset(root_dir=jpeg_path,df=v_df, transforms = data_transforms['val'])\n\n# Create training and validation dataloaders\ndataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=16, pin_memory=True) for x in ['train', 'val']}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_sizes['train']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_datasets['train'][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install efficientnet_pytorch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class EfficientNetEncoderHead(nn.Module):\n    def __init__(self, depth, num_classes):\n        super(EfficientNetEncoderHead, self).__init__()\n        self.depth = depth\n        self.base = EfficientNet.from_pretrained(f'efficientnet-b{self.depth}')\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.output_filter = self.base._fc.in_features\n        self.classifier = nn.Linear(self.output_filter, num_classes)\n    def forward(self, x):\n        x = self.base.extract_features(x)\n        x = self.avg_pool(x).squeeze(-1).squeeze(-1)\n        x = self.classifier(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = EfficientNetEncoderHead(depth=0, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = model.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(model, criterion, optimizer, scheduler, no_epochs=25):\n    since = time.time()\n\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_acc = 0.0\n\n    for epoch in range(no_epochs):\n        print('Epoch {}/{}'.format(epoch, no_epochs - 1))\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            current_loss_mean = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            tqdm_loader = tqdm(dataloaders_dict[phase])\n\n            for batch_idx, (inputs,labels) in enumerate(tqdm_loader):\n                #inputs = inputs.to(device)\n                #labels = labels.to(device)\n                \n                inputs, labels = inputs.cuda().float(), labels.cuda().float() \n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward\n                # track history only if in train\n                with torch.set_grad_enabled(phase == 'train'):\n\n                    outputs = model(inputs)\n                    loss = criterion(outputs.float(), labels)\n\n                    # backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # statistics\n                running_loss += loss.item() * inputs.size(0)\n                current_loss_mean = (current_loss_mean * batch_idx + loss) / (batch_idx + 1)\n                tqdm_loader.set_description('loss: {:.4}'.format(\n                    current_loss_mean))\n                running_corrects += torch.sum(outputs == labels.data)\n            if phase == 'train':\n                scheduler.step()\n\n            epoch_loss = running_loss / dataset_sizes[phase]\n            score = 1-current_loss_mean\n            print('metric {}'.format(score))\n            \n            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n\n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n                phase, epoch_loss, epoch_acc))\n\n            # deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_wts = copy.deepcopy(model.state_dict())\n\n        print()\n\n    time_elapsed = time.time() - since\n    print('Training complete in {:.0f}m {:.0f}s'.format(\n        time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    # load best model weights\n    model.load_state_dict(best_model_wts)\n    torch.save(model.state_dict(),f'model{k}.bin')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def radam(parameters, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n    if isinstance(betas, str):\n        betas = eval(betas)\n    return optim.RAdam(parameters,\n                      lr=lr,\n                      betas=betas,\n                      eps=eps,\n                      weight_decay=weight_decay)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ncriterion = torch.nn.BCEWithLogitsLoss()\n\n# Observe that all parameters are being optimized\n\noptimizer = radam(model.parameters(), lr=1e-3, betas=(0.9,0.999), eps=1e-3, weight_decay=1e-4)\nscheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(dataloaders_dict['train'])*no_epochs, eta_min=1e-6) # dataloader should be train one - change later\n\n# Decay LR by a factor of 0.1 every 7 epochs\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import copy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_model(model, criterion, optimizer, scheduler, no_epochs=no_epochs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}