{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is one of the first ***Vision Transformer*** implementations, a simple way to achieve SOTA in vision classification with only a single transformer encoder, in Pytorch, based on \"https://github.com/lucidrains/vit-pytorch\" with much lower parameters. It challenges the paradigm of Convolutions for vision. It instead represents images as a set of visual tokens and applies visual transformers to find relationships between visual semantic concepts. Given an input image,it dynamically extracts a set of visual tokens from the image to obtain a compact representation for high-level semantics. Then, It uses visual transformers to operate over the visual tokens to densely model relationships between them.\n\nWe replaced the attention layer with a more efficient network, called **\"*Linformer\"***. This is attention with only linear complexity in n, allowing for very long sequence lengths (1mil+) to be attended to on GPU. One can also use axial attention instead as was mention in their paper. The paper is under open review for ICLR 2021.\n\n\nThe data management and pipeline is also based on \"https://www.kaggle.com/orkatz2/pulmonary-embolism-pytorch-train\".\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"!pip install linformer\n!pip install axial_attention","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport glob\nimport cv2\nimport os\nfrom matplotlib import pyplot as plt\nimport os\nimport cv2\nimport numpy as np\nimport pandas as pd\nfrom torch.utils.data import TensorDataset, DataLoader,Dataset\nimport albumentations as albu\nfrom skimage.color import gray2rgb\nimport functools\nfrom sklearn.metrics import hamming_loss, accuracy_score\n\nimport torch\nfrom torch import nn\nfrom linformer import Linformer\nfrom axial_attention import AxialAttention\n\n\nfrom tqdm.auto import tqdm\ntorch.cuda.empty_cache()\nos.environ['CUDA_LAUNCH_BLOCKING'] = '1'\nos.environ[\"CUDA_VISIBLE_DEVICES\"]='0'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nclass data_config:\n    train_csv_path = 'train.csv'\n    jpeg_dir ='train-jpegs'\n    ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']    \n    label_lstm = ['pe_present_on_image','negative_exam_for_pe', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1',\n       'leftsided_pe', 'chronic_pe','rightsided_pe', 'acute_and_chronic_pe', 'central_pe', 'indeterminate']\n        \nclass visualTrans:  \n    model_name=\"visualT\"\n    batch_size = 1\n    WORKERS = 4\n    classes =9\n    epochs = 64\n    optimizer = \"torch.optim.AdamW\"\n    optimizer_parm = {'lr':1e-4,'weight_decay':0.0001}\n    scheduler = \"torch.optim.lr_scheduler.CosineAnnealingLR\"\n    scheduler_parm = {'T_max':5500,'eta_min':0.00001}\n    loss_fn = 'torch.nn.BCEWithLogitsLoss'\n    MODEL_PATH = 'log/cpt'\n    if not os.path.exists(MODEL_PATH):\n        os.makedirs(MODEL_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CFG = {\n    'image_target_cols': [\n        'pe_present_on_image', # only image level\n    ],\n    \n    'exam_target_cols': [\n        'negative_exam_for_pe', # exam level\n        'rv_lv_ratio_gte_1', # exam level\n        'rv_lv_ratio_lt_1', # exam level\n        'leftsided_pe', # exam level\n        'chronic_pe', # exam level\n        'rightsided_pe', # exam level\n        'acute_and_chronic_pe', # exam level\n        'central_pe', # exam level\n        'indeterminate' # exam level\n    ], \n    \n    'image_weight': 0.07361963,\n    'exam_weights': [0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988],\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_csv_path = 'train.csv'\njpeg_dir = 'train-jpegs'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\ndef get_fold(train,FOLD_NUM = 5):\n    train_image_num_per_patient = train.groupby('StudyInstanceUID')['SOPInstanceUID'].nunique()\n    target_cols = [c for i, c in enumerate(train.columns) if i > 2]\n    \n    train_per_patient_char = pd.DataFrame(index=train_image_num_per_patient.index, columns=['image_per_patient'], data=train_image_num_per_patient.values.copy())\n    for t in target_cols:\n        train_per_patient_char[t] = train_per_patient_char.index.map(train.groupby('StudyInstanceUID')[t].mean())\n        \n    \n    bin_counts = [40] #, 20]\n    digitize_cols = ['image_per_patient'] #, 'pe_present_on_image']\n    non_digitize_cols = [c for c in train_per_patient_char.columns if c not in digitize_cols]\n    for i, c in enumerate(digitize_cols):\n        bin_count = bin_counts[i]\n        percentiles = np.percentile(train_per_patient_char[c], q=np.arange(bin_count)/bin_count*100.)\n        train_per_patient_char[c+'_digitize'] = np.digitize(train_per_patient_char[c], percentiles, right=False)\n        \n    train_per_patient_char['key'] = train_per_patient_char[digitize_cols[0]+'_digitize'].apply(str)\n    for c in digitize_cols[1:]:\n        train_per_patient_char['key'] = train_per_patient_char['key']+'_'+train_per_patient_char[c+'_digitize'].apply(str)\n    folds = FOLD_NUM\n    kfolder = StratifiedKFold(n_splits=folds, shuffle=True, random_state=719)\n    val_indices = [val_indices for _, val_indices in kfolder.split(train_per_patient_char['key'], train_per_patient_char['key'])]\n    train_per_patient_char['fold'] = -1\n    for i, vi in enumerate(val_indices):\n        patients = train_per_patient_char.index[vi]\n        train_per_patient_char.loc[patients, 'fold'] = i\n    return train_per_patient_char\n\ndef split_train_val_lstm(data_config, fold, FOLD_NUM=5):\n    main_df = pd.read_csv(data_config.train_csv_path)\n    train_df = main_df[data_config.ids+ data_config.label_lstm]\n    train_per_patient_char = get_fold(main_df, FOLD_NUM)\n    TID = train_per_patient_char[train_per_patient_char.fold!=fold].index\n    VID = train_per_patient_char[train_per_patient_char.fold==fold].index\n    t_df = train_df[train_df['StudyInstanceUID'].isin(TID)]\n    v_df = train_df[train_df['StudyInstanceUID'].isin(VID)]\n    return t_df,v_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t_df,v_df = split_train_val_lstm(data_config,fold=0,FOLD_NUM=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path256 = f\"{data_config.jpeg_dir}/*/*/*.jpg\"\ndata = glob.glob(path256)\nnew_df = []\nfor row in tqdm(data):\n    StudyInstanceUID,SeriesInstanceUID,SOPInstanceUID = row.split(\"/\")[-3:]\n    num,SOPInstanceUID = SOPInstanceUID.replace(\".jpg\",\"\").split(\"_\")\n    new_df.append([StudyInstanceUID,SeriesInstanceUID,SOPInstanceUID,num])\ns_df = pd.DataFrame(new_df)\ns_df.columns = list(t_df.columns[:3])+[\"slice\"]\nt_df = t_df.merge(s_df,on=list(t_df.columns[:3]),how='left')\nv_df = v_df.merge(s_df,on=list(v_df.columns[:3]),how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t = t_df.groupby(list(t_df.columns[:2]))\nmini_dfs= []\nfor i,row in tqdm(t_df.groupby(list(t_df.columns[:2]))):\n    if len(row)>400:\n        continue\n    mini_dfs.append(row.sort_values(\"slice\"))\nmini_dfs_val = []\nfor i,row in tqdm(v_df.groupby(list(v_df.columns[:2]))):\n    if len(row)>400:\n        continue\n    mini_dfs_val.append(row.sort_values(\"slice\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CTDataset(Dataset):\n    def __init__(self,df,jpeg_dir,transforms = None, preprocessing=None, size=256, mode='val'):\n        self.df_main = df\n        self.jpeg_dir = jpeg_dir\n        self.transforms = transforms\n        self.preprocessing = preprocessing\n        self.size=size\n\n    def __getitem__(self, idx):\n        mini = self.df_main[idx].values\n        all_paths = [f\"{self.jpeg_dir}/{row[0]}/{row[1]}/{row[-1]}_{row[2]}.jpg\" for row in mini]\n        img = [self.transforms(image=cv2.imread(p))['image'] for p in all_paths]\n        label = mini[:,3:-1].astype(int)        \n            \n        if self.preprocessing:\n            img = [self.preprocessing(image=im)['image'] for im in img]\n        return np.array(img),torch.from_numpy(label[:,0]), torch.from_numpy(label[0,1:])\n    \n    def __len__(self):\n        return len(self.df_main)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_augmentation(y=256,x=256):\n    train_transform = [albu.RandomBrightnessContrast(p=0.3),\n                           albu.VerticalFlip(p=0.5),\n                           albu.HorizontalFlip(p=0.5),\n                           albu.Downscale(p=1.0,scale_min=0.35,scale_max=0.75,),\n                           albu.Resize(y, x)]\n    return albu.Compose(train_transform)\n\n\nformatted_settings = {\n            'input_size': [3, 224, 224],\n            'input_range': [0, 1],\n            'mean': [0.485, 0.456, 0.406],\n            'std': [0.229, 0.224, 0.225],}\n\ndef mono_tr(x):\n    train_transforms = Compose([ScaleIntensity(), \n                            Resize((image_size, image_size, image_size)), \n                            RandAffine( \n                                      prob=0.5,\n                                      translate_range=(5, 5, 5),\n                                      rotate_range=(np.pi * 4, np.pi * 4, np.pi * 4),\n                                      scale_range=(0.15, 0.15, 0.15),\n                                      padding_mode='border'),\n                            ToTensor()])\n    apply_transform(train_transforms, x)\n    \ndef mono_val(x):\n    val_transforms = Compose([ScaleIntensity(), ToTensor()])\n    \n    apply_transform(val_transforms, x)\n    \n\ndef preprocess_input(\n    x, mean=None, std=None, input_space=\"RGB\", input_range=None, **kwargs\n):\n\n    if input_space == \"BGR\":\n        x = x[..., ::-1].copy()\n\n    if input_range is not None:\n        if x.max() > 1 and input_range[1] == 1:\n            x = x / 255.0\n\n    if mean is not None:\n        mean = np.array(mean)\n        x = x - mean\n\n    if std is not None:\n        std = np.array(std)\n        x = x / std\n\n    return x\n\ndef get_preprocessing(preprocessing_fn):\n    _transform = [\n        albu.Lambda(image=preprocessing_fn),\n        #albu.Lambda(image=preprocessing_fn_2),\n        albu.Lambda(image=to_tensor, mask=to_tensor),\n    ]\n    return albu.Compose(_transform)\n\ndef get_validation_augmentation(y=256,x=256):\n    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n    test_transform = [albu.Resize(y, x)]\n    return albu.Compose(test_transform)\n\ndef to_tensor(x, **kwargs):\n    \"\"\"\n    Convert image or mask.\n    \"\"\"\n    return x.transpose(2, 0, 1).astype('float32')\n\ndef norm(img):\n    img-=img.min()\n    return img/img.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessing_fn = functools.partial(preprocess_input, **formatted_settings)\ntrain_dataset = CTDataset(mini_dfs,data_config.jpeg_dir,\n                            transforms=get_training_augmentation(), preprocessing=get_preprocessing(preprocessing_fn))\nval_dataset = CTDataset(mini_dfs_val,data_config.jpeg_dir,\n                            transforms=get_validation_augmentation(),preprocessing=get_preprocessing(preprocessing_fn))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x,y,y1 = train_dataset[0]\nx.shape,y.shape,y1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"global view_output\ndef hook_fn(module, input, output):\n    global view_output\n    view_output = output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom einops import rearrange\nfrom torch import nn\n\nclass ViT(nn.Module):\n    def __init__(self, *, image_size, patch_size, out_dim, dim, transformer, channels = 3):\n        super().__init__()\n        assert image_size % patch_size == 0, 'image dimensions must be divisible by the patch size'\n        num_patches = (image_size // patch_size) ** 2\n        patch_dim = channels * patch_size ** 2\n\n        self.patch_size = patch_size\n\n        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n        self.patch_to_embedding = nn.Linear(patch_dim, dim)\n        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n        self.transformer = transformer\n\n        self.to_cls_token = nn.Identity()\n\n        self.mlp_head = nn.Sequential(\n            nn.LayerNorm(dim),\n            nn.Linear(dim, dim * 4),\n            nn.GELU(),\n            nn.Linear(dim * 4, out_dim)\n        )\n\n    def forward(self, img):\n        p = self.patch_size\n\n        x = rearrange(img, 'b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = p, p2 = p)\n        x = self.patch_to_embedding(x)\n\n        cls_tokens = self.cls_token.expand(img.shape[0], -1, -1)\n        x = torch.cat((cls_tokens, x), dim=1)\n        x += self.pos_embedding\n        x = self.transformer(x)\n\n        x = self.to_cls_token(x[:, 0])\n        return self.mlp_head(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch import nn\nfrom torch.nn import functional as F\n\nclass TransNET(nn.Module):\n    def __init__(self, embed_size= 256, LSTM_UNITS= 64):\n        super(TransNET, self).__init__()\n        #self.axttn = AxialAttention(dim = 3, dim_index = 1, dim_heads = 16, heads = 1, num_dimensions = 2, sum_axial_out = True)\n        self.lin = Linformer(dim = 128, seq_len = 65, depth = 6, heads = 8, k = 256)\n        self.vit = ViT(image_size = 256, patch_size = 32, out_dim = 256, dim = 128, transformer = self.lin)\n        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n\n        self.linear1 = nn.Linear(LSTM_UNITS*2, LSTM_UNITS*2, bias = False)\n        self.linear2 = nn.Linear(LSTM_UNITS*2, LSTM_UNITS*2, bias = False)\n\n        self.linear_pe = nn.Linear(LSTM_UNITS*2, 1)\n        self.linear_global = nn.Linear(LSTM_UNITS*2, 9)\n\n    def forward(self, x, lengths=None):\n        #embedding = self.axttn(x)\n        embedding = self.vit(x)\n        b,f = embedding.shape\n        embedding = embedding.reshape(1,b,f)\n            \n        self.lstm1.flatten_parameters()\n        h_lstm1, _ = self.lstm1(embedding)\n        self.lstm2.flatten_parameters()\n        h_lstm2, _ = self.lstm2(h_lstm1)\n        \n        h_conc_linear1  = F.relu(self.linear1(h_lstm1))\n        h_conc_linear2  = F.relu(self.linear2(h_lstm2))\n        \n        hidden = h_lstm1 + h_lstm2 + h_conc_linear1 + h_conc_linear2\n\n        output = self.linear_pe(hidden)\n        output_global = self.linear_global(hidden.mean(1))\n        return output,output_global","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = TransNET().cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_config = visualT\noptimizer = eval(model_config.optimizer)(model.parameters(),**model_config.optimizer_parm)\nscheduler = eval(model_config.scheduler)(optimizer,**model_config.scheduler_parm)\n#loss_fn = torch.nn.BCEWithLogitsLoss(pos_weight=pos_weight)\nloss_fn = eval(model_config.loss_fn)(reduction='none')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_w = torch.cuda.FloatTensor(CFG['exam_weights']).view(1, -1)\nimg_w = CFG['image_weight']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport numpy as np\nfrom tqdm.auto import tqdm\nimport os\nclass trainer:\n    def __init__(self,loss_fn,model,optimizer,scheduler,config,label_w, img_w):\n        self.loss_fn = loss_fn\n        self.model = model\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.config = config\n        self.label_w = label_w\n        self.img_w = label_w\n\n        \n    def batch_train(self, batch_imgs, batch_labels0, batch_labels1, batch_idx):\n        batch_imgs, batch_labels0, batch_labels1 = batch_imgs.cuda().float(), batch_labels0.cuda().float(),batch_labels1.cuda().float()\n        predicted = self.model(batch_imgs)\n        loss0 = self.loss_fn(predicted[0].float().reshape(-1), batch_labels0.reshape(-1))\n        loss1 = self.loss_fn(predicted[1].float().reshape(-1), batch_labels1.reshape(-1))\n        loss1 = torch.sum(torch.mul(loss1, self.label_w), 1)[0]\n        img_num = batch_labels0.shape[1]\n        qi = torch.sum(batch_labels0.reshape(-1))/img_num\n        loss0 = torch.sum(img_w* qi* loss0)\n        loss = loss0 + loss1\n        total = label_w.sum() + img_w*qi*img_num\n        loss = loss/total\n        #print(loss)\n        loss.backward()\n        self.optimizer.step()\n        self.optimizer.zero_grad()\n        return loss.item(), predicted\n    \n    def batch_valid(self, batch_imgs,get_fet):\n        self.model.eval()\n        batch_imgs = batch_imgs.cuda()\n        with torch.no_grad():\n            predicted = self.model(batch_imgs)\n            predicted[0] = torch.sigmoid(predicted[0])\n            predicted[1] = torch.sigmoid(predicted[1])\n        return predicted\n    \n    def train_epoch(self, loader):\n        self.model.train()\n        tqdm_loader = tqdm(loader)\n        current_loss_mean = 0\n        for batch_idx, (imgs,labels,labels1) in enumerate(tqdm_loader):\n            loss, predicted = self.batch_train(imgs[0], labels,labels1, batch_idx)\n            current_loss_mean = (current_loss_mean * batch_idx + loss) / (batch_idx + 1)\n            tqdm_loader.set_description('loss: {:.4} lr:{:.6}'.format(\n                    current_loss_mean, self.optimizer.param_groups[0]['lr']))\n            self.scheduler.step(batch_idx)\n            if batch_idx>10:\n                break\n        return current_loss_mean\n    \n    def valid_epoch(self, loader,name=\"valid\"):\n        self.model.eval()\n        tqdm_loader = tqdm(loader)\n        current_loss_mean = 0\n        correct = 0\n        for batch_idx, (imgs,labels0,labels1) in enumerate(tqdm_loader):\n            with torch.no_grad():\n                batch_imgs = imgs.cuda().float()[0]\n                batch_labels0 = labels0.cuda().float()\n                batch_labels1 = labels1.cuda().float()\n                \n                predicted = self.model(batch_imgs)\n                loss0 = self.loss_fn(predicted[0].float().reshape(-1),batch_labels0.float().reshape(-1)) #.item()\n                loss1 = self.loss_fn(predicted[1].float().reshape(-1),batch_labels1.float().reshape(-1))#.item()\n                loss1 = torch.sum(torch.mul(loss1, self.label_w), 1)[0]\n                img_num = batch_labels0.shape[1]\n                qi = torch.sum(batch_labels0.reshape(-1))/img_num\n                loss0 = torch.sum(img_w* qi* loss0)\n                loss = loss0 + loss1\n                total = label_w.sum() + img_w*qi*img_num\n                loss = loss/total\n                hm_1 = hamming_loss(batch_labels1.reshape(-1).detach().cpu(), \n                                    torch.round(torch.sigmoid(predicted[1].reshape(-1).detach().cpu())))\n                hm_0 = hamming_loss(batch_labels0.reshape(-1).detach().cpu(), \n                                    torch.round(torch.sigmoid(predicted[0].reshape(-1).detach().cpu())))\n                print(\"Hamming_loss_0:\", hm_0)\n                print(\"Hamming_loss_1:\", hm_1)\n                tqdm_loader.set_description(f\"loss : {loss:.4}\")\n            if batch_idx>10:\n                break\n        score = 1- loss\n        print('metric {}'.format(score))\n        return score\n    \n    def run(self,train_loder,val_loder):\n        best_score = -100000\n        for e in range(self.config.epochs):\n            print(\"----------Epoch {}-----------\".format(e))\n            current_loss_mean = self.train_epoch(train_loder)\n            score = self.valid_epoch(val_loder)\n            if best_score < score:\n                best_score = score\n                torch.save(self.model.state_dict(),self.config.MODEL_PATH+\"/{}_best.pth\".format(self.config.model_name))\n\n    def batch_valid_tta(self, batch_imgs):\n        batch_imgs = batch_imgs.cuda()\n        predicted = model(batch_imgs)\n        tta_flip = [[-1],[-2]]\n        for axis in tta_flip:\n            predicted += torch.flip(model(torch.flip(batch_imgs, axis)), axis)\n        predicted = predicted/(1+len(tta_flip))\n        predicted = torch.sigmoid(predicted)\n        return predicted.cpu().numpy()\n            \n    def load_best_model(self):\n        if os.path.exists(self.config.MODEL_PATH+\"/{}_best.pth\".format(self.config.model_name)):\n            self.model.load_state_dict(torch.load(self.config.MODEL_PATH+\"/{}_best.pth\".format(self.config.model_name)))\n            print(\"load best model\")\n        \n    def predict(self,imgs_tensor,get_fet = False):\n        self.model.eval()\n        with torch.no_grad():\n            return self.batch_valid(imgs_tensor,get_fet=get_fet)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Trainer = trainer(loss_fn, model, optimizer, scheduler, config=model_config, label_w=label_w, img_w=img_w)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = DataLoader(train_dataset, batch_size= 1, shuffle=True, num_workers= model_config.WORKERS, pin_memory = False)\nval = DataLoader(val_dataset, batch_size= 1 , shuffle= True, num_workers= model_config.WORKERS, pin_memory = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Trainer.run(train,val)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}