{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Initialization","metadata":{}},{"cell_type":"code","source":"!pip install -qU python-gdcm pydicom pylibjpeg","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-11T13:45:36.948443Z","iopub.execute_input":"2023-02-11T13:45:36.948975Z","iopub.status.idle":"2023-02-11T13:45:48.190463Z","shell.execute_reply.started":"2023-02-11T13:45:36.948926Z","shell.execute_reply":"2023-02-11T13:45:48.189138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport cv2\nimport glob\nimport gdcm\nimport pydicom\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom tqdm.notebook import tqdm\nfrom joblib import Parallel, delayed\n\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objs as go\nimport copy\nimport torch\nfrom PIL import Image\nfrom PIL import Image, ImageDraw\nfrom torch.utils.data import Dataset\nimport torchvision.transforms as transforms\nfrom torch.utils.data import random_split\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\nimport torch.nn as nn\nfrom torchvision import utils\n%matplotlib inline\nprint('import fin')","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:45:51.278087Z","iopub.execute_input":"2023-02-11T13:45:51.278512Z","iopub.status.idle":"2023-02-11T13:45:51.298388Z","shell.execute_reply.started":"2023-02-11T13:45:51.278475Z","shell.execute_reply":"2023-02-11T13:45:51.297046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examples","metadata":{}},{"cell_type":"markdown","source":"## Save the processed data\n**Images are quite big so resizing them is necessary.**\n  - Use 256 to train your first models, or if you don't have a lot of compute\n  - use 512 to have competitive models\n  - Check if 768/1024 is better, if you have the compute power\n\n**I advise using the `png` format because the jpg compression can be annoying during inference.**","metadata":{}},{"cell_type":"code","source":"SAVE_FOLDER = \"output/\"\nSIZE = 512\nEXTENSION = \"png\"\n\nos.makedirs(SAVE_FOLDER, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:45:57.365846Z","iopub.execute_input":"2023-02-11T13:45:57.366201Z","iopub.status.idle":"2023-02-11T13:45:57.371997Z","shell.execute_reply.started":"2023-02-11T13:45:57.366172Z","shell.execute_reply":"2023-02-11T13:45:57.370799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process(f, size=512, save_folder=\"\", extension=\"png\"):\n    patient = f.split('/')[-2]\n    image = f.split('/')[-1][:-4]\n\n    dicom = pydicom.dcmread(f)\n    img = dicom.pixel_array\n\n    img = (img - img.min()) / (img.max() - img.min())\n\n    if dicom.PhotometricInterpretation == \"MONOCHROME1\":\n        img = 1 - img\n    \n    img = cv2.resize(img, (size, size))\n    return Image.fromarray(img)\n    #cv2.imwrite(save_folder + where + '/' + str(cancer) + '/' + f\"{patient}_{image}.{extension}\", (img * 255).astype(np.uint8))","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:45:59.549082Z","iopub.execute_input":"2023-02-11T13:45:59.549445Z","iopub.status.idle":"2023-02-11T13:45:59.556539Z","shell.execute_reply.started":"2023-02-11T13:45:59.549413Z","shell.execute_reply":"2023-02-11T13:45:59.555521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport csv\n\ncsvPaths = {'test':'/kaggle/input/d/seungha164/new-csv/new_test.csv',\n           'train':'/kaggle/input/d/seungha164/new-csv/new_train.csv',\n           'val': '/kaggle/input/d/seungha164/new-csv/new_validation.csv'}\n\ntestDic, trainDic, valDic = {}, {}, {}\n\nfor mode in csvPaths:\n    imgs, labels = [],[]\n    df = pd.read_csv(csvPaths[mode])\n    for i, row in df.iterrows():\n        imgs.append(f\"{row['patient_id']}/{row['image_id']}.dcm\")\n        labels.append(int(row['cancer']))\n    if mode=='test':\n        testDic = {'imgs': imgs,'labels': labels}\n    elif mode == 'train':\n        trainDic = {'imgs': imgs,'labels': labels}\n    else:\n        valDic = {'imgs': imgs,'labels': labels}\n    print(mode+' done! - '+str(len(df)))","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:02.197865Z","iopub.execute_input":"2023-02-11T13:46:02.198601Z","iopub.status.idle":"2023-02-11T13:46:05.642636Z","shell.execute_reply.started":"2023-02-11T13:46:02.198564Z","shell.execute_reply":"2023-02-11T13:46:05.641694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main loop","metadata":{}},{"cell_type":"code","source":"class pytorch_data(Dataset):\n    \n    def __init__(self,dataset,transform,rootDir):\n        self.root = rootDir\n        self.imgPaths = dataset['imgs']\n        self.labels = dataset['labels']\n        self.transform = transform\n      \n    def __len__(self):\n        return len(self.imgPaths) # size of dataset\n      \n    def __getitem__(self, idx):\n        # open image, apply transforms and return with label\n        image = process(self.root + self.imgPaths[idx])  # Open Image with PIL\n        image = self.transform(image) # Apply Specific Transformation to Image\n        return image, self.labels[idx]\n","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:08.831768Z","iopub.execute_input":"2023-02-11T13:46:08.832927Z","iopub.status.idle":"2023-02-11T13:46:08.840785Z","shell.execute_reply.started":"2023-02-11T13:46:08.832881Z","shell.execute_reply":"2023-02-11T13:46:08.839754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **사이즈 고치기**","metadata":{}},{"cell_type":"markdown","source":"## Data Split","metadata":{}},{"cell_type":"code","source":"# define transformation that converts a PIL image into PyTorch tensors\nimport torchvision.transforms as transforms\n\n# Define the following transformations for the training dataset\ntr_transf = transforms.Compose([\n    transforms.Resize((512,512)),\n#     transforms.Resize((40,40)),\n    transforms.RandomHorizontalFlip(p=0.5), \n    transforms.RandomVerticalFlip(p=0.5),  \n    transforms.RandomRotation(45),         \n#     transforms.RandomResizedCrop(50,scale=(0.8,1.0),ratio=(1.0,1.0)),\n    transforms.ToTensor()])\n\n# For the validation dataset, we don't need any augmentation; simply convert images into tensors\nval_transf = transforms.Compose([\n    transforms.Resize((512,512)),\n    transforms.ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:11.431661Z","iopub.execute_input":"2023-02-11T13:46:11.432119Z","iopub.status.idle":"2023-02-11T13:46:11.440317Z","shell.execute_reply.started":"2023-02-11T13:46:11.432085Z","shell.execute_reply":"2023-02-11T13:46:11.439168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define an object of the custom dataset for the train folder.\n# define dataset - train.val.test\ntrainRoot = '/kaggle/input/rsna-breast-cancer-detection/train_images/'\ntestRoot = '/kaggle/input/rsna-breast-cancer-detection/test_images/'\ntrainDataset = pytorch_data(trainDic, tr_transf, trainRoot)\nvalidDataet = pytorch_data(valDic, val_transf, trainRoot)\ntestDataset = pytorch_data(testDic, val_transf, testRoot)\n\nprint(\"train dataset size:\", len(trainDataset))\nprint(\"validation dataset size:\", len(validDataet))\nprint(\"test dataset size:\", len(testDataset))","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:14.246233Z","iopub.execute_input":"2023-02-11T13:46:14.246608Z","iopub.status.idle":"2023-02-11T13:46:14.25485Z","shell.execute_reply.started":"2023-02-11T13:46:14.246577Z","shell.execute_reply":"2023-02-11T13:46:14.253693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The subset can also have transform attribute (if we asign)\ntrainDataset.transform","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:16.799227Z","iopub.execute_input":"2023-02-11T13:46:16.799585Z","iopub.status.idle":"2023-02-11T13:46:16.806766Z","shell.execute_reply.started":"2023-02-11T13:46:16.799554Z","shell.execute_reply":"2023-02-11T13:46:16.805619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\ndef plot_img(x,y,title=None):\n\n    npimg = x.numpy() # convert tensor to numpy array\n    npimg_tr=np.transpose(npimg, (1,2,0)) # Convert to H*W*C shape\n    fig = px.imshow(npimg_tr)\n    fig.update_layout(template='plotly_white')\n    fig.update_layout(title=title,height=300,margin={'l':10,'r':20,'b':10})\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:18.963279Z","iopub.execute_input":"2023-02-11T13:46:18.963689Z","iopub.status.idle":"2023-02-11T13:46:18.970491Z","shell.execute_reply.started":"2023-02-11T13:46:18.963657Z","shell.execute_reply":"2023-02-11T13:46:18.969389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataloaders","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import DataLoader\n\n# Training DataLoader\ntrain_dl = DataLoader(trainDataset, batch_size=32, shuffle=True)\n\n# Validation DataLoader\nval_dl = DataLoader(validDataet,batch_size=32, shuffle=False)\n# dataloader 딕셔너리 정의\ndataLoaders = {'train' : train_dl, 'val':val_dl}\nprint(dataLoaders)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:22.274795Z","iopub.execute_input":"2023-02-11T13:46:22.275861Z","iopub.status.idle":"2023-02-11T13:46:22.287013Z","shell.execute_reply.started":"2023-02-11T13:46:22.275818Z","shell.execute_reply":"2023-02-11T13:46:22.285744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define Binary Classifier","metadata":{}},{"cell_type":"code","source":"def findConv2dOutShape(hin,win,conv,pool=2):\n    # get conv arguments\n    kernel_size=conv.kernel_size\n    stride=conv.stride\n    padding=conv.padding\n    dilation=conv.dilation\n\n    hout=np.floor((hin+2*padding[0]-dilation[0]*(kernel_size[0]-1)-1)/stride[0]+1)\n    wout=np.floor((win+2*padding[1]-dilation[1]*(kernel_size[1]-1)-1)/stride[1]+1)\n\n    if pool:\n        hout/=pool\n        wout/=pool\n    return int(hout),int(wout)\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n# Neural Network\nclass Network(nn.Module):\n    \n    # Network Initialisation\n    def __init__(self, params):\n        \n        super(Network, self).__init__()\n    \n        Cin,Hin,Win=params[\"shape_in\"]\n        init_f=params[\"initial_filters\"] \n        num_fc1=params[\"num_fc1\"]  \n        num_classes=params[\"num_classes\"] \n        self.dropout_rate=params[\"dropout_rate\"] \n        \n        # Convolution Layers\n        self.conv1 = nn.Conv2d(Cin, init_f, kernel_size=3)\n        h,w=findConv2dOutShape(Hin,Win,self.conv1)\n        self.conv2 = nn.Conv2d(init_f, 2*init_f, kernel_size=3)\n        h,w=findConv2dOutShape(h,w,self.conv2)\n        self.conv3 = nn.Conv2d(2*init_f, 4*init_f, kernel_size=3)\n        h,w=findConv2dOutShape(h,w,self.conv3)\n        self.conv4 = nn.Conv2d(4*init_f, 8*init_f, kernel_size=3)\n        h,w=findConv2dOutShape(h,w,self.conv4)\n        \n        # compute the flatten size\n        self.num_flatten=h*w*8*init_f\n        self.fc1 = nn.Linear(self.num_flatten, num_fc1)\n        self.fc2 = nn.Linear(num_fc1, num_classes)\n\n    def forward(self,X):\n        \n        # Convolution & Pool Layers\n        X = F.relu(self.conv1(X)); \n        X = F.max_pool2d(X, 2, 2)\n        X = F.relu(self.conv2(X))\n        X = F.max_pool2d(X, 2, 2)\n        X = F.relu(self.conv3(X))\n        X = F.max_pool2d(X, 2, 2)\n        X = F.relu(self.conv4(X))\n        X = F.max_pool2d(X, 2, 2)\n\n        X = X.view(-1, self.num_flatten)\n        \n        X = F.relu(self.fc1(X))\n        X=F.dropout(X, self.dropout_rate)\n        X = self.fc2(X)\n        return X\n        #return F.log_softmax(X, dim=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:46:24.725769Z","iopub.execute_input":"2023-02-11T13:46:24.726688Z","iopub.status.idle":"2023-02-11T13:46:24.744123Z","shell.execute_reply.started":"2023-02-11T13:46:24.726654Z","shell.execute_reply":"2023-02-11T13:46:24.742813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nos.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"\n# Neural Network Predefined Parameters\nparams_model={\n        \"shape_in\": (1,512,512), \n        \"initial_filters\": 8,    \n        \"num_fc1\": 100,\n        \"dropout_rate\": 0.25,\n        \"num_classes\": 1}\n\n# Create instantiation of Network class\ncnn_model = Network(params_model)\n\n# define computation hardware approach (GPU/CPU)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)\nmodel = cnn_model.to(torch.device('cpu'))\n#model = cnn_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:52:01.782241Z","iopub.execute_input":"2023-02-11T13:52:01.78265Z","iopub.status.idle":"2023-02-11T13:52:01.840383Z","shell.execute_reply.started":"2023-02-11T13:52:01.782618Z","shell.execute_reply":"2023-02-11T13:52:01.839396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:54:40.363653Z","iopub.execute_input":"2023-02-11T13:54:40.364367Z","iopub.status.idle":"2023-02-11T13:54:50.142169Z","shell.execute_reply.started":"2023-02-11T13:54:40.36433Z","shell.execute_reply":"2023-02-11T13:54:50.140804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torchsummary import summary\nsummary(cnn_model, input_size=(1, 512, 512),device=device.type)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:54:50.14421Z","iopub.execute_input":"2023-02-11T13:54:50.144537Z","iopub.status.idle":"2023-02-11T13:54:50.19494Z","shell.execute_reply.started":"2023-02-11T13:54:50.144505Z","shell.execute_reply":"2023-02-11T13:54:50.19354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss function\ncriterion = nn.CrossEntropyLoss()\n#loss_func = nn.NLLLoss(reduction=\"sum\")","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:52:20.836921Z","iopub.execute_input":"2023-02-11T13:52:20.837328Z","iopub.status.idle":"2023-02-11T13:52:20.84291Z","shell.execute_reply.started":"2023-02-11T13:52:20.837289Z","shell.execute_reply":"2023-02-11T13:52:20.841901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import optim\nopt = optim.Adam(cnn_model.parameters(), lr=3e-4)\nlr_scheduler = ReduceLROnPlateau(opt, mode='min',factor=0.5, patience=20,verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:47:15.917301Z","iopub.execute_input":"2023-02-11T13:47:15.917683Z","iopub.status.idle":"2023-02-11T13:47:15.923659Z","shell.execute_reply.started":"2023-02-11T13:47:15.917651Z","shell.execute_reply":"2023-02-11T13:47:15.922648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Model","metadata":{}},{"cell_type":"code","source":"import time\ndef train_model(\n    model, dataloaders, optimizer, criterion, num_epochs, is_inception=False):\n    \n    start = time.time()\n    val_acc_history = []\n    \n    best_model_weights = copy.deepcopy(model.state_dict())\n    best_acc = 0\n    \n    for epoch in range(num_epochs):\n        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n        print('-'*10)\n        # Training & Val\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()   # Do this if model is in training phase\n            else:\n                model.eval()    # Do this if model is in validation phase\n            print(phase)\n            '''\n            running_loss, running_corrects = 0, 0\n            \n            # Iteration over the data\n            for inputs, labels in dataloaders[phase]:\n                inputs, labels = inputs.to(device), labels.to(device)\n                print(inputs.shape, labels)\n                # Parameter gradients are initialized to 0\n                optimizer.zero_grad()\n                \n                # Forward Pass - model output & loss 계산\n                with torch.set_grad_enabled(phase == 'train'):\n                    if is_inception and phase == 'train':      # Special case of inception because InceptionV3 has auxillary outputs as well. \n                        outputs = model(inputs)\n                        \n                        loss = criterion(outputs, labels)\n                        print(loss)\n                    else:\n                        outputs = model(inputs)\n                        loss = criterion(outputs, labels)\n                        \n                    _, preds = torch.max(outputs, 1)\n                    \n                    # Backward pass and Optimization in training phase \n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n                \n                # Stats\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds==labels.data)\n                \n            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n            \n            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n            \n            # Deep copy the model\n            if phase == 'val' and epoch_acc > best_acc:\n                best_acc = epoch_acc\n                best_model_weights = copy.deepcopy(model.state_dict())\n            if phase == 'val':\n                val_acc_history.append(epoch_acc)\n                \n    time_elapsed = time.time()\n    \n    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n    print('Best val Acc: {:4f}'.format(best_acc))\n    \n    # Best model weights are loaded here\n    model.load_state_dict(best_model_weights)\n    return model, val_acc_history\n    '''   ","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:49:35.498634Z","iopub.execute_input":"2023-02-11T13:49:35.499048Z","iopub.status.idle":"2023-02-11T13:49:35.511556Z","shell.execute_reply.started":"2023-02-11T13:49:35.499014Z","shell.execute_reply":"2023-02-11T13:49:35.510325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# dataloader 구조 - {'train' : loader() , 'val': loader()}\ntrain_model(model, dataLoaders, opt, criterion, 50, False)\n#train_model(model, dataLoaders, opt, criterion, num_epochs=50, is_inception=False)\n#bese_model, val_acc_history = train_model(model, dataLoaders, opt, criterion, num_epochs=50, is_inception=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:50:48.97607Z","iopub.execute_input":"2023-02-11T13:50:48.976442Z","iopub.status.idle":"2023-02-11T13:50:49.022999Z","shell.execute_reply.started":"2023-02-11T13:50:48.97641Z","shell.execute_reply":"2023-02-11T13:50:49.021431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Process","metadata":{}},{"cell_type":"code","source":"params_train={\n \"train\": train_dl,\"val\": val_dl,\n \"epochs\": 50,\n \"optimiser\": optim.Adam(cnn_model.parameters(),lr=3e-4),\n \"lr_change\": ReduceLROnPlateau(opt,\n                                mode='min',\n                                factor=0.5,\n                                patience=20,\n                                verbose=0),\n \"f_loss\": nn.NLLLoss(reduction=\"sum\"),\n \"weight_path\": \"weights.pt\",\n}\n\n''' Actual Train / Evaluation of CNN Model '''\n# train and validate the model\n\ncnn_model,loss_hist,metric_hist=train_val(cnn_model,params_train)","metadata":{"execution":{"iopub.status.busy":"2023-02-11T13:48:51.776017Z","iopub.execute_input":"2023-02-11T13:48:51.77639Z","iopub.status.idle":"2023-02-11T13:48:51.799735Z","shell.execute_reply.started":"2023-02-11T13:48:51.776357Z","shell.execute_reply":"2023-02-11T13:48:51.798032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns; sns.set(style='whitegrid')\n\nepochs=params_train[\"epochs\"]\n\nfig,ax = plt.subplots(1,2,figsize=(12,5))\n\nsns.lineplot(x=[*range(1,epochs+1)],y=loss_hist[\"train\"],ax=ax[0],label='loss_hist[\"train\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=loss_hist[\"val\"],ax=ax[0],label='loss_hist[\"val\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=metric_hist[\"train\"],ax=ax[1],label='metric_hist[\"train\"]')\nsns.lineplot(x=[*range(1,epochs+1)],y=metric_hist[\"val\"],ax=ax[1],label='metric_hist[\"val\"]')\nplt.title('Convergence History')","metadata":{"execution":{"iopub.status.busy":"2023-02-07T11:01:45.90773Z","iopub.status.idle":"2023-02-07T11:01:45.910199Z","shell.execute_reply.started":"2023-02-07T11:01:45.909929Z","shell.execute_reply":"2023-02-07T11:01:45.909955Z"},"trusted":true},"execution_count":null,"outputs":[]}]}