{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Multi-output","metadata":{}},{"cell_type":"code","source":"!yes | sudo dpkg -i /kaggle/input/libvips-pyvips-installation-and-getting-started/libvips/*.deb\n!pip install /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips/pyvips-2.2.1-py2.py3-none-any.whl --no-index --find-links /kaggle/input/libvips-pyvips-installation-and-getting-started/pyvips","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:18:29.784286Z","iopub.execute_input":"2023-11-12T03:18:29.784758Z","iopub.status.idle":"2023-11-12T03:19:06.36771Z","shell.execute_reply.started":"2023-11-12T03:18:29.784725Z","shell.execute_reply":"2023-11-12T03:19:06.36654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os, gc, time, copy\nimport h5py\nos.environ[\"OPENCV_IO_MAX_IMAGE_PIXELS\"] = pow(2,40).__str__()\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\ntqdm.pandas()\nfrom collections import defaultdict\n\nimport math\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn import model_selection\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\nimport tensorflow as tf\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\n# from torchvision.transforms import v2\n\nimport timm\nfrom timm.data import resolve_data_config\nfrom timm.data.transforms_factory import create_transform\n\nimport IPython.display as display\n\nfrom PIL import Image\nimport cv2 as cv\nimport pyvips\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:06.369982Z","iopub.execute_input":"2023-11-12T03:19:06.370287Z","iopub.status.idle":"2023-11-12T03:19:20.814222Z","shell.execute_reply.started":"2023-11-12T03:19:06.370264Z","shell.execute_reply":"2023-11-12T03:19:20.813413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = dict(\n    seed = 42,\n    folds = 5,\n    img_size = [512, 512],\n    learning_rate = 3e-4, # 2e-5, 3e-4\n    eta_min = 1e-5,\n    epochs = 8,\n    batch_size = 32,\n)\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n#     os.environ['TF_CUDNN_DETERMINISTIC'] = str(SEED)\n#     tf.random.set_seed(SEED)\n#     keras.utils.set_random_seed(seed=SEED)\n    print('seeding done!!!')\n\ndef flush():\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n    \nseeding(config['seed'])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:20.815368Z","iopub.execute_input":"2023-11-12T03:19:20.815919Z","iopub.status.idle":"2023-11-12T03:19:20.863108Z","shell.execute_reply.started":"2023-11-12T03:19:20.815892Z","shell.execute_reply":"2023-11-12T03:19:20.862121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_PATH = Path(\"../input/UBC-OCEAN/\")\nos.listdir(DATA_PATH)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:20.866038Z","iopub.execute_input":"2023-11-12T03:19:20.866427Z","iopub.status.idle":"2023-11-12T03:19:20.980353Z","shell.execute_reply.started":"2023-11-12T03:19:20.866394Z","shell.execute_reply":"2023-11-12T03:19:20.979278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv(DATA_PATH/'train.csv')\ntest_df = pd.read_csv(DATA_PATH/'test.csv')\nsample_df = pd.read_csv(DATA_PATH/'sample_submission.csv')\n\nget_train_images = lambda x: \"/kaggle/input/UBC-OCEAN/train_thumbnails/\" + str(x) +\"_thumbnail\" + \".png\"\nget_test_images = lambda x: \"/kaggle/input/UBC-OCEAN/test_images/\" + str(x) + \".png\"\n\ncheck_path = lambda path: tf.io.gfile.exists(path)\n\ntrain_df['image_path'] = train_df.loc[:, 'image_id'].progress_apply(get_train_images)\ntrain_df['exists'] = train_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking training data ...\")\ndisplay.display(train_df['exists'].value_counts())\ntrain_df = train_df[train_df['exists'] == True]\ntrain_df.reset_index(drop=True, inplace=True)\n\ntest_df['image_path'] = test_df.loc[:, 'image_id'].progress_apply(get_test_images)\ntest_df['exists'] = test_df.loc[:, 'image_path'].map(check_path)\n\nprint(\"Checking test data ...\")\ndisplay.display(test_df['exists'].value_counts())\ntest_df = test_df[test_df['exists'] == True]\ntest_df.reset_index(drop=True, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:20.981578Z","iopub.execute_input":"2023-11-12T03:19:20.981904Z","iopub.status.idle":"2023-11-12T03:19:21.608363Z","shell.execute_reply.started":"2023-11-12T03:19:20.981873Z","shell.execute_reply":"2023-11-12T03:19:21.607477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_thumbnails = os.listdir(DATA_PATH/'train_thumbnails')\n# get_thumbnail = lambda thumbnail: int(thumbnail.split(\"_\")[0])\n# train_thumbnails = [get_thumbnail(t) for t in train_thumbnails]\n# train_df.loc[train_df['image_id'].isin(train_thumbnails), 'is_tma'] = False","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.609639Z","iopub.execute_input":"2023-11-12T03:19:21.609925Z","iopub.status.idle":"2023-11-12T03:19:21.614319Z","shell.execute_reply.started":"2023-11-12T03:19:21.609901Z","shell.execute_reply":"2023-11-12T03:19:21.613427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_thumbnails = os.listdir(DATA_PATH/'test_thumbnails')\n# get_thumbnail = lambda thumbnail: int(thumbnail.split(\"_\")[0])\n# test_thumbnails = [get_thumbnail(t) for t in test_thumbnails]\n# test_df['is_tma'] = True\n# test_df.loc[test_df['image_id'].isin(test_thumbnails), 'is_tma'] = False","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.615388Z","iopub.execute_input":"2023-11-12T03:19:21.61566Z","iopub.status.idle":"2023-11-12T03:19:21.628164Z","shell.execute_reply.started":"2023-11-12T03:19:21.615636Z","shell.execute_reply":"2023-11-12T03:19:21.62709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = train_df['label'].unique().tolist()\nid2label = {l:i for i, l in enumerate(labels)}\nlabel2id = {i:l for i, l in enumerate(labels)}\n\ntrain_df['target'] = train_df['label'].map(id2label)\ntrain_df['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.629398Z","iopub.execute_input":"2023-11-12T03:19:21.630241Z","iopub.status.idle":"2023-11-12T03:19:21.64521Z","shell.execute_reply.started":"2023-11-12T03:19:21.630215Z","shell.execute_reply":"2023-11-12T03:19:21.644344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wsi_images = train_df[train_df['is_tma'] == False]\n# tma_images = train_df[train_df['is_tma'] == True]\n# wsi_index = wsi_images.index.values\n# tma_index = tma_images.index.values","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.646213Z","iopub.execute_input":"2023-11-12T03:19:21.646454Z","iopub.status.idle":"2023-11-12T03:19:21.653624Z","shell.execute_reply.started":"2023-11-12T03:19:21.646424Z","shell.execute_reply":"2023-11-12T03:19:21.652881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create patches","metadata":{}},{"cell_type":"code","source":"# img = cv.imread(train_df.loc[0, 'image_path'])\n# img = cv.resize(img, (2048, 2048))\n# h, w = img.shape[:2]\n# rows = np.split(img, h // 256, axis=0)\n# patches = np.array([np.split(row, w // ps, axis=1) for row in rows])\n# black = np.array([0, 0, 0])\n# not_black = np.any(patches != black, axis=(2, 3, 4))\n# non_black_patches = patches[not_black]\n\n# from mpl_toolkits.axes_grid1 import ImageGrid\n\n# fig = plt.figure(figsize=(10, 5))\n# grid = ImageGrid(fig, 111, nrows_ncols=(2, 4), axes_pad=0.01)\n# for ax, patch in zip(grid, non_black_patches):\n#     ax.imshow(patch)\n#     ax.axis('off')","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.656938Z","iopub.execute_input":"2023-11-12T03:19:21.657178Z","iopub.status.idle":"2023-11-12T03:19:21.665092Z","shell.execute_reply.started":"2023-11-12T03:19:21.657158Z","shell.execute_reply":"2023-11-12T03:19:21.66431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Method 2","metadata":{}},{"cell_type":"code","source":"def read_images(image_path, scale_factor=2):\n    image = pyvips.Image.new_from_file(image_path, access='sequential')\n    return image.resize(1.0/scale_factor).numpy()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.666102Z","iopub.execute_input":"2023-11-12T03:19:21.666335Z","iopub.status.idle":"2023-11-12T03:19:21.674764Z","shell.execute_reply.started":"2023-11-12T03:19:21.666315Z","shell.execute_reply":"2023-11-12T03:19:21.674006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_patches(data, patch_size=512, scale_factor=4):\n    flush()\n\n    k=0\n    IMAGES = list()\n    LABELS = list()\n\n    # define the lower and upper ranges for the black and white colors in HSV\n    lower_black = np.array([0, 0, 0])\n    upper_black = np.array([30, 30, 30])     # [179, 50, 50]\n    lower_white = np.array([179, 30, 30])   # [0, 0, 200]\n    upper_white = np.array([179, 240, 50])   # [179, 50, 255]\n\n    for i in tqdm(range(data.shape[0])):\n#         img = cv.imread(data.loc[i, 'image_path'])\n        img = read_images(data.loc[i, 'image_path'], scale_factor=scale_factor)\n        img = cv.resize(img, (4096, 2048))\n        h, w = img.shape[:2]\n        rows = np.split(img, h // patch_size, axis=0)\n        patches = np.array([np.split(row, w // patch_size, axis=1) for row in rows])\n        img_mean = np.mean(img)\n\n        black = np.array([0, 0, 0])\n        not_black = np.any(patches != black, axis=(2, 3, 4))\n        non_black_patches = patches[not_black]\n\n        for j in range(non_black_patches.shape[0]):\n            patch = non_black_patches[j]\n            p_mean = np.mean(patch)\n            img_id = data.loc[i, 'image_id']\n            label = data.loc[i, 'target']\n\n            # convert the patch to HSV\n            patch_hsv = cv.cvtColor(patch, cv.COLOR_BGR2HSV)\n\n            # create masks for the black and white colors\n            black_mask = cv.inRange(patch_hsv, lower_black, upper_black)\n            white_mask = cv.inRange(patch_hsv, lower_white, upper_white)\n\n            # check if any of the masks contain any non-zero values\n            if not np.any(black_mask) and not np.any(white_mask):\n                IMAGES.append(patch)\n                LABELS.append([img_id, label])\n\n    flush()\n    \n    return {\"images\": np.array(IMAGES), \"target\": np.array(LABELS)}\n\nPATCHES = create_patches(train_df)\nflush()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:19:21.67579Z","iopub.execute_input":"2023-11-12T03:19:21.676046Z","iopub.status.idle":"2023-11-12T03:24:08.752208Z","shell.execute_reply.started":"2023-11-12T03:19:21.676025Z","shell.execute_reply":"2023-11-12T03:24:08.751207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.save(\"image_patches.npy\", PATCHES)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.753779Z","iopub.execute_input":"2023-11-12T03:24:08.754056Z","iopub.status.idle":"2023-11-12T03:24:08.757991Z","shell.execute_reply.started":"2023-11-12T03:24:08.754033Z","shell.execute_reply":"2023-11-12T03:24:08.756954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# p = np.load('image_patches.npy', allow_pickle=True).item()\n# p['images']","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.759061Z","iopub.execute_input":"2023-11-12T03:24:08.759327Z","iopub.status.idle":"2023-11-12T03:24:08.769776Z","shell.execute_reply.started":"2023-11-12T03:24:08.759304Z","shell.execute_reply":"2023-11-12T03:24:08.768953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_channels = batch.shape[-1]\n# MEANS = list();\n# STDS = list()\n\n# for c in range(n_channels):\n#     mean = np.mean(batch[:, :, :, c])\n#     std = np.std(batch[:, :, :, c])\n#     MEANS.append(mean)\n#     STDS.append(std)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.770805Z","iopub.execute_input":"2023-11-12T03:24:08.771057Z","iopub.status.idle":"2023-11-12T03:24:08.78153Z","shell.execute_reply.started":"2023-11-12T03:24:08.771035Z","shell.execute_reply":"2023-11-12T03:24:08.780806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# STDS, MEANS","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.782706Z","iopub.execute_input":"2023-11-12T03:24:08.783032Z","iopub.status.idle":"2023-11-12T03:24:08.795027Z","shell.execute_reply.started":"2023-11-12T03:24:08.783009Z","shell.execute_reply":"2023-11-12T03:24:08.794232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class UBCPatchDataset(Dataset):\n    \n    def __init__(self, image_data, labels, transform):\n        super(UBCPatchDataset, self).__init__()\n        self.data = image_data\n        self.labels = labels\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        image = self.data[idx]\n        image = image.astype(np.float32) / 255.0\n    \n        label = self.labels[idx]\n        image = self.transform(image=image)['image']\n        return {'image': image, 'target': label}","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.795931Z","iopub.execute_input":"2023-11-12T03:24:08.796201Z","iopub.status.idle":"2023-11-12T03:24:08.805017Z","shell.execute_reply.started":"2023-11-12T03:24:08.796178Z","shell.execute_reply":"2023-11-12T03:24:08.804323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PATCHES['images'][[0, 2, 3, 5], ...].shape","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.806091Z","iopub.execute_input":"2023-11-12T03:24:08.806335Z","iopub.status.idle":"2023-11-12T03:24:08.815453Z","shell.execute_reply.started":"2023-11-12T03:24:08.806313Z","shell.execute_reply":"2023-11-12T03:24:08.814558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tsfm = A.Compose([\n# #     A.ShiftScaleRotate(interpolation=cv.INTER_LANCZOS4, p=1),\n# #     A.Perspective(interpolation=cv.INTER_LANCZOS4, p=1),\n#     A.ImageCompression(quality_lower=10, quality_upper=100, p=0.5),\n#     A.ColorJitter(brightness=0.2, saturation=0.5, hue=0.5, p=0.5),\n    \n# #     A.RandomContrast(limit=0.7, p=1),\n# #     A.HueSaturationValue(hue_shift_limit=-10, sat_shift_limit=30, val_shift_limit=10, p=1),\n# #     A.ChannelDropout(p=1),\n#     A.Resize(height=512, width=512, p=1),\n#     ToTensorV2(),\n# ])\n\n# ds = UBCPatchDataset(image_data=PATCHES['images'], labels=PATCHES['target'], transform=tsfm)\n# dls = DataLoader(ds, batch_size=64, shuffle=True, num_workers=2, pin_memory=True)\n# b = next(iter(dls))\n# # b['image'].size(), b['target']\n# b.keys()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.816547Z","iopub.execute_input":"2023-11-12T03:24:08.816818Z","iopub.status.idle":"2023-11-12T03:24:08.824957Z","shell.execute_reply.started":"2023-11-12T03:24:08.816795Z","shell.execute_reply":"2023-11-12T03:24:08.824092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# b = next(iter(dls))\n# b_size = b['image'].size()[0]\n# row = 4\n# col = b_size // row\n\n# plt.figure(figsize=(16, 6))\n\n# for i in tqdm(range(b_size)):\n#     image, target = b['image'][i], b['target'][i]\n#     image = image.permute(1,2,0).cpu().numpy()\n    \n#     plt.subplot(row, col, i + 1)\n#     plt.xticks([])\n#     plt.yticks([])\n#     plt.title(target.cpu().numpy(), fontsize=8)\n#     plt.imshow(image);","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.82601Z","iopub.execute_input":"2023-11-12T03:24:08.82625Z","iopub.status.idle":"2023-11-12T03:24:08.834822Z","shell.execute_reply.started":"2023-11-12T03:24:08.826229Z","shell.execute_reply":"2023-11-12T03:24:08.83407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get data loaders\n\n```python\nstd, mean = ([0.10156386, 0.15501982, 0.11310415], [0.81773764, 0.7082714, 0.8277454]),\nstd, mean = ([0.10604138, 0.16353978, 0.11568955], [0.8172852, 0.71010786, 0.82839715])\n```","metadata":{}},{"cell_type":"code","source":"def get_transforms(img_size):\n    train_tsfm = A.Compose(\n        [\n            A.Resize(height=img_size[0], width=img_size[1]),\n            A.ImageCompression(quality_lower=10, quality_upper=100, p=0.5),\n            A.ColorJitter(brightness=0.2, saturation=0.5, hue=0.5, p=0.5),\n#             A.RandomResizedCrop(height=1024, width=1024, always_apply=True),\n#             A.CenterCrop(height=384, width=384, always_apply=True),\n            \n            A.VerticalFlip(p=0.5),\n            A.HorizontalFlip(p=0.5),\n            A.Rotate(limit=(-25, 25), p=0.5),\n            \n#             A.Normalize(mean=(0.8172852, 0.71010786, 0.82839715), std=(0.10604138, 0.16353978, 0.11568955)),\n            ToTensorV2(),\n        ]\n    )\n    \n    valid_tsfm = A.Compose(\n        [\n            A.Resize(height=img_size[0], width=img_size[1]),\n            ToTensorV2(),\n        ]\n    )\n    \n    return {\"train\": train_tsfm, \"valid\": valid_tsfm}\n\n\ndef get_dataloaders(patches, img_size, batch_size, split='train'):\n    tsfm = get_transforms(img_size=img_size)\n    if split.lower() == 'train':\n#         ds = MemCornDataset(data, transform=tsfm[split], labels=labels)\n        ds = UBCPatchDataset(image_data=patches['images'], labels=patches['target'], transform=tsfm[split])\n        dls = DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=2, pin_memory=True, drop_last=True)\n        \n    elif split.lower() == 'valid':\n        ds = UBCPatchDataset(image_data=patches['images'], labels=patches['target'], transform=tsfm[split])\n        dls = DataLoader(ds, batch_size=batch_size*2, shuffle=False, num_workers=2, pin_memory=True, drop_last=True)\n        \n    else:\n        raise ValueError('Invalid split choose either train or valid')\n    return dls","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.835784Z","iopub.execute_input":"2023-11-12T03:24:08.836035Z","iopub.status.idle":"2023-11-12T03:24:08.849353Z","shell.execute_reply.started":"2023-11-12T03:24:08.836013Z","shell.execute_reply":"2023-11-12T03:24:08.848555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dls = get_dataloaders(PATCHES, config['img_size'], config['batch_size'])\n\nb = next(iter(dls))\nb_size = b['image'].size()[0]\nrow = 4\ncol = b_size // row\n\nplt.figure(figsize=(16, 6))\n\nfor i in tqdm(range(b_size)):\n    image, target = b['image'][i], b['target'][i]\n    image = image.permute(1,2,0).cpu().numpy()\n    \n    plt.subplot(row, col, i + 1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.title(target.cpu().numpy(), fontsize=8)\n    plt.imshow(image);","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:08.850566Z","iopub.execute_input":"2023-11-12T03:24:08.85084Z","iopub.status.idle":"2023-11-12T03:24:18.782263Z","shell.execute_reply.started":"2023-11-12T03:24:08.850816Z","shell.execute_reply":"2023-11-12T03:24:18.781312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Layers\n\nCredit to [TheoViel](https://github.com/TheoViel/kaggle_rsna_abdominal_trauma/blob/cleaning/src/model_zoo/layers.py) For the brilliant work","metadata":{}},{"cell_type":"code","source":"def gem(x, p=3, eps=1e-6):\n    \"\"\"\n    Apply Generalized Mean Pooling (GeM) to a tensor.\n\n    Args:\n        x (torch.Tensor): Input tensor of shape (batch_size, channels, height, width).\n        p (float): The p-value for the generalized mean. Default is 3.\n        eps (float): A small constant added to the denominator to prevent division by zero. Default is 1e-6.\n\n    Returns:\n        torch.Tensor: GeM-pooled representation of the input tensor.\n    \"\"\"\n    return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1))).pow(1.0 / p)\n\n\nclass GeM(nn.Module):\n    \"\"\"\n    Generalized Mean Pooling (GeM) layer for global average pooling.\n\n    Attributes:\n        p (float or torch.Tensor): The p-value for the generalized mean.\n        eps (float): A small constant added to the denominator to prevent division by zero.\n    \"\"\"\n    def __init__(self, p=3, eps=1e-6, p_trainable=False):\n        \"\"\"\n        Initialize the GeM layer.\n\n        Args:\n            p (float or torch.Tensor): The p-value for the generalized mean.\n            eps (float, optional): Eps to prevent division by zero. Defaults to 1e-6.\n            p_trainable (bool, optional): Whether p is trainable. Defaults to False.\n        \"\"\"\n        super(GeM, self).__init__()\n        if p_trainable:\n            self.p = Parameter(torch.ones(1) * p)\n        else:\n            self.p = p\n        self.eps = eps\n\n    def forward(self, x):\n        \"\"\"\n        Perform the GeM pooling operation on the input tensor.\n\n        Args:\n            x (torch.Tensor): Input tensor of shape (batch_size, channels, height, width).\n\n        Returns:\n            torch.Tensor: GeM-pooled representation of the input tensor.\n        \"\"\"\n        ret = gem(x, p=self.p, eps=self.eps)\n        return ret\n\n\nclass Attention(nn.Module):\n    \"\"\"\n    Attention module for sequence data.\n\n    Attributes:\n        hidden_dim (int): The dimension of the input sequence.\n        attention_dim (int): The dimension of the attention layer.\n    \"\"\"\n    def __init__(self, hidden_dim, attention_dim=None):\n        \"\"\"\n        Constructor\n\n        Args:\n            hidden_dim (int): The dimension of the input sequence.\n            attention_dim (int, optional): The dimension of the attention layer.\n                Defaults to None, in which case it's set to `hidden_dim`.\n        \"\"\"\n        super().__init__()\n\n        self.hidden_dim = hidden_dim\n        self.attention_dim = attention_dim\n        if self.attention_dim is None:\n            self.attention_dim = self.hidden_dim\n        # W * x + b\n        self.proj_w = nn.Linear(self.hidden_dim, self.attention_dim, bias=True)\n        # v.T\n        self.proj_v = nn.Linear(self.attention_dim, 1, bias=False)\n\n    def forward(self, x):\n        \"\"\"\n        Perform the forward pass of the attention mechanism.\n\n        Args:\n            x (torch.Tensor): Input sequence data of shape (batch_size, seq_len, input_dim).\n\n        Returns:\n            torch.Tensor: Attention-weighted representation of the input sequence.\n        \"\"\"\n        batch_size, seq_len, _ = x.size()\n        H = torch.tanh(self.proj_w(x))\n        att_scores = torch.softmax(self.proj_v(H), axis=1)\n        attn_x = (x * att_scores).sum(1)\n        return attn_x","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:18.784106Z","iopub.execute_input":"2023-11-12T03:24:18.784573Z","iopub.status.idle":"2023-11-12T03:24:18.801022Z","shell.execute_reply.started":"2023-11-12T03:24:18.784535Z","shell.execute_reply":"2023-11-12T03:24:18.800118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load pretrained models","metadata":{"execution":{"iopub.status.busy":"2023-11-10T10:42:42.52132Z","iopub.execute_input":"2023-11-10T10:42:42.522349Z","iopub.status.idle":"2023-11-10T10:42:42.534058Z","shell.execute_reply.started":"2023-11-10T10:42:42.522313Z","shell.execute_reply":"2023-11-10T10:42:42.533061Z"}}},{"cell_type":"code","source":"if config['img_size'][0] > 224:\n    model_id_timm = \"timm/tf_efficientnetv2_b3.in21k_ft_in1k\"\nelif config['img_size'][0] == 224:\n#     model_id_timm = \"timm/vit_base_patch32_clip_224.openai_ft_in1k\"\n    model_id_timm = \"timm/vit_base_patch8_224.dino\"\n# timm.list_pretrained(\"*swin*\")\nmodel = timm.create_model(model_id_timm, pretrained=True, num_classes=512)\n\nclass Regr(torch.nn.Module):\n    \n    def __init__(self, base_model, rate=0.1):\n        super(Regr, self).__init__()\n        self.base_model = base_model\n        self.base_model.classifier = nn.Identity()\n        self.base_model.global_pool = nn.Identity()\n        self.pooling = GeM()\n        \n        self.linear_model = torch.nn.Sequential(\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(512),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyBatchNorm1d(),\n            nn.LazyLinear(128),\n            nn.ReLU(),\n            nn.Dropout(rate),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n        \n        \n    def forward(self, x):\n        x = self.base_model(x)\n        x = self.pooling(x)\n        x = x.view(x.size(0), -1)\n        x = self.linear_model(x)\n        return x\n\n\nclass LSTMClassifier(torch.nn.Module):\n\n    def __init__(self, base_model, rate=0.1):\n        super(LSTMClassifier, self).__init__()\n        self.base_model = base_model\n\n        # Replace the linear model with an LSTM layer\n        self.lstm_model = nn.LSTM(input_size=512, hidden_size=256, num_layers=2, dropout=0.0, bidirectional=True, batch_first=True)\n#         self.cnn_model = nn.LazyConv1d(out_channels=128, kernel_size=3, padding=1, stride=1)\n\n        # Maintain the same output layer\n        self.output_layer = nn.Sequential(\n            nn.LazyLinear(128),\n            nn.LazyBatchNorm1d(),\n            nn.Dropout(rate),\n#             nn.LeakyReLU(0.1),\n            nn.ReLU(),\n            nn.LazyLinear(5),\n            nn.Softmax(dim=1),\n        )\n\n    def forward(self, x):\n        x = self.base_model(x)\n        x, _ = self.lstm_model(x)\n        x = x.view(x.size(0), -1)\n\n        x = self.output_layer(x)\n\n        return x\n\n# lstm_classifier = LSTMClassifier(base_model, rate=0.1)\n    \n# regr_model = Regr(base_model=model)\n\n# regr_model.eval()\n# out = regr_model(b['image'])\n# # out = torch.argmax(out, dim=1)\n# loss = torch.nn.CrossEntropyLoss()\n# loss(out, b['target'])\n# out, b['target'].long()\n# torch.argmax(out, dim=1)\n\n# x1 = torch.randn(3, 5, requires_grad=True)\n# x2 = torch.empty(3, dtype=torch.long).random_(5)\n# out = loss(x1, x2)\n\n# loss(out, b['target'])","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:18.802647Z","iopub.execute_input":"2023-11-12T03:24:18.80298Z","iopub.status.idle":"2023-11-12T03:24:19.727765Z","shell.execute_reply.started":"2023-11-12T03:24:18.802949Z","shell.execute_reply":"2023-11-12T03:24:19.726793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FocalLoss(nn.Module):\n    def __init__(self, alpha=1, gamma=2, reduction='mean'):\n        super(FocalLoss, self).__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        ce_loss = nn.CrossEntropyLoss(reduction='none')(inputs, targets)\n        pt = torch.exp(-ce_loss)\n        ft = self.alpha * (1 - pt) ** self.gamma\n        focal_loss = ft * ce_loss\n        \n        if self.reduction == 'mean':\n            return torch.mean(focal_loss)\n        elif self.reduction == 'sum':\n            return torch.sum(focal_loss)\n        else:\n            return focal_loss","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:19.729049Z","iopub.execute_input":"2023-11-12T03:24:19.729343Z","iopub.status.idle":"2023-11-12T03:24:19.736768Z","shell.execute_reply.started":"2023-11-12T03:24:19.729318Z","shell.execute_reply":"2023-11-12T03:24:19.735688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MetricMonitor:\n    def __init__(self, float_precision=4):\n        self.float_precision = float_precision\n        self.reset()\n\n    def reset(self):\n        self.metrics = defaultdict(lambda: {\"val\": 0, \"count\": 0, \"avg\": 0})\n\n    def update(self, metric_name, val):\n        metric = self.metrics[metric_name]\n\n        metric[\"val\"] += val\n        metric[\"count\"] += 1\n        metric[\"avg\"] = metric[\"val\"] / metric[\"count\"]\n\n    def __str__(self):\n        return \" | \".join(\n            [\n                \"{metric_name}: {avg:.{float_precision}f}\".format(\n                    metric_name=metric_name, avg=metric[\"avg\"], float_precision=self.float_precision\n                )\n                for (metric_name, metric) in self.metrics.items()\n            ]\n        )\n\n\ndef ACC(y_true, y_preds):\n    y_true = y_true.detach().cpu().numpy()\n    y_preds = torch.argmax(y_preds, dim=1)\n    y_preds = y_preds.detach().cpu().numpy()\n    return metrics.balanced_accuracy_score(y_true, y_preds)\n\n# def flush():\n#     gc.collect()\n#     torch.cuda.empty_cache()\n#     torch.cuda.reset_peak_memory_stats()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:19.737651Z","iopub.execute_input":"2023-11-12T03:24:19.737913Z","iopub.status.idle":"2023-11-12T03:24:19.748766Z","shell.execute_reply.started":"2023-11-12T03:24:19.73788Z","shell.execute_reply":"2023-11-12T03:24:19.747905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mixup(inputs, truth, clip=[0, 1]):\n    indices = torch.randperm(inputs.size(0))\n    shuffled_input = inputs[indices]\n    shuffled_labels = truth[indices]\n    \n    lam = np.random.uniform(clip[0], clip[1])\n    inputs = inputs * lam + shuffled_input * (1 - lam)\n    return inputs, truth, shuffled_labels, lam\n\n\ndef train_with_mixup(model, optimizer, criterion, data_loader, scaler, device='cpu', epoch=1, n_iters=1000):\n    model.train()\n    train_loss = 0\n    correct = 0\n    n_total = 0\n    example_ct = 0\n    step_ct = 0\n    metric_monitor = MetricMonitor()\n    stream = tqdm(data_loader)\n    for i, batch in enumerate(stream, start=1):\n        xb, yb = batch['image'], batch['target']\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        \n        do_mixup = False\n        if random.random() < 0.4:\n            do_mixup = True\n            xb, yb, yb_mix, lam = mixup(xb, yb)\n        \n        with torch.autocast(device_type=device, dtype=torch.float16):\n            outputs = model(xb)\n            loss = criterion(outputs, yb)\n            if do_mixup:\n                loss11 = criterion(outputs, yb_mix)\n                loss = loss * lam + loss11 * (1 - lam)\n            \n        train_loss += loss.detach().float()\n        scaler.scale(loss).backward()\n        \n        acc = ACC(yb, outputs)\n        metric_monitor.update('Loss', loss.item())\n        metric_monitor.update('Balanced Accuracy', acc)\n        \n        example_ct += len(xb)\n        METRICS = {\n            \"train/train_loss\": train_loss,\n            \"train/epoch\": (i + 1 + (n_iters * epoch)) / n_iters,\n            \"train/example_ct\": example_ct,\n            \"train/balanced_acc\": acc,\n        }\n        \n#         if (i + 1) < n_iters:\n#             # log train metrics to wandb\n#             wandb.log(METRICS)\n            \n        step_ct += 1\n        \n        if (i+1) % n_iters == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad(set_to_none=True)\n            \n        stream.set_description(\n        \"Epoch: {epoch}. Train.      {metric_monitor}\".format(epoch=epoch, metric_monitor=metric_monitor))\n    \n    train_loss_total = (train_loss / len(data_loader)).item()\n    flush()\n    \n    return train_loss_total, METRICS\n\n\ndef train_one_loop(model, optimizer, criterion, data_loader, scaler, device='cpu', epoch=1, n_iters=1000):\n    model.train()\n    train_loss = 0\n    correct = 0\n    n_total = 0\n    example_ct = 0\n    step_ct = 0\n    metric_monitor = MetricMonitor()\n    stream = tqdm(data_loader)\n    for i, batch in enumerate(stream, start=1):\n        xb, yb = batch['image'], batch['target']\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        \n        with torch.autocast(device_type=device, dtype=torch.float16):\n            outputs = model(xb)\n            loss = criterion(outputs, yb)\n            \n        train_loss += loss.detach().float()\n        scaler.scale(loss).backward()\n        \n        acc = ACC(yb, outputs)\n        metric_monitor.update('Loss', loss.item())\n        metric_monitor.update('Balanced Accuracy', acc)\n        \n        example_ct += len(xb)\n        METRICS = {\n            \"train/train_loss\": train_loss,\n            \"train/epoch\": (i + 1 + (n_iters * epoch)) / n_iters,\n            \"train/example_ct\": example_ct,\n            \"train/balanced_acc\": acc,\n        }\n        \n#         if (i + 1) < n_iters:\n#             # log train metrics to wandb\n#             wandb.log(METRICS)\n            \n        step_ct += 1\n        \n        if (i+1) % n_iters == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad(set_to_none=True)\n            \n        stream.set_description(\n        \"Epoch: {epoch}. Train.      {metric_monitor}\".format(epoch=epoch, metric_monitor=metric_monitor))\n    \n    train_loss_total = (train_loss / len(data_loader)).item()\n    flush()\n    \n    return train_loss_total, METRICS\n        \n    \ndef valid_one_loop(model, criterion, data_loader, device='cpu', epoch=1):\n    model.eval()\n    valid_loss = 0\n    correct = 0\n    n_total = 0\n    metric_monitor = MetricMonitor()\n    stream = tqdm(data_loader)\n    for i, batch in enumerate(stream, start=1):\n        xb, yb = batch['image'], batch['target']\n        xb, yb = xb.to(device, non_blocking=True), yb.to(device, non_blocking=True)\n        \n        with torch.autocast(device_type=device, dtype=torch.float16):\n            with torch.no_grad():\n                outputs = model(xb)\n            loss = criterion(outputs, yb)\n            \n        valid_loss += loss.detach().float()\n        acc = ACC(yb, outputs)\n        metric_monitor.update('Loss', loss.item())\n        metric_monitor.update('Balanced Accuracy', acc)\n        stream.set_description(\n        \"Epoch: {epoch}. Valid.      {metric_monitor}\".format(epoch=epoch, metric_monitor=metric_monitor))\n        val_metrics = {\n            \"valid/val_loss\": valid_loss,\n            \"valid/balanced_acc\": acc,\n        }\n#         wandb.log(val_metrics)\n    \n    valid_loss_total = (valid_loss / len(data_loader)).item()\n    flush()\n    return valid_loss_total, val_metrics","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:19.749935Z","iopub.execute_input":"2023-11-12T03:24:19.750187Z","iopub.status.idle":"2023-11-12T03:24:19.777417Z","shell.execute_reply.started":"2023-11-12T03:24:19.750165Z","shell.execute_reply":"2023-11-12T03:24:19.776564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"name = model_id_timm.split('/')[-1]\n\ndef train(model, optimizer, criterion, train_loader, valid_loader, epochs, scheduler, lr_reduce, \n          scaler, device='cpu', fold=0, n_iters=100):\n    \n    best_metric = np.inf\n    loss_min = np.inf\n    \n    for epoch in tqdm(range(1, epochs+1)):\n        train_loss, train_metrics = train_one_loop(model, optimizer, criterion, train_loader, \n                                    scaler=scaler, device=device, epoch=epoch, n_iters=n_iters)\n        \n#         train_loss, train_metrics = train_with_mixup(model, optimizer, criterion, train_loader, \n#                                     scaler=scaler, device=device, epoch=epoch, n_iters=n_iters)\n        \n        valid_loss, valid_metrics = valid_one_loop(model, criterion, valid_loader, device=device, epoch=epoch)\n        scheduler.step(epoch-1)\n        lr_reduce.step(valid_loss)\n        \n        train_metrics[\"train/rmse\"] = train_loss\n        valid_metrics[\"valid/rmse\"] = valid_loss\n#         wandb.log({**train_metrics, **valid_metrics})\n        \n        metric = valid_loss\n        if metric < best_metric:\n            print(f\"Best metric: ({best_metric:.6f} --> {metric:.6f}). Saving model ...\")\n            torch.save(model.state_dict(), f\"{name}_fold_{fold}.pth\")\n            best_metric = metric","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:19.781924Z","iopub.execute_input":"2023-11-12T03:24:19.782238Z","iopub.status.idle":"2023-11-12T03:24:19.792946Z","shell.execute_reply.started":"2023-11-12T03:24:19.782215Z","shell.execute_reply":"2023-11-12T03:24:19.791977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Using {device} ...\")\n\n# train_df = train_df.sample(frac=1)\n\ngc.collect()\n\nkfold = model_selection.StratifiedKFold(n_splits=config['folds'], shuffle=True, random_state=config['seed'])\n\nimages = PATCHES['images']\nx = np.arange(images.shape[0])\ny = PATCHES['target'][:, -1]\n\nfor fold, (tr_idx, val_idx) in enumerate(kfold.split(x, y)):\n#     run = wandb.init(\n#         project=\"cgair-pytorch-baseline\"\n#     )\n#     artifact = wandb.Artifact(f'fold_{fold}_weights', type='model')\n    \n    if fold in [0,1,4]:\n        epochs = 3+fold\n    else:\n        epochs = config['epochs']\n    print(f\"\\n===> Fold {fold} ...\")\n    \n    train_ds = {\"images\": images[tr_idx, ...], \"target\": y[tr_idx]}\n    valid_ds = {\"images\": images[val_idx, ...], \"target\": y[val_idx]}\n    \n#     train_ds = train_ds.reset_index(drop=True)\n#     valid_ds = valid_ds.reset_index(drop=True)\n    \n    train_loader = get_dataloaders(train_ds, img_size=config['img_size'], batch_size=config['batch_size'], split='train')\n    valid_loader = get_dataloaders(valid_ds, img_size=config['img_size'], batch_size=config['batch_size'], split='valid')\n    \n#     regr_model = Regr(base_model=model)\n    regr_model = LSTMClassifier(base_model=model, rate=0.2)\n    regr_model = regr_model.to(device)\n    \n#     optimizer = torch.optim.AdamW(regr_model.parameters(), lr=config['learning_rate'])\n    optimizer = torch.optim.Adam(regr_model.parameters(), lr=config['learning_rate'])\n    criterion = torch.nn.CrossEntropyLoss().to(device)\n#     criterion = FocalLoss().to(device)\n    scaler = torch.cuda.amp.GradScaler()\n#     scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n    lr_reduce = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=2, verbose=True)\n#     scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=config['epochs'], eta_min=config['eta_min'])\n    scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, epochs, eta_min=config['eta_min'])\n#     scheduler = torch.optim.lr_scheduler.CyclicLR(),\n    num_training_steps = math.ceil(len(train_loader)/config['batch_size'])\n#     wandb.config = config\n\n    train(regr_model, optimizer, criterion, train_loader, valid_loader, epochs=epochs, \n          scheduler=scheduler, lr_reduce=lr_reduce, scaler=scaler, device=device, fold=fold, n_iters=num_training_steps)\n    \n#     torch.save(regr_model.state_dict(), f\"{name}_fold_{fold}.pth\")\n    del regr_model\n#     break\n#     artifact.add_file(f\"{name}_fold_{fold}.pth\")\n#     run.log_artifact(artifact)\n# wandb.finish()\n    flush()","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:24:19.794364Z","iopub.execute_input":"2023-11-12T03:24:19.794728Z","iopub.status.idle":"2023-11-12T04:05:07.394678Z","shell.execute_reply.started":"2023-11-12T03:24:19.794702Z","shell.execute_reply":"2023-11-12T04:05:07.393541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Updated image ids","metadata":{}},{"cell_type":"markdown","source":"```python\n# Import the modules\nimport json\nimport pandas as pd\n\n# Load the updated_image_ids.json file as a dictionary\nwith open('updated_image_ids.json', 'r') as f:\n    updated_image_ids = json.load(f)\n\n# Load the old dataframe as a pandas dataframe\nold_df = pd.read_csv('old_dataframe.csv')\n\n# Iterate over the rows of the dataframe and update the image_id column\nfor index, row in old_df.iterrows():\n    # Get the image_name and image_id from the row\n    image_name = row['image_name']\n    image_id = row['image_id']\n    # Check if the image_id is different from the updated_image_ids dictionary\n    if image_id != updated_image_ids[image_name]:\n        # Update the image_id column with the new value\n        old_df.loc[index, 'image_id'] = updated_image_ids[image_name]\n\n# Save the updated dataframe as a new file\nold_df.to_csv('new_dataframe.csv', index=False)\n\n```","metadata":{"execution":{"iopub.status.busy":"2023-11-12T04:05:07.396837Z","iopub.execute_input":"2023-11-12T04:05:07.397208Z","iopub.status.idle":"2023-11-12T04:05:07.402418Z","shell.execute_reply.started":"2023-11-12T04:05:07.397176Z","shell.execute_reply":"2023-11-12T04:05:07.401308Z"}}},{"cell_type":"code","source":"# # dls = get_dataloaders(PATCHES, config['img_size'], config['batch_size'])\n\n# b = next(iter(train_loader))\n# b_size = b['image'].size()[0]\n# row = 4\n# col = b_size // row\n\n# plt.figure(figsize=(16, 6))\n\n# for i in tqdm(range(b_size)):\n#     image, target = b['image'][i], b['target'][i]\n#     image = image.permute(1,2,0).cpu().numpy()\n    \n#     plt.subplot(row, col, i + 1)\n#     plt.xticks([])\n#     plt.yticks([])\n#     plt.title(target.cpu().numpy(), fontsize=8)\n#     plt.imshow(image);","metadata":{"execution":{"iopub.status.busy":"2023-11-12T04:05:07.404789Z","iopub.execute_input":"2023-11-12T04:05:07.405209Z","iopub.status.idle":"2023-11-12T04:05:07.416168Z","shell.execute_reply.started":"2023-11-12T04:05:07.405173Z","shell.execute_reply":"2023-11-12T04:05:07.41494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}