{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":45867,"databundleVersionId":6924515,"sourceType":"competition"},{"sourceId":6654895,"sourceType":"datasetVersion","datasetId":3840540},{"sourceId":6654988,"sourceType":"datasetVersion","datasetId":3840578},{"sourceId":6655032,"sourceType":"datasetVersion","datasetId":3840603},{"sourceId":6655122,"sourceType":"datasetVersion","datasetId":3840660},{"sourceId":6655159,"sourceType":"datasetVersion","datasetId":3840681},{"sourceId":150637573,"sourceType":"kernelVersion"},{"sourceId":150674369,"sourceType":"kernelVersion"},{"sourceId":151666102,"sourceType":"kernelVersion"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"[Tellez *et al.* (2020)](https://arxiv.org/abs/1811.02840) proposed two-stage method to build deep neural networks for gigapixel distopathology image analysis solely using image-level labels.\n\n<img src=\"https://storage.googleapis.com/ubco/Screenshot%202023-11-16%20at%2011.24.50.png\">\n\nThe encoder compress gigapixel image to much smaller feature space and the classifier is trained feeding the features created by the encoder.\nThey tested three different encoding methods: autoencoder, contrastive training, and Bidirectional GAN.\n\nIn this notebook, an autoencoder is trained on 128 x 128 patches with latent dimension is 128. Thus, the compression rate is $384 = \\frac{3 \\times 128 \\times 128}{128}$.","metadata":{}},{"cell_type":"code","source":"from itertools import chain\nfrom pathlib import Path\nfrom typing import Iterator\nimport random\nimport json\nimport os\n\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchvision import io, transforms\nfrom torchvision.transforms import functional as F \nfrom torchvision.utils import make_grid\nimport pandas as pd\nfrom tqdm.auto import tqdm\nimport wandb\nimport matplotlib.pyplot as plt\nfrom kaggle_secrets import UserSecretsClient","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-23T08:10:03.044758Z","iopub.execute_input":"2023-11-23T08:10:03.045396Z","iopub.status.idle":"2023-11-23T08:10:03.053156Z","shell.execute_reply.started":"2023-11-23T08:10:03.045341Z","shell.execute_reply":"2023-11-23T08:10:03.051978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMG_SIZE = 128\nINITIAL_EPOCH = 0\nLOAD_WEIGHT = None\nLAST_EPOCH = 30\nBATCH_SIZE = 1024\nAMP = torch.cuda.is_available()\nDEVICE = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:39:59.908034Z","iopub.execute_input":"2023-11-18T01:39:59.908852Z","iopub.status.idle":"2023-11-18T01:39:59.936633Z","shell.execute_reply.started":"2023-11-18T01:39:59.90882Z","shell.execute_reply":"2023-11-18T01:39:59.935733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.login(key=UserSecretsClient().get_secret(\"WANDB_KEY\"))\nwandb.init(\n    project=\"UBC-OCEAN\",\n    config={\n        \"dataset\": \"ubco\",\n        \"model\": \"VAE\",\n        \"last_epoch\": LAST_EPOCH,\n        \"initial_epoch\": INITIAL_EPOCH,\n        \"batch_size\": BATCH_SIZE,\n        \"device\": DEVICE,\n        \"amp\": AMP,\n    },\n)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-18T01:40:07.135912Z","iopub.execute_input":"2023-11-18T01:40:07.136719Z","iopub.status.idle":"2023-11-18T01:40:41.744179Z","shell.execute_reply.started":"2023-11-18T01:40:07.13669Z","shell.execute_reply":"2023-11-18T01:40:41.74327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"class AverageMeter:\n    def __init__(self) -> None:\n        self.reset()\n\n    def reset(self) -> None:\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val: float, n: int = 1) -> None:\n        self.sum += val * n\n        self.count += n\n\n    @property\n    def average(self) -> float:\n        if self.count == 0:\n            return 0.0\n        return self.sum / self.count","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:40:41.746216Z","iopub.execute_input":"2023-11-18T01:40:41.746799Z","iopub.status.idle":"2023-11-18T01:40:41.754647Z","shell.execute_reply.started":"2023-11-18T01:40:41.746765Z","shell.execute_reply":"2023-11-18T01:40:41.753662Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset and transforms","metadata":{}},{"cell_type":"code","source":"class UBCODataset(Dataset):\n    def __init__(\n        self, image_dirs: list[Path], transforms: transforms.Compose\n    ) -> None:\n        self.image_paths = list(\n            chain.from_iterable([d.glob(\"*.png\") for d in image_dirs])\n        )\n        self.transforms = transforms\n\n    def __len__(self) -> int:\n        return len(self.image_paths)\n\n    def __getitem__(self, index: int) -> torch.Tensor:\n        img = self.read_image(index)\n        return self.transforms(img)\n\n    def read_image(self, idx: int) -> torch.FloatTensor:\n        return io.read_image(str(self.image_paths[idx])).float()\n    \ntrain_transforms = transforms.Compose(\n    [\n        transforms.RandomCrop(size=IMG_SIZE),\n        transforms.RandomHorizontalFlip(p=0.5),\n        transforms.Normalize(mean=[0.5 * 255,]*3, std=[0.5 * 255,]*3),\n    ]\n)\n\neval_transforms = transforms.Compose(\n    [\n        transforms.CenterCrop(size=IMG_SIZE),\n        transforms.Normalize(mean=[0.5 * 255,]*3, std=[0.5 * 255,]*3),\n    ]\n)\n\nreverse_transform = transforms.Compose(\n    [\n        transforms.Normalize(mean=[-1,]*3, std=[2,]*3),\n        transforms.ToPILImage()\n    ]\n)","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:40:41.755992Z","iopub.execute_input":"2023-11-18T01:40:41.756304Z","iopub.status.idle":"2023-11-18T01:40:41.770727Z","shell.execute_reply.started":"2023-11-18T01:40:41.756276Z","shell.execute_reply":"2023-11-18T01:40:41.769763Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model","metadata":{}},{"cell_type":"code","source":"class EncoderBlock(nn.Module):\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        kernel_size: int = 3,\n        padding: int = 1,\n        stride: int = 2,\n    ) -> None:\n        super().__init__()\n        self.conv = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=kernel_size,\n            stride=stride,\n            padding=padding,\n        )\n        self.bn = nn.BatchNorm2d(num_features=out_channels)\n        self.activation = nn.LeakyReLU(inplace=True)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.activation(x)\n        return x\n\n\nclass DecoderBlock(nn.Module):\n    def __init__(\n        self,\n        in_channels: int,\n        out_channels: int,\n        kernel_size: int = 3,\n        padding: int = 1,\n    ) -> None:\n        super().__init__()\n        self.upsample = nn.UpsamplingNearest2d(scale_factor=2)\n        self.conv = nn.Conv2d(\n            in_channels=in_channels,\n            out_channels=out_channels,\n            kernel_size=kernel_size,\n            padding=padding,\n        )\n        self.bn = nn.BatchNorm2d(num_features=out_channels)\n        self.activation = nn.LeakyReLU(inplace=True)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.upsample(x)\n        x = self.conv(x)\n        x = self.bn(x)\n        x = self.activation(x)\n        return x\n\n\nclass Encoder128(nn.Module):\n    def __init__(\n        self,\n        latent_dim: int = 128,\n        out_channels: tuple[int, int, int, int, int] = (32, 64, 128, 256, 512),\n    ) -> None:\n        super().__init__()\n        self.blocks = nn.Sequential()\n        in_channels = 3\n        for out_channel in out_channels:\n            self.blocks.append(EncoderBlock(in_channels=in_channels, out_channels=out_channel))\n            in_channels = out_channel\n\n        self.linear1 = nn.Linear(in_features=out_channel * 4 * 4, out_features=512)\n        self.bn = nn.BatchNorm1d(num_features=512)\n        self.activation = nn.LeakyReLU(inplace=True)\n        self.linear2 = nn.Linear(in_features=512, out_features=latent_dim)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.blocks(x)  # bs, 512, 4, 4\n        x = x.flatten(start_dim=1)\n        x = self.linear1(x)\n        x = self.bn(x)\n        x = self.activation(x)\n        x = self.linear2(x)\n        return x\n\n\nclass Decoder128(nn.Module):\n    def __init__(\n        self,\n        latent_dim: int = 128,\n        out_channels: tuple[int, int, int, int, int] = (256, 128, 64, 32, 16),\n    ) -> None:\n        super().__init__()\n        self.pre_linear = nn.Sequential(\n            nn.Linear(in_features=latent_dim, out_features=512 * 4 * 4),\n            nn.BatchNorm1d(num_features=512 * 4 * 4),\n            nn.LeakyReLU(inplace=True),\n        )\n\n        self.blocks = nn.Sequential()\n        in_channels = 512\n        for out_channel in out_channels:\n            self.blocks.append(DecoderBlock(in_channels=in_channels, out_channels=out_channel))\n            in_channels = out_channel\n\n        self.final_conv = nn.Sequential(\n            nn.Conv2d(in_channels=out_channel, out_channels=3, kernel_size=3, padding=1),\n            nn.Tanh(),\n        )\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        x = self.pre_linear(x)\n        x = x.view(x.size(0), 512, 4, 4)\n        x = self.blocks(x)\n        x = self.final_conv(x)\n        return x\n\n\nclass VanillaVAE(nn.Module):\n    def __init__(\n        self,\n        encoder: nn.Module,\n        decoder: nn.Module,\n        latent_dim: int = 128,\n        kl_w: float = 5e-5,\n        device: torch.device = torch.device(\"cuda\"),\n        amp: bool = False,\n    ) -> None:\n        super().__init__()\n        self.encoder = encoder\n        self.decoder = decoder\n\n        self.fc_mu = nn.Linear(latent_dim, latent_dim)\n        self.fc_var = nn.Linear(latent_dim, latent_dim)\n        self.kl_w = kl_w\n        self.optimizer = self.configure_optimizer()\n        self.device = device\n        self.scaler = torch.cuda.amp.grad_scaler.GradScaler(enabled=amp)\n\n    def configure_optimizer(self) -> torch.optim.Optimizer:\n        self.optimizer = torch.optim.Adam(self.parameters(), lr=1e-4)\n        return self.optimizer\n\n    def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:\n        x = self.encoder(x)\n        mu = self.fc_mu(x)\n        log_var = self.fc_var(x)\n\n        return mu, log_var\n\n    def decode(self, z: torch.Tensor) -> torch.Tensor:\n        return self.decoder(z)\n\n    def reparameterize(self, mu: torch.Tensor, log_var: torch.Tensor) -> torch.Tensor:\n        std = torch.exp(0.5 * log_var)\n        eps = torch.randn_like(std)\n\n        return mu + eps * std\n\n    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n        x = x.to(self.device)\n        mu, log_var = self.encode(x)\n        z = self.reparameterize(mu, log_var)\n        x_hat = self.decode(z)\n        return x_hat, mu, log_var\n\n    def compute_losses(\n        self,\n        x: torch.Tensor,\n        x_hat: torch.Tensor,\n        mu: torch.Tensor,\n        log_var: torch.Tensor,\n    ) -> tuple[torch.Tensor, torch.Tensor]:\n        x = x.to(x_hat.device)\n        rec_loss = nn.functional.mse_loss(input=x_hat, target=x, reduction=\"mean\")\n\n        kl_loss = -0.5 * torch.sum(1 + log_var - mu**2 - log_var.exp(), dim=1)\n        kl_loss = torch.mean(kl_loss)\n\n        return rec_loss, kl_loss\n\n    def train_step(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:\n        self.optimizer.zero_grad()\n        with torch.cuda.amp.autocast(enabled=self.scaler.is_enabled()):\n            x_hat, mu, log_var = self.forward(x)\n            rec_loss, kl_loss = self.compute_losses(x, x_hat, mu, log_var)\n            loss = rec_loss + self.kl_w * kl_loss\n        self.scaler.scale(loss).backward()\n        self.scaler.step(self.optimizer)\n        self.scaler.update()\n        return loss, rec_loss, kl_loss\n\n    @torch.no_grad()\n    def eval_step(self, x: torch.Tensor) -> torch.Tensor:\n        with torch.cuda.amp.autocast(enabled=self.scaler.is_enabled()):\n            x_hat, mu, log_var = self.forward(x)\n            rec_loss, kl_loss = self.compute_losses(x, x_hat, mu, log_var)\n            loss = rec_loss + self.kl_w * kl_loss\n        return loss, rec_loss, kl_loss\n\n    def train_single_epoch(self, data_loader: DataLoader) -> tuple[float, float, float]:\n        self.train()\n        loss = AverageMeter()\n        rec_loss = AverageMeter()\n        kl_loss = AverageMeter()\n        pbar = tqdm(total=len(data_loader))\n        for x in data_loader:\n            _loss, _rec_loss, _kl_loss = self.train_step(x)\n            if not torch.isfinite(_loss):\n                raise RuntimeError(f\"loss not finite: {_loss}\")\n            loss.update(_loss.item())\n            rec_loss.update(_rec_loss.item())\n            kl_loss.update(_kl_loss.item())\n            pbar.set_description(\n                f\"loss: {loss.average:.3f} rec loss: {rec_loss.average:.3f} kl loss: {kl_loss.average:.3f}\"\n            )\n            pbar.update()\n        pbar.close()\n        return loss.average, rec_loss.average, kl_loss.average\n\n    def evaluate(self, data_loader: DataLoader) -> tuple[float, float, float]:\n        self.eval()\n        loss = AverageMeter()\n        rec_loss = AverageMeter()\n        kl_loss = AverageMeter()\n        pbar = tqdm(total=len(data_loader))\n        for x in data_loader:\n            _loss, _rec_loss, _kl_loss = self.eval_step(x)\n            loss.update(_loss.item())\n            rec_loss.update(_rec_loss.item())\n            kl_loss.update(_kl_loss.item())\n            pbar.set_description(\n                f\"loss: {loss.average:.3f} rec loss: {rec_loss.average:.3f} kl loss: {kl_loss.average:.3f}\"\n            )\n            pbar.update()\n        pbar.close()\n        return loss.average, rec_loss.average, kl_loss.average","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:40:41.773688Z","iopub.execute_input":"2023-11-18T01:40:41.774479Z","iopub.status.idle":"2023-11-18T01:40:41.82886Z","shell.execute_reply.started":"2023-11-18T01:40:41.774425Z","shell.execute_reply":"2023-11-18T01:40:41.827899Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Instantiate datasets","metadata":{}},{"cell_type":"code","source":"# For details of tile creation, please refer pjmathematician's notebook: \n# https://www.kaggle.com/code/pjmathematician/ucbo-tilemaker\n# For details of the split, please refer: \n# https://www.kaggle.com/code/emiz6413/stratified-shuffle-split/notebook\nmeta_df = pd.read_csv(\"/kaggle/input/save-stratified-split-to-csv/train_split.csv\")\ntrain_image_ids = meta_df[meta_df[\"is_train\"]][\"image_id\"].values\neval_image_ids = meta_df[~meta_df[\"is_train\"]][\"image_id\"].values\ntrain_image_dirs = [next(Path(\"/kaggle/input/\").glob(f\"ucbo-tiles-256-*/256_{img_id}\")) for img_id in train_image_ids]\neval_image_dirs = [next(Path(\"/kaggle/input/\").glob(f\"ucbo-tiles-256-*/256_{img_id}\")) for img_id in eval_image_ids]","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:40:41.829973Z","iopub.execute_input":"2023-11-18T01:40:41.830834Z","iopub.status.idle":"2023-11-18T01:40:43.488548Z","shell.execute_reply.started":"2023-11-18T01:40:41.83081Z","shell.execute_reply":"2023-11-18T01:40:43.487501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = UBCODataset(train_image_dirs, transforms=train_transforms)\neval_ds = UBCODataset(eval_image_dirs, transforms=eval_transforms)\n\nprint(\"before filtering\")\nprint(f\"len(train_ds): {len(train_ds)}\")\nprint(f\"len(eval_ds): {len(eval_ds)}\")\n\n# Some patches are completely black and white useless patches.\n# We will discard those patches by thresholding colorfulness of an image.\nwith open(\"/kaggle/input/make-image-colorfulness-json/colorfulness.json\", \"r\") as f:\n    colorfulness = json.load(f)\n\nvalid_train_idx = np.arange(len(train_ds))[np.array([colorfulness[str(i.stem)] for i in train_ds.image_paths]) > 5]\nvalid_eval_idx = np.arange(len(eval_ds))[np.array([colorfulness[str(i.stem)] for i in eval_ds.image_paths]) > 5]\ntrain_ds = Subset(train_ds, valid_train_idx)\neval_ds = Subset(eval_ds, valid_eval_idx)\n\nprint(\"after filtering\")\nprint(f\"len(train_ds): {len(train_ds)}\")\nprint(f\"len(eval_ds): {len(eval_ds)}\")\nwandb.log({\"train_size\": len(train_ds), \"eval_size\": len(eval_ds)})\n\ntrain_loader = DataLoader(dataset=train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=os.cpu_count())\neval_loader = DataLoader(dataset=eval_ds, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=os.cpu_count())","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:40:43.490223Z","iopub.execute_input":"2023-11-18T01:40:43.490896Z","iopub.status.idle":"2023-11-18T01:43:33.252954Z","shell.execute_reply.started":"2023-11-18T01:40:43.490861Z","shell.execute_reply":"2023-11-18T01:43:33.251882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualize the first few samples\nNote: Although entirely black and white images are filtered, crop of colorful image may be black and white","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2, figsize=(10, 5))\n\ntrain_imgs = next(iter(train_loader))[:64]\nval_imgs = next(iter(eval_loader))[:64]\n\nfor ax, img, title in zip(axes, [train_imgs, val_imgs], [\"train\", \"eval\"]):\n    ax.imshow(reverse_transform(make_grid(img, nrow=8)))\n    ax.axis(\"off\")\n    ax.set_title(title)\n\nfig.tight_layout()\nwandb.log({\"sample\": fig})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:43:33.254084Z","iopub.execute_input":"2023-11-18T01:43:33.254423Z","iopub.status.idle":"2023-11-18T01:44:08.529713Z","shell.execute_reply.started":"2023-11-18T01:43:33.254392Z","shell.execute_reply":"2023-11-18T01:44:08.528431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train","metadata":{}},{"cell_type":"code","source":"decoder = Decoder128()\nencoder = Encoder128()\n\nmodel = VanillaVAE(encoder=encoder, decoder=decoder, amp=AMP, device=DEVICE)\nmodel = model.to(DEVICE)\nif LOAD_WEIGHT is not None:\n    model.load_state_dict(torch.load(LOAD_WEIGHT))","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:44:08.531026Z","iopub.execute_input":"2023-11-18T01:44:08.531376Z","iopub.status.idle":"2023-11-18T01:44:11.520089Z","shell.execute_reply.started":"2023-11-18T01:44:08.531349Z","shell.execute_reply":"2023-11-18T01:44:11.519036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pbar = tqdm(total=LAST_EPOCH-INITIAL_EPOCH)\n\nfor epoch in range(INITIAL_EPOCH, LAST_EPOCH):\n    train_loss, train_rec_loss, train_kl_loss = model.train_single_epoch(data_loader=train_loader)\n\n    eval_loss, eval_rec_loss, eval_kl_loss = model.evaluate(data_loader=eval_loader)\n    pbar.set_description(f\"epoch: {epoch} train loss: {train_loss:.3f} eval loss: {eval_loss:.3f}\")\n    wandb.log(\n        {\n            \"train_loss\": train_loss,\n            \"train_rec_loss\": train_rec_loss,\n            \"train_kl_loss\": train_kl_loss,\n            \"eval_loss\": eval_loss,\n            \"eval_rec_loss\": eval_rec_loss,\n            \"eval_kl_loss\": eval_kl_loss,\n        },\n        step=epoch\n    )\n    \n    with torch.no_grad():\n        sample_img = next(iter(eval_loader))[:64]\n        rec_img, *_ = model(sample_img)\n\n    fig, axes = plt.subplots(1, 2)\n\n    for ax, img, title in zip(axes, [sample_img, rec_img], [\"original\", \"reconstructed\"]):\n        ax.imshow(reverse_transform(make_grid(img, nrow=8)))\n        ax.axis(\"off\")\n        ax.set_title(title)\n\n    fig.tight_layout()\n    fig.suptitle(f\"epoch: {epoch}\")\n    wandb.log({\"reconstruted_image\": fig}, step=epoch)\n    torch.save(model.state_dict(), f\"vae_ubco_epoch{epoch}_loss{eval_loss:.3f}.pth\")\n    \n    pbar.update()\n\npbar.close()","metadata":{"execution":{"iopub.status.busy":"2023-11-18T01:44:11.657785Z","iopub.execute_input":"2023-11-18T01:44:11.658508Z","iopub.status.idle":"2023-11-18T01:45:35.452786Z","shell.execute_reply.started":"2023-11-18T01:44:11.658461Z","shell.execute_reply":"2023-11-18T01:45:35.451148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}