{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":177694049,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Variational Autoencoder as a Dimension Reduction technique\n\nAlong with PCA the embedding of the initial features (vector representation in low dimensional space) could be used.\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nfrom pathlib import Path","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-20T11:49:19.938097Z","iopub.execute_input":"2024-05-20T11:49:19.939106Z","iopub.status.idle":"2024-05-20T11:49:33.602357Z","shell.execute_reply.started":"2024-05-20T11:49:19.939068Z","shell.execute_reply":"2024-05-20T11:49:33.601333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fix the seed","metadata":{}},{"cell_type":"code","source":"import os \nimport random\nimport numpy as np \n\nDEFAULT_RANDOM_SEED = 1000\n\nrandom.seed(DEFAULT_RANDOM_SEED)\nos.environ['PYTHONHASHSEED'] = str(DEFAULT_RANDOM_SEED)\nnp.random.seed(DEFAULT_RANDOM_SEED)\ntf.random.set_seed(DEFAULT_RANDOM_SEED)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:49:33.604524Z","iopub.execute_input":"2024-05-20T11:49:33.605584Z","iopub.status.idle":"2024-05-20T11:49:33.611735Z","shell.execute_reply.started":"2024-05-20T11:49:33.605543Z","shell.execute_reply":"2024-05-20T11:49:33.610539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading data\n\nTo train Variational Autoencoder the random subsample of the all data will be used. It's about 1M rows.\n","metadata":{}},{"cell_type":"code","source":"data_dir = Path(\"/kaggle/input/create-random-sample\")\n\ndf = pd.concat(\n    pd.read_parquet(parquet_file)\n    for i,parquet_file in enumerate(data_dir.glob('*.parquet'))\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:49:33.613281Z","iopub.execute_input":"2024-05-20T11:49:33.613717Z","iopub.status.idle":"2024-05-20T11:50:39.704854Z","shell.execute_reply.started":"2024-05-20T11:49:33.613669Z","shell.execute_reply":"2024-05-20T11:50:39.7038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [\n    \"state_t\",\n    \"state_q0001\",\n    \"state_q0002\",\n    \"state_q0003\",\n    \"state_u\",\n    \"state_v\",\n    \"state_ps\",\n    \"pbuf_SOLIN\",\n    \"pbuf_LHFLX\",\n    \"pbuf_SHFLX\",\n    \"pbuf_TAUX\",\n    \"pbuf_TAUY\",\n    \"pbuf_COSZRS\",\n    \"cam_in_ALDIF\",\n    \"cam_in_ALDIR\",\n    \"cam_in_ASDIF\",\n    \"cam_in_ASDIR\",\n    \"cam_in_LWUP\",\n    \"cam_in_ICEFRAC\",\n    \"cam_in_LANDFRAC\",\n    \"cam_in_OCNFRAC\",\n    \"cam_in_SNOWHLAND\",\n    \"pbuf_ozone\",\n#     \"pbuf_CH4\",\n#     \"pbuf_N2O\",\n]\n\ntargets = [\n    \"ptend_t\",\n    \"ptend_q0001\",\n    \"ptend_q0002\",\n    \"ptend_q0003\",\n    \"ptend_u\",\n    \"ptend_v\",\n    \"cam_out_NETSW\",\n    \"cam_out_FLWDS\",\n    \"cam_out_PRECSC\",\n    \"cam_out_PRECC\",\n    \"cam_out_SOLS\",\n    \"cam_out_SOLL\",\n    \"cam_out_SOLSD\",\n    \"cam_out_SOLLD\",\n]","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:39.707532Z","iopub.execute_input":"2024-05-20T11:50:39.707894Z","iopub.status.idle":"2024-05-20T11:50:39.715014Z","shell.execute_reply.started":"2024-05-20T11:50:39.707865Z","shell.execute_reply":"2024-05-20T11:50:39.713817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# collect all feature and target column names with indices\nall_targets = []\nfor t in targets:\n    all_targets += list(filter(lambda x: x.startswith(t), df.columns))\n\nall_targets = list(set(all_targets))\n\nall_features = []\nfor f in features:\n    all_features += list(filter(lambda x: x.startswith(f), df.columns))","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:39.716162Z","iopub.execute_input":"2024-05-20T11:50:39.716469Z","iopub.status.idle":"2024-05-20T11:50:39.736377Z","shell.execute_reply.started":"2024-05-20T11:50:39.716442Z","shell.execute_reply":"2024-05-20T11:50:39.735031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Variational Autoencoder \n\nRead more about Variational Autoencoder [here](https://keras.io/examples/generative/vae/).\n\nUsually the embedding approach is used to represent complex data such as pictures, text etc. In this notebook we will build shorter representation of longer vector.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras import ops\nfrom keras import layers\n\nclass Sampling(layers.Layer):\n    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        self.seed_generator = keras.random.SeedGenerator(1337)\n\n    def call(self, inputs):\n        z_mean, z_log_var = inputs\n        batch = ops.shape(z_mean)[0]\n        dim = ops.shape(z_mean)[1]\n        epsilon = keras.random.normal(shape=(batch, dim), seed=self.seed_generator)\n        return z_mean + ops.exp(0.5 * z_log_var) * epsilon\n\nclass VEncoder(keras.Model):\n    def __init__(self, input_dim: int, latent_dim:int, **kwargs):\n        super().__init__(**kwargs)\n\n        encoder_inputs = keras.Input(shape=(input_dim,))\n        x = layers.Dense(input_dim // 2, activation = \"silu\")(encoder_inputs)\n        x = layers.Dense(input_dim // 3, activation = \"silu\")(x)\n        z_mean = layers.Dense(latent_dim, name=\"z_mean\")(x)\n        z_log_var = layers.Dense(latent_dim, name=\"z_log_var\")(x)\n        z = Sampling()([z_mean, z_log_var])\n        self.encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name=\"encoder\")\n\n    def call(self, x):\n        return self.encoder(x)\n\nclass VDecoder(keras.Model):\n    def __init__(self, latent_dim:int, output_dim:int, **kwargs):\n        super().__init__(**kwargs)\n\n        latent_inputs = keras.Input(shape=(latent_dim,))\n        x = layers.Dense(output_dim // 4, activation = \"silu\")(latent_inputs)\n        x = layers.Dense(output_dim // 2, activation = \"silu\")(x)\n        decoder_outputs = layers.Dense(output_dim, activation=\"relu\")(x)\n        \n        self.decoder = keras.Model(latent_inputs, decoder_outputs, name=\"decoder\")\n\n    def call(self, x):\n        return self.decoder(x)\n\n\nclass VAE(keras.Model):\n    def __init__(self, encoder, decoder, **kwargs):\n        super().__init__(**kwargs)\n        self.encoder = encoder\n        self.decoder = decoder\n        \n        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n        self.reconstruction_loss_tracker = keras.metrics.Mean(\n            name=\"reconstruction_loss\"\n        )\n        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n    \n    def call(self, x):\n        z_mean, z_log_var, z = self.encoder(x)\n        reconstruction = self.decoder(z)\n        return z_mean, z_log_var, reconstruction\n\n    # also used during validation\n    @property\n    def metrics(self):\n        return [\n            self.total_loss_tracker,\n            self.reconstruction_loss_tracker,\n            self.kl_loss_tracker,\n        ]\n    \n    def calculate_reconstruction_loss(self, data, reconstruction):\n        \"\"\"\n        In case of computer vision tasks use the following:\n            keras.losses.binary_crossentropy(data, reconstruction),\n            axis=(1, 2),\n        \"\"\"\n        return ops.mean(ops.sum(keras.losses.mean_absolute_error(data, reconstruction)))\n    \n    def calculate_kl_loss(self, z_mean, z_log_var):\n        kl_loss = -0.5 * (1 + z_log_var - ops.square(z_mean) - ops.exp(z_log_var))\n        kl_loss = ops.mean(ops.sum(kl_loss, axis=1))\n        return kl_loss\n    \n    def calculate_total_loss(self, reconstruction_loss, kl_loss):\n        return reconstruction_loss + kl_loss * 3 # * (reconstruction_loss // kl_loss)\n        \n    def train_step(self, data):\n        with tf.GradientTape() as tape:\n            z_mean, z_log_var, z = self.encoder(data)\n            reconstruction = self.decoder(z)\n            \n            reconstruction_loss = self.calculate_reconstruction_loss(data, reconstruction)\n            kl_loss = self.calculate_kl_loss(z_mean, z_log_var)\n            total_loss = self.calculate_total_loss(reconstruction_loss, kl_loss)\n\n        grads = tape.gradient(total_loss, self.trainable_weights)\n        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n        \n        self.total_loss_tracker.update_state(total_loss)\n        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n        self.kl_loss_tracker.update_state(kl_loss)\n        \n        return {\n            \"loss\": self.total_loss_tracker.result(),\n            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n            \"kl_loss\": self.kl_loss_tracker.result(),\n        }\n    \n    def test_step(self, data):\n        z_mean, z_log_var, z = self.encoder(data, training=False)\n        reconstruction = self.decoder(z)\n        \n        reconstruction_loss = self.calculate_reconstruction_loss(data, reconstruction)\n        kl_loss = self.calculate_kl_loss(z_mean, z_log_var)\n        total_loss = self.calculate_total_loss(reconstruction_loss, kl_loss)\n        \n        self.total_loss_tracker.update_state(total_loss)\n        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n        self.kl_loss_tracker.update_state(kl_loss)\n        \n        return {\n            \"loss\": self.total_loss_tracker.result(),\n            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n            \"kl_loss\": self.kl_loss_tracker.result(),\n        }","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:39.738378Z","iopub.execute_input":"2024-05-20T11:50:39.738753Z","iopub.status.idle":"2024-05-20T11:50:39.764758Z","shell.execute_reply.started":"2024-05-20T11:50:39.738709Z","shell.execute_reply":"2024-05-20T11:50:39.763553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation\n\nBefore being used as an input to neural net the data should be scaled and aometimes normalized. Let's apply simplified min-max scaling and filter out outliers using z-score.","metadata":{}},{"cell_type":"code","source":"def scale_minmax(df: pd.DataFrame, mins: pd.Series = None, maxs: pd.Series = None) -> tuple:\n    mins = mins if mins else df.min() \n    maxs = maxs if maxs else df.max()\n\n    scaled = (df - mins) / (maxs - mins)\n    \n    return scaled, mins, maxs\n\ndef scale_minmax_inverse(df: pd.DataFrame, mins: pd.Series, maxs: pd.Series) -> pd.DataFrame:\n    return df * (maxs - mins) + mins\n\ndef normalize_zscore(df: pd.DataFrame, means: pd.Series=None, stds:pd.Series=None) -> tuple:\n    means = means if means else df.mean()\n    stds = stds if stds else df.std()\n    \n    normalized = (df - means) / stds\n    \n    return normalized, means, stds\n\ndef normalize_zscore_inverse(df: pd.DataFrame, means:pd.Series, maxs: pd.Series) -> pd.DataFrame:\n    return df * std + means","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:39.766094Z","iopub.execute_input":"2024-05-20T11:50:39.766462Z","iopub.status.idle":"2024-05-20T11:50:39.781451Z","shell.execute_reply.started":"2024-05-20T11:50:39.76643Z","shell.execute_reply":"2024-05-20T11:50:39.780447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaled, mins, maxs = scale_minmax(df[all_features])\nnormalized, means, stds = normalize_zscore(scaled)\n\nfiltered = scaled[(normalized.abs() < 4).all(axis=1)]","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:39.783025Z","iopub.execute_input":"2024-05-20T11:50:39.783349Z","iopub.status.idle":"2024-05-20T11:50:54.414802Z","shell.execute_reply.started":"2024-05-20T11:50:39.783323Z","shell.execute_reply":"2024-05-20T11:50:54.413125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shuffle data\nfiltered = filtered.sample(frac=1, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:54.416311Z","iopub.execute_input":"2024-05-20T11:50:54.41677Z","iopub.status.idle":"2024-05-20T11:50:56.990343Z","shell.execute_reply.started":"2024-05-20T11:50:54.41671Z","shell.execute_reply":"2024-05-20T11:50:56.989126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# observe data that actually comes to our model\nfiltered","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:56.995943Z","iopub.execute_input":"2024-05-20T11:50:56.996328Z","iopub.status.idle":"2024-05-20T11:50:57.118359Z","shell.execute_reply.started":"2024-05-20T11:50:56.996294Z","shell.execute_reply":"2024-05-20T11:50:57.117046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features\n\nAs an example we will use only subset of all available features.","metadata":{}},{"cell_type":"code","source":"states_t = list(filter(lambda x: x.startswith(\"state_t\"), all_features))","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:57.119704Z","iopub.execute_input":"2024-05-20T11:50:57.120115Z","iopub.status.idle":"2024-05-20T11:50:57.125482Z","shell.execute_reply.started":"2024-05-20T11:50:57.120083Z","shell.execute_reply":"2024-05-20T11:50:57.124431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train-val split\n\nSplit data using 80/20 rule","metadata":{}},{"cell_type":"code","source":"TRAIN_SIZE = int(filtered.shape[0] * 0.8)\nBATCH_SIZE = 1024","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:57.126875Z","iopub.execute_input":"2024-05-20T11:50:57.127257Z","iopub.status.idle":"2024-05-20T11:50:57.145925Z","shell.execute_reply.started":"2024-05-20T11:50:57.127227Z","shell.execute_reply":"2024-05-20T11:50:57.144468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building tf.data Dataset","metadata":{}},{"cell_type":"code","source":"# in the simple version of Autoencoder it is required to duplicate inputs as outputs\n\ntrain_sample = filtered[states_t][:TRAIN_SIZE]\ntest_sample = filtered[states_t][TRAIN_SIZE:]\n\ntrain_dataset = tf.data.Dataset.from_tensor_slices(train_sample).batch(BATCH_SIZE).prefetch(10).cache()\ntest_dataset = tf.data.Dataset.from_tensor_slices(test_sample).batch(BATCH_SIZE).prefetch(10).cache()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:50:57.148701Z","iopub.execute_input":"2024-05-20T11:50:57.149133Z","iopub.status.idle":"2024-05-20T11:50:57.896175Z","shell.execute_reply.started":"2024-05-20T11:50:57.149101Z","shell.execute_reply":"2024-05-20T11:50:57.89494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initializing the model\n\nLet's initialize model with default parameters.\n\nThe model will optimize mean absolute difference between input sequence and reconstructed one. We will also track percentage error MAPE as a main metric.","metadata":{}},{"cell_type":"code","source":"encoder = VEncoder(60, 16)\ndecoder = VDecoder(16, 60)\n\nvae = VAE(encoder, decoder)\n\noptimizer = tf.keras.optimizers.AdamW(5e-3)\nvae.compile(optimizer=optimizer)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T12:03:37.37468Z","iopub.execute_input":"2024-05-20T12:03:37.375497Z","iopub.status.idle":"2024-05-20T12:03:37.479644Z","shell.execute_reply.started":"2024-05-20T12:03:37.375462Z","shell.execute_reply":"2024-05-20T12:03:37.478357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model\n\nThe current light-weight Autoencoder implementation doesn't requre GPU to train.","metadata":{}},{"cell_type":"code","source":"history = vae.fit(\n    train_dataset,\n    epochs=250,\n    validation_data=test_dataset,\n    shuffle=True,\n    callbacks=[\n        tf.keras.callbacks.EarlyStopping(\n            monitor=\"val_total_loss\",\n            min_delta=0.0,\n            patience=50,\n            mode='min'\n        ),\n        keras.callbacks.ModelCheckpoint(\n            \"/kaggle/working/checkpoint.model.keras\",\n            monitor=\"val_total_loss\",\n            verbose=0,\n            save_best_only=True,\n            save_weights_only=False,\n            mode=\"min\",\n            save_freq=\"epoch\",\n            initial_value_threshold=None,\n        )\n    ],\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T12:03:37.867019Z","iopub.execute_input":"2024-05-20T12:03:37.867403Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Collect predictions on validation set","metadata":{}},{"cell_type":"code","source":"embeddings_val = vae.encoder(test_sample)[2]\npreds = vae(test_sample)[2]","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:54.005476Z","iopub.execute_input":"2024-05-20T11:57:54.005909Z","iopub.status.idle":"2024-05-20T11:57:54.449249Z","shell.execute_reply.started":"2024-05-20T11:57:54.00587Z","shell.execute_reply":"2024-05-20T11:57:54.448129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing the results","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objects as go","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:54.450518Z","iopub.execute_input":"2024-05-20T11:57:54.450889Z","iopub.status.idle":"2024-05-20T11:57:55.086659Z","shell.execute_reply.started":"2024-05-20T11:57:54.450858Z","shell.execute_reply":"2024-05-20T11:57:55.085564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Learning curve","metadata":{}},{"cell_type":"code","source":"history_to_vis = history.history.copy()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:55.087969Z","iopub.execute_input":"2024-05-20T11:57:55.088286Z","iopub.status.idle":"2024-05-20T11:57:55.093457Z","shell.execute_reply.started":"2024-05-20T11:57:55.08826Z","shell.execute_reply":"2024-05-20T11:57:55.092297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.line(history_to_vis, title='Learning curve')\nfig.update_xaxes(title={\"text\": \"Epoch\"})","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:55.095338Z","iopub.execute_input":"2024-05-20T11:57:55.095888Z","iopub.status.idle":"2024-05-20T11:57:57.123055Z","shell.execute_reply.started":"2024-05-20T11:57:55.095846Z","shell.execute_reply":"2024-05-20T11:57:57.121945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Embeddings","metadata":{}},{"cell_type":"code","source":"px.line(embeddings_val[1000:1020].numpy().T)","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:57.12642Z","iopub.execute_input":"2024-05-20T11:57:57.127388Z","iopub.status.idle":"2024-05-20T11:57:57.278259Z","shell.execute_reply.started":"2024-05-20T11:57:57.127341Z","shell.execute_reply":"2024-05-20T11:57:57.277133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing Ground truths against predictions","metadata":{}},{"cell_type":"markdown","source":"## Comparison graph","metadata":{}},{"cell_type":"code","source":"fig = go.Figure()\n\nbatch_number = 50\nitem_number = 100\n\nfig.add_trace(go.Scatter(y=preds[BATCH_SIZE * batch_number + item_number].numpy().T, mode='lines', name=\"pred\"))\nfig.add_trace(go.Scatter(y=test_sample.iloc[BATCH_SIZE * batch_number + item_number], mode='lines', name='true'))\nfig.update_layout(title=dict(text=\"Reconstructed input vs ground truth\"))\n\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:57.279622Z","iopub.execute_input":"2024-05-20T11:57:57.279972Z","iopub.status.idle":"2024-05-20T11:57:57.298652Z","shell.execute_reply.started":"2024-05-20T11:57:57.279941Z","shell.execute_reply":"2024-05-20T11:57:57.297498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Error analysis","metadata":{}},{"cell_type":"code","source":"fig = px.box((preds - test_sample)[:10000])\nfig.update_layout(title=dict(text=\"Errors distribution per feature\"))","metadata":{"execution":{"iopub.status.busy":"2024-05-20T11:57:57.299909Z","iopub.execute_input":"2024-05-20T11:57:57.300219Z","iopub.status.idle":"2024-05-20T11:57:58.812214Z","shell.execute_reply.started":"2024-05-20T11:57:57.300195Z","shell.execute_reply":"2024-05-20T11:57:58.810994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}