{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"},{"sourceId":8409068,"sourceType":"datasetVersion","datasetId":5004471},{"sourceId":8901446,"sourceType":"datasetVersion","datasetId":5351317},{"sourceId":8944739,"sourceType":"datasetVersion","datasetId":5378637},{"sourceId":8962332,"sourceType":"datasetVersion","datasetId":5386327},{"sourceId":187534996,"sourceType":"kernelVersion"},{"sourceId":188452444,"sourceType":"kernelVersion"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"The overall architecture is copied from the code in this Kaggle notebook.\nhttps://www.kaggle.com/code/abiolatti/keras-baseline-seq2seq?scriptVersionId=180403717\n\n\nThe model architecture is based on the paper discussed in this Kaggle discussion and is detailed in this arXiv paper.\n\nhttps://www.kaggle.com/competitions/leap-atmospheric-physics-ai-climsim/discussion/516605\nhttps://arxiv.org/abs/2407.00124\n\nI trained the model for 22 epochs. It achieved its highest performance at epoch 20 but showed signs of overfitting afterward.","metadata":{}},{"cell_type":"code","source":"import os\nos.environ[\"KERAS_BACKEND\"] = \"jax\"\n\nimport gc\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\n \nfrom keras.regularizers import l2\n\nimport tensorflow as tf\nimport jax\nimport keras\nfrom keras.layers import Conv1D, GroupNormalization, Activation\n\nfrom sklearn import metrics\n\nfrom tqdm.notebook import tqdm\n\nprint(tf.__version__)\nprint(jax.__version__)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:18.048739Z","iopub.execute_input":"2024-07-21T05:41:18.049052Z","iopub.status.idle":"2024-07-21T05:41:31.484358Z","shell.execute_reply.started":"2024-07-21T05:41:18.049021Z","shell.execute_reply":"2024-07-21T05:41:31.483379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def is_interactive():\n    return 'runtime' in get_ipython().config.IPKernelApp.connection_file\n\nprint('Interactive?', is_interactive())","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.486548Z","iopub.execute_input":"2024-07-21T05:41:31.487671Z","iopub.status.idle":"2024-07-21T05:41:31.49311Z","shell.execute_reply.started":"2024-07-21T05:41:31.487634Z","shell.execute_reply":"2024-07-21T05:41:31.492052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"isTrain = False\nSEED = 2024\nkeras.utils.set_random_seed(SEED)\ntf.random.set_seed(SEED)\ntf.config.experimental.enable_op_determinism()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.499242Z","iopub.execute_input":"2024-07-21T05:41:31.499864Z","iopub.status.idle":"2024-07-21T05:41:31.553568Z","shell.execute_reply.started":"2024-07-21T05:41:31.499828Z","shell.execute_reply":"2024-07-21T05:41:31.55255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA = \"/kaggle/input/leap-atmospheric-physics-ai-climsim\"\nDATA_TFREC = \"/kaggle/input/leap-train-tfrecords\"","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.554806Z","iopub.execute_input":"2024-07-21T05:41:31.555223Z","iopub.status.idle":"2024-07-21T05:41:31.562885Z","shell.execute_reply.started":"2024-07-21T05:41:31.555193Z","shell.execute_reply":"2024-07-21T05:41:31.561973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pl.read_csv(os.path.join(DATA, \"sample_submission.csv\"), n_rows=1)\nTARGETS = sample.select(pl.exclude('sample_id')).columns\nprint(len(TARGETS))","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.56388Z","iopub.execute_input":"2024-07-21T05:41:31.56416Z","iopub.status.idle":"2024-07-21T05:41:31.675313Z","shell.execute_reply.started":"2024-07-21T05:41:31.564137Z","shell.execute_reply":"2024-07-21T05:41:31.674272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _parse_function(example_proto):\n    feature_description = {\n        'x': tf.io.FixedLenFeature([556], tf.float32),\n        'targets': tf.io.FixedLenFeature([368], tf.float32)\n    }\n    e = tf.io.parse_single_example(example_proto, feature_description)\n    return e['x'], e['targets']","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.676595Z","iopub.execute_input":"2024-07-21T05:41:31.676975Z","iopub.status.idle":"2024-07-21T05:41:31.683716Z","shell.execute_reply.started":"2024-07-21T05:41:31.676941Z","shell.execute_reply":"2024-07-21T05:41:31.682125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if is_interactive():\n    train_files = [os.path.join(DATA_TFREC, \"train_%.3d.tfrec\" % i) for i in range(1)]\n    valid_files = [os.path.join(DATA_TFREC, \"train_%.3d.tfrec\" % i) for i in range(100, 101)]\nelse: \n    train_files = [os.path.join(DATA_TFREC, \"train_%.3d.tfrec\" % i) for i in range(100)]\n    valid_files = [os.path.join(DATA_TFREC, \"train_%.3d.tfrec\" % i) for i in range(100, 101)]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.685183Z","iopub.execute_input":"2024-07-21T05:41:31.685695Z","iopub.status.idle":"2024-07-21T05:41:31.693089Z","shell.execute_reply.started":"2024-07-21T05:41:31.685661Z","shell.execute_reply":"2024-07-21T05:41:31.69218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 512\n\ntrain_options = tf.data.Options()\ntrain_options.deterministic = True\n\nds_train = (\n    tf.data.Dataset.from_tensor_slices(train_files)\n    .with_options(train_options)\n    .shuffle(100)\n    .interleave(\n        lambda file: tf.data.TFRecordDataset(file).map(_parse_function, num_parallel_calls=tf.data.AUTOTUNE),\n        num_parallel_calls=tf.data.AUTOTUNE,\n        cycle_length=10,\n        block_length=1000,\n        deterministic=True\n    )\n    .shuffle(4 * BATCH_SIZE)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.AUTOTUNE)\n)\n\nds_valid = (\n    tf.data.TFRecordDataset(valid_files)\n    .map(_parse_function)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.AUTOTUNE)\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:31.694159Z","iopub.execute_input":"2024-07-21T05:41:31.694424Z","iopub.status.idle":"2024-07-21T05:41:33.765791Z","shell.execute_reply.started":"2024-07-21T05:41:31.694399Z","shell.execute_reply":"2024-07-21T05:41:33.764784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_x = keras.layers.Normalization()\nnorm_x.adapt(ds_train.map(lambda x, y: x).take(20 if is_interactive() else 10000))\n\nplt.scatter(\n    norm_x.mean.squeeze(),\n    norm_x.variance.squeeze() ** 0.5,\n    marker=\".\",\n    alpha=0.5\n)\nplt.xscale('log')\nplt.yscale('log')","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:33.768657Z","iopub.execute_input":"2024-07-21T05:41:33.768963Z","iopub.status.idle":"2024-07-21T05:41:37.70785Z","shell.execute_reply.started":"2024-07-21T05:41:33.768936Z","shell.execute_reply":"2024-07-21T05:41:37.706978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_y = keras.layers.Normalization()\nnorm_y.adapt(ds_train.map(lambda x, y: y).take(20 if is_interactive() else 10000))\n\nmean_y = norm_y.mean\nstdd_y = keras.ops.maximum(1e-10, norm_y.variance ** 0.5)\n\nplt.scatter(\n    mean_y.squeeze(),\n    stdd_y.squeeze(),\n    marker=\".\",\n    alpha=0.5\n)\nplt.xscale('log')\nplt.yscale('log')\n","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:37.70922Z","iopub.execute_input":"2024-07-21T05:41:37.710118Z","iopub.status.idle":"2024-07-21T05:41:39.513209Z","shell.execute_reply.started":"2024-07-21T05:41:37.71006Z","shell.execute_reply":"2024-07-21T05:41:39.512047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"min_y = np.min(np.stack([np.min(yb, 0) for _, yb in ds_train.take(20 if is_interactive() else 10000)], 0), 0, keepdims=True)\nmax_y = np.max(np.stack([np.max(yb, 0) for _, yb in ds_train.take(20 if is_interactive() else 10000)], 0), 0, keepdims=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:39.514559Z","iopub.execute_input":"2024-07-21T05:41:39.514904Z","iopub.status.idle":"2024-07-21T05:41:40.615197Z","shell.execute_reply.started":"2024-07-21T05:41:39.514873Z","shell.execute_reply":"2024-07-21T05:41:40.614066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model definition & Training","metadata":{}},{"cell_type":"code","source":"if is_interactive(): \n    epochs = 1 # \nelse:  \n    epochs = 11 # 25  # 15  # 12 \n    \nlearning_rate = 1e-3\nearly_patience = 5\n# epochs_warmup = 1\n# epochs_ending = 2\n# steps_per_epoch = int(np.ceil(len(train_files) * 100_000 / BATCH_SIZE))\n\n# lr_scheduler = keras.optimizers.schedules.CosineDecay(\n#     1e-4, \n#     (epochs - epochs_warmup - epochs_ending) * steps_per_epoch, \n#     warmup_target=learning_rate,\n#     warmup_steps=steps_per_epoch * epochs_warmup,\n#     alpha=0.1\n# )\n\n# plt.plot([lr_scheduler(it) for it in range(0, epochs * steps_per_epoch, steps_per_epoch)]);","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:40.616415Z","iopub.execute_input":"2024-07-21T05:41:40.616726Z","iopub.status.idle":"2024-07-21T05:41:40.622131Z","shell.execute_reply.started":"2024-07-21T05:41:40.616702Z","shell.execute_reply":"2024-07-21T05:41:40.621067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import LayerNormalization, MultiHeadAttention, Dense, Dropout\n\n@keras.saving.register_keras_serializable()\nclass TransformerEncoderLayer(tf.keras.layers.Layer):\n    def __init__(self, head_size, num_heads, ff_dim, dropout=0.1, **kwargs):\n        super(TransformerEncoderLayer, self).__init__(**kwargs)\n        self.att = MultiHeadAttention(key_dim=head_size, num_heads=num_heads, dropout=dropout)\n        self.ffn = tf.keras.Sequential([\n            Dense(ff_dim, activation='gelu'),\n            Dense(25)\n        ])\n        self.layernorm1 = LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = LayerNormalization(epsilon=1e-6)\n        self.dropout1 = Dropout(dropout)\n        self.dropout2 = Dropout(dropout)\n\n    def call(self, inputs, training=False):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output) ","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:40.623198Z","iopub.execute_input":"2024-07-21T05:41:40.623734Z","iopub.status.idle":"2024-07-21T05:41:40.638251Z","shell.execute_reply.started":"2024-07-21T05:41:40.623708Z","shell.execute_reply":"2024-07-21T05:41:40.637248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transformer block\n@keras.saving.register_keras_serializable()\ndef transformer_block(inputs, head_size, num_heads, ff_dim, dropout=0.2):\n    attention_output = tf.keras.layers.MultiHeadAttention(\n        key_dim=head_size, num_heads=num_heads, dropout=dropout)(inputs, inputs)\n    attention_output = tf.keras.layers.Dropout(dropout)(attention_output)\n    attention_output = tf.keras.layers.LayerNormalization(epsilon=1e-6)(attention_output + inputs)\n\n    ff_output = tf.keras.layers.Dense(ff_dim, activation='gelu')(attention_output)\n    ff_output = tf.keras.layers.Dropout(dropout)(ff_output)\n    ff_output = tf.keras.layers.Dense(inputs.shape[-1])(ff_output)\n    ff_output = tf.keras.layers.LayerNormalization(epsilon=1e-6)(ff_output + attention_output)\n\n    return ff_output\n\n# ResBlock function\n@keras.saving.register_keras_serializable()\ndef res_block(x, filters, output_filters=None, groups=8):\n    if output_filters is None:\n        output_filters = filters\n    norm1 = GroupNormalization(groups=groups, axis=-1)(x)\n    silu1 = tf.keras.layers.Activation('swish')(norm1)\n    conv1 = tf.keras.layers.Conv1D(filters, kernel_size=3, padding='same')(silu1)\n\n    norm2 = GroupNormalization(groups=groups, axis=-1)(conv1)\n    silu2 = tf.keras.layers.Activation('swish')(norm2)\n    conv2 = tf.keras.layers.Conv1D(output_filters, kernel_size=3, padding='same')(silu2)\n\n    if x.shape[-1] != conv2.shape[-1]:\n        x = tf.keras.layers.Conv1D(output_filters, kernel_size=1, padding='same')(x)\n    output = tf.keras.layers.Add()([conv2, x])\n    return output\n\n# Downsample block\n@keras.saving.register_keras_serializable()\ndef repeat_block(x, filters, repeat):\n    for _ in range(repeat):\n        x = res_block(x, filters) \n    return x\n\n# Upsample block\n@keras.saving.register_keras_serializable()\ndef upsample_block(x, filters, repeat, concat_layer):\n    x = tf.keras.layers.Conv1DTranspose(filters, kernel_size=2, strides=2, padding='same')(x)\n    x = tf.keras.layers.Concatenate()([x, concat_layer])\n    for _ in range(repeat):\n        x = res_block(x, filters)\n    return x\n\n# Define a custom Lambda function to print and remove specific data slices\n# TPU does not support printing in this manner, for compatibility, direct value retrieval is used instead.\n# This function was previously used but later removed. Hence, there might be inconsistencies.\n@keras.saving.register_keras_serializable()\ndef slice_and_print(x):\n    # Print the first 2 time steps that are being removed\n    tf.print(\"Removed start:\", x[:, :2, :], summarize=-1)\n    # Print the last 2 time steps that are being removed\n    tf.print(\"Removed end:\", x[:, -2:, :], summarize=-1)\n    # Return the data after removing the padding from the start and end\n    return x[:, 2:-2, :] ","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:40.639508Z","iopub.execute_input":"2024-07-21T05:41:40.639779Z","iopub.status.idle":"2024-07-21T05:41:40.65835Z","shell.execute_reply.started":"2024-07-21T05:41:40.639757Z","shell.execute_reply":"2024-07-21T05:41:40.657327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# public score 0.68647\nmodel_filepath = '/kaggle/input/leap-u-net-models/model_epoch_07.keras'\n\n# Check if the model file exists\nif os.path.exists(model_filepath):\n    print(f\"Loading model from {model_filepath}\")\n    keras.utils.clear_session()\n    custom_objects = {\n        'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n    }\n    model = keras.models.load_model(model_filepath, custom_objects)\nelse:\n    print(\"Saved model not found, training a new model\")\n    # Clear the current Keras session\n    keras.utils.clear_session() \n\n    def x_to_seq(x):\n        x_seq0 = keras.ops.transpose(keras.ops.reshape(x[:, 0:60 * 6], (-1, 6, 60)), (0, 2, 1))\n        x_seq1 = keras.ops.transpose(keras.ops.reshape(x[:, 60 * 6 + 16:60 * 9 + 16], (-1, 3, 60)), (0, 2, 1))\n        x_flat = keras.ops.reshape(x[:, 60 * 6:60 * 6 + 16], (-1, 1, 16))\n        x_flat = keras.ops.repeat(x_flat, 60, axis=1)\n        return keras.ops.concatenate([x_seq0, x_seq1, x_flat], axis=-1) \n  \n   \n    # Build 1D U-Net model\n    def create_unet(input_shape):\n        inputs = keras.layers.Input(shape=input_shape)\n\n        # Encoder\n        encoder_1 = repeat_block(inputs, 128, 2)  # 64 x 128 x 2\n        encoder_1_down = keras.layers.MaxPooling1D(pool_size=2, strides=2)(encoder_1)  # Downsample # 32 x 128\n\n        encoder_2 = repeat_block(encoder_1_down, 256, 2)  # 32 x 256 x 2\n        encoder_2_down = keras.layers.MaxPooling1D(pool_size=2, strides=2)(encoder_2)  # Downsample # 16 x 256\n\n        encoder_3 = repeat_block(encoder_2_down, 256, 2)  # 16 x 256 x 2\n        encoder_3_down = keras.layers.MaxPooling1D(pool_size=2, strides=2)(encoder_3)  # Downsample # 8 x 256\n\n        encoder_4 = repeat_block(encoder_3_down, 256, 2)  # 8 x 256 x 2\n\n        # Bottleneck (Transformer)\n        bottleneck = transformer_block(encoder_4, head_size=4, num_heads=64, ff_dim=512)\n        \n        decoder_1 = keras.layers.Concatenate()([bottleneck, encoder_4])\n        decoder_1_block = repeat_block(decoder_1, 256, 3)  # 8 x 256 x 3\n        decoder_1_upsample = keras.layers.Conv1DTranspose(256, kernel_size=2, strides=2, padding='same')(decoder_1_block)  # Upsample # 16 x 256\n\n        decoder_2 = keras.layers.Concatenate()([decoder_1_upsample, encoder_3])\n        decoder_2_block = repeat_block(decoder_2, 256, 3)  # 16 x 256 x 3\n        decoder_2_upsample = keras.layers.Conv1DTranspose(256, kernel_size=2, strides=2, padding='same')(decoder_2_block)  # Upsample # 32 x 256\n\n        decoder_3 = keras.layers.Concatenate()([decoder_2_upsample, encoder_2])\n        decoder_3_block = repeat_block(decoder_3, 256, 3)  # 32 x 256 x 3\n        decoder_3_upsample = keras.layers.Conv1DTranspose(256, kernel_size=2, strides=2, padding='same')(decoder_3_block)  # Upsample # 64 x 256\n\n        decoder_4 = keras.layers.Concatenate()([decoder_3_upsample, encoder_1])\n        decoder_4_block = repeat_block(decoder_4, 256, 3)  # 64 x 256 x 3\n\n        model = keras.models.Model(inputs, decoder_4_block)\n        return model\n    \n    X_input = x = keras.layers.Input(ds_train.element_spec[0].shape[1:]) \n    x = keras.layers.Normalization(mean=norm_x.mean, variance=norm_x.variance)(x)\n    x = x_to_seq(x) \n  \n    # Zero-padding at the beginning and end of the sequence to extend the length from 60 to 64\n    x = keras.layers.ZeroPadding1D(padding=(2, 2))(x)\n    # \n    e = keras.layers.Conv1D(48, 1, padding='same')(x)   \n    e = create_unet(e.shape[1:])(e)      \n      \n    # Use a Lambda layer to remove the first and last 2 time steps \n    e = e[:, 2:-2, :]\n\n    p_all = keras.layers.Conv1D(14, 1, padding='same')(e)\n    print(p_all.shape)\n    \n    p_seq = p_all[:, :, :6]\n    p_seq = keras.ops.transpose(p_seq, (0, 2, 1))\n    p_seq = keras.layers.Flatten()(p_seq)\n    assert p_seq.shape[-1] == 360\n\n    p_flat = p_all[:, :, 6:6 + 8]\n    p_flat = keras.ops.mean(p_flat, axis=1)\n    assert p_flat.shape[-1] == 8\n\n    P = keras.ops.concatenate([p_seq, p_flat], axis=1)\n\n    # Build & compile the model\n    model = keras.Model(X_input, P)\n    model.compile(\n        loss='mse', \n        optimizer=keras.optimizers.Adam(0.0002),\n        metrics=[keras.metrics.MeanSquaredError(), \n                     keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"), \n        ]  # Updated R2Score\n    )\n    model.build(tuple(ds_train.element_spec[0].shape))\n    model.summary()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-07-21T05:41:40.65958Z","iopub.execute_input":"2024-07-21T05:41:40.659985Z","iopub.status.idle":"2024-07-21T05:41:52.81165Z","shell.execute_reply.started":"2024-07-21T05:41:40.65996Z","shell.execute_reply":"2024-07-21T05:41:52.810672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model2**","metadata":{}},{"cell_type":"code","source":"# public score 0.68202\nmodel_filepath2 = '/kaggle/input/leap-k07092221-u-net-e50/model_epoch_12.keras'\n\nprint(f\"Loading model from {model_filepath}\")\nkeras.utils.clear_session()\ncustom_objects = {\n    'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n}\nmodel2 = keras.models.load_model(model_filepath2, custom_objects)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:41:52.813125Z","iopub.execute_input":"2024-07-21T05:41:52.813676Z","iopub.status.idle":"2024-07-21T05:42:00.908504Z","shell.execute_reply.started":"2024-07-21T05:41:52.813635Z","shell.execute_reply":"2024-07-21T05:42:00.907563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model3**","metadata":{}},{"cell_type":"code","source":"# public score 0.69257\nmodel_filepath3 = '/kaggle/input/als-b-130-unet-512/model_epoch_12.keras'\n\nprint(f\"Loading model from {model_filepath3}\")\nkeras.utils.clear_session()\ncustom_objects = {\n    'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n}\nmodel3 = keras.models.load_model(model_filepath3, custom_objects)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:42:41.055915Z","iopub.execute_input":"2024-07-21T05:42:41.056845Z","iopub.status.idle":"2024-07-21T05:43:12.488524Z","shell.execute_reply.started":"2024-07-21T05:42:41.05681Z","shell.execute_reply":"2024-07-21T05:43:12.487663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model3.1**","metadata":{}},{"cell_type":"code","source":"# ???\nmodel_filepath31 = '/kaggle/input/als-b-130-unet-512/model_epoch_13.keras'\n\nprint(f\"Loading model from {model_filepath31}\")\nkeras.utils.clear_session()\ncustom_objects = {\n    'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n}\nmodel31 = keras.models.load_model(model_filepath31, custom_objects)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:43:12.490528Z","iopub.execute_input":"2024-07-21T05:43:12.490993Z","iopub.status.idle":"2024-07-21T05:43:41.330187Z","shell.execute_reply.started":"2024-07-21T05:43:12.490958Z","shell.execute_reply":"2024-07-21T05:43:41.329332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model3.2**","metadata":{}},{"cell_type":"code","source":"# ???\nmodel_filepath32 = '/kaggle/input/als-b-130-unet-512/model_epoch_11.keras'\n\nprint(f\"Loading model from {model_filepath32}\")\nkeras.utils.clear_session()\ncustom_objects = {\n    'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n}\nmodel32 = keras.models.load_model(model_filepath32, custom_objects)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:43:41.331557Z","iopub.execute_input":"2024-07-21T05:43:41.331839Z","iopub.status.idle":"2024-07-21T05:44:10.366875Z","shell.execute_reply.started":"2024-07-21T05:43:41.331816Z","shell.execute_reply":"2024-07-21T05:44:10.366032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model4**","metadata":{}},{"cell_type":"code","source":"# public score 0.68037\nmodel_filepath4 = '/kaggle/input/als-b-113-unet-pre-interactivef-bs2048-only-e16/model_epoch_29.keras'\n\nprint(f\"Loading model from {model_filepath4}\")\nkeras.utils.clear_session()\ncustom_objects = {\n    'R2Score': tf.keras.metrics.R2Score(class_aggregation=\"variance_weighted_average\"),\n}\nmodel4 = keras.models.load_model(model_filepath4, custom_objects)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:44:10.369184Z","iopub.execute_input":"2024-07-21T05:44:10.369488Z","iopub.status.idle":"2024-07-21T05:44:18.488222Z","shell.execute_reply.started":"2024-07-21T05:44:10.369463Z","shell.execute_reply":"2024-07-21T05:44:18.487329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model5**","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# Normalize target values for training and validation datasets\nds_train_target_normalized = ds_train.map(lambda x, y: (x, (y - mean_y) / stdd_y))\nds_valid_target_normalized = ds_valid.map(lambda x, y: (x, (y - mean_y) / stdd_y))\n\nif isTrain:\n    from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping \n\n    # Model checkpoint callback\n    model_checkpoint = ModelCheckpoint(\n        filepath='model_epoch_{epoch:02d}.keras',  # Save model with epoch number\n        monitor='val_loss',  # Monitor validation loss\n        save_best_only=False,  # Save all models, not just the best one\n        save_weights_only=False,  # Save the entire model structure and weights\n        mode='min',  # 'min' indicates saving when the monitored value decreases\n        verbose=2  # Provide detailed logging\n    )\n\n    # Early stopping callback\n    early_stopping = EarlyStopping(\n        monitor='val_loss',  # Monitor validation loss\n        patience=early_patience,  # Number of epochs to wait before stopping\n        restore_best_weights=True  # Restore model weights from the epoch with the best value of the monitored quantity\n    )\n\n    # Train the model\n    history = model.fit(\n        ds_train_target_normalized,  # Training data\n        validation_data=ds_valid_target_normalized,  # Validation data\n        epochs=epochs,  # Number of epochs to train\n        verbose=1 if is_interactive() else 2,  # Verbose output\n        callbacks=[early_stopping, model_checkpoint]  # List of callbacks to apply during training\n    )\n","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-07-21T05:44:44.059453Z","iopub.execute_input":"2024-07-21T05:44:44.059843Z","iopub.status.idle":"2024-07-21T05:44:44.096265Z","shell.execute_reply.started":"2024-07-21T05:44:44.059813Z","shell.execute_reply":"2024-07-21T05:44:44.095316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if isTrain:\n    plt.plot(history.history['loss'], color='tab:blue')\n    plt.plot(history.history['val_loss'], color='tab:red')\n    plt.yscale('log');","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:44:48.651993Z","iopub.execute_input":"2024-07-21T05:44:48.652741Z","iopub.status.idle":"2024-07-21T05:44:48.658598Z","shell.execute_reply.started":"2024-07-21T05:44:48.652688Z","shell.execute_reply":"2024-07-21T05:44:48.657491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_valid = np.concatenate([yb for _, yb in ds_valid])\np_valid = model.predict(ds_valid, batch_size=BATCH_SIZE) * stdd_y + mean_y","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:44:51.109196Z","iopub.execute_input":"2024-07-21T05:44:51.110171Z","iopub.status.idle":"2024-07-21T05:45:15.616428Z","shell.execute_reply.started":"2024-07-21T05:44:51.110128Z","shell.execute_reply":"2024-07-21T05:45:15.615429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_valid = np.array([metrics.r2_score(y_valid[:, i], p_valid[:, i]) for i in range(len(TARGETS))])\nplt.plot(scores_valid.clip(-1, 1))","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:15.618707Z","iopub.execute_input":"2024-07-21T05:45:15.619456Z","iopub.status.idle":"2024-07-21T05:45:16.917588Z","shell.execute_reply.started":"2024-07-21T05:45:15.619414Z","shell.execute_reply":"2024-07-21T05:45:16.916678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask = scores_valid <= 1e-3\nf\"Number of under-performing targets: {sum(mask)}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:16.919018Z","iopub.execute_input":"2024-07-21T05:45:16.919453Z","iopub.status.idle":"2024-07-21T05:45:16.926012Z","shell.execute_reply.started":"2024-07-21T05:45:16.91942Z","shell.execute_reply":"2024-07-21T05:45:16.925048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f\"Clipped score: {scores_valid.clip(0, 1).mean()}\"","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:40.324193Z","iopub.execute_input":"2024-07-21T05:45:40.325271Z","iopub.status.idle":"2024-07-21T05:45:40.331125Z","shell.execute_reply.started":"2024-07-21T05:45:40.325235Z","shell.execute_reply":"2024-07-21T05:45:40.330202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del y_valid, p_valid\ngc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:41.734485Z","iopub.execute_input":"2024-07-21T05:45:41.734841Z","iopub.status.idle":"2024-07-21T05:45:42.026586Z","shell.execute_reply.started":"2024-07-21T05:45:41.734815Z","shell.execute_reply":"2024-07-21T05:45:42.025254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# **Submission**\n---","metadata":{}},{"cell_type":"code","source":"sample = pl.read_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:43.72974Z","iopub.execute_input":"2024-07-21T05:45:43.730594Z","iopub.status.idle":"2024-07-21T05:45:46.894003Z","shell.execute_reply.started":"2024-07-21T05:45:43.73056Z","shell.execute_reply":"2024-07-21T05:45:46.893148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = (\n    pl.scan_csv(\"/kaggle/input/leap-atmospheric-physics-ai-climsim/test.csv\")\n    .select(pl.exclude(\"sample_id\"))\n    .collect()\n)","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:45:50.466517Z","iopub.execute_input":"2024-07-21T05:45:50.46688Z","iopub.status.idle":"2024-07-21T05:46:18.666419Z","shell.execute_reply.started":"2024-07-21T05:45:50.466853Z","shell.execute_reply":"2024-07-21T05:46:18.665504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect();","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:46:18.668295Z","iopub.execute_input":"2024-07-21T05:46:18.668758Z","iopub.status.idle":"2024-07-21T05:46:18.969927Z","shell.execute_reply.started":"2024-07-21T05:46:18.668721Z","shell.execute_reply":"2024-07-21T05:46:18.968817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Pred Model1**","metadata":{}},{"cell_type":"code","source":"%%time\np_test = model.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test = np.array(p_test)\np_test[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:46:18.971542Z","iopub.execute_input":"2024-07-21T05:46:18.97214Z","iopub.status.idle":"2024-07-21T05:47:25.661808Z","shell.execute_reply.started":"2024-07-21T05:46:18.972044Z","shell.execute_reply":"2024-07-21T05:47:25.660794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test = pd.DataFrame(p_test, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test = df_p_test.values\ndel p_test; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:47:25.664396Z","iopub.execute_input":"2024-07-21T05:47:25.6647Z","iopub.status.idle":"2024-07-21T05:47:27.504383Z","shell.execute_reply.started":"2024-07-21T05:47:25.664673Z","shell.execute_reply":"2024-07-21T05:47:27.503265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel2**","metadata":{}},{"cell_type":"code","source":"%%time\np_test2 = model2.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test2 = np.array(p_test2)\np_test2[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:47:27.505815Z","iopub.execute_input":"2024-07-21T05:47:27.506224Z","iopub.status.idle":"2024-07-21T05:48:23.432248Z","shell.execute_reply.started":"2024-07-21T05:47:27.506186Z","shell.execute_reply":"2024-07-21T05:48:23.431121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test2 = pd.DataFrame(p_test2, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test2[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test2 = df_p_test2.values\ndel p_test2; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:48:23.433655Z","iopub.execute_input":"2024-07-21T05:48:23.434062Z","iopub.status.idle":"2024-07-21T05:48:25.186222Z","shell.execute_reply.started":"2024-07-21T05:48:23.434031Z","shell.execute_reply":"2024-07-21T05:48:25.185138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel3**","metadata":{}},{"cell_type":"code","source":"%%time\np_test3 = model3.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test3 = np.array(p_test3)\np_test3[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:48:25.187673Z","iopub.execute_input":"2024-07-21T05:48:25.188444Z","iopub.status.idle":"2024-07-21T05:51:16.406423Z","shell.execute_reply.started":"2024-07-21T05:48:25.188404Z","shell.execute_reply":"2024-07-21T05:51:16.405361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test3 = pd.DataFrame(p_test3, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test3[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test3 = df_p_test3.values\ndel p_test3; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:51:16.407981Z","iopub.execute_input":"2024-07-21T05:51:16.408369Z","iopub.status.idle":"2024-07-21T05:51:18.207184Z","shell.execute_reply.started":"2024-07-21T05:51:16.408302Z","shell.execute_reply":"2024-07-21T05:51:18.206299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel3.1**","metadata":{}},{"cell_type":"code","source":"%%time\np_test31 = model31.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test31 = np.array(p_test31)\np_test31[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:51:18.208512Z","iopub.execute_input":"2024-07-21T05:51:18.208839Z","iopub.status.idle":"2024-07-21T05:53:51.610426Z","shell.execute_reply.started":"2024-07-21T05:51:18.208811Z","shell.execute_reply":"2024-07-21T05:53:51.609383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test31 = pd.DataFrame(p_test31, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test31[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test31 = df_p_test31.values\ndel p_test31; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:53:51.614557Z","iopub.execute_input":"2024-07-21T05:53:51.61489Z","iopub.status.idle":"2024-07-21T05:53:53.34375Z","shell.execute_reply.started":"2024-07-21T05:53:51.614862Z","shell.execute_reply":"2024-07-21T05:53:53.342644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel3.2**","metadata":{}},{"cell_type":"code","source":"%%time\np_test32 = model32.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test32 = np.array(p_test32)\np_test32[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:53:53.344948Z","iopub.execute_input":"2024-07-21T05:53:53.345267Z","iopub.status.idle":"2024-07-21T05:56:26.767501Z","shell.execute_reply.started":"2024-07-21T05:53:53.34524Z","shell.execute_reply":"2024-07-21T05:56:26.766476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test32 = pd.DataFrame(p_test32, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test32[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test32 = df_p_test32.values\ndel p_test32; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:56:26.769321Z","iopub.execute_input":"2024-07-21T05:56:26.769719Z","iopub.status.idle":"2024-07-21T05:56:28.507674Z","shell.execute_reply.started":"2024-07-21T05:56:26.769683Z","shell.execute_reply":"2024-07-21T05:56:28.506765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel4**","metadata":{}},{"cell_type":"code","source":"%%time\np_test4 = model4.predict(df_test.to_numpy(), batch_size=4 * BATCH_SIZE) * stdd_y + mean_y\np_test4 = np.array(p_test4)\np_test4[:, mask] = mean_y[:, mask]","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:56:28.50904Z","iopub.execute_input":"2024-07-21T05:56:28.509363Z","iopub.status.idle":"2024-07-21T05:57:26.15046Z","shell.execute_reply.started":"2024-07-21T05:56:28.509337Z","shell.execute_reply":"2024-07-21T05:57:26.149335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correction of ptend_q0002 targets (from 12 to 29)\ndf_p_test4 = pd.DataFrame(p_test4, columns=TARGETS)\n\nfor idx in range(12, 29):\n    df_p_test4[f\"ptend_q0002_{idx}\"] = -df_test[f\"state_q0002_{idx}\"].to_numpy() / 1200\n    \np_test4 = df_p_test4.values\ndel p_test4; gc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:57:26.152182Z","iopub.execute_input":"2024-07-21T05:57:26.152602Z","iopub.status.idle":"2024-07-21T05:57:28.132851Z","shell.execute_reply.started":"2024-07-21T05:57:26.152552Z","shell.execute_reply":"2024-07-21T05:57:28.131975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **PredModel5**","metadata":{}},{"cell_type":"markdown","source":"---\n### **FinalyCreateSub**\n---","metadata":{}},{"cell_type":"code","source":"p_test_sub = ((df_p_test+df_p_test2+df_p_test3+df_p_test31+df_p_test32+df_p_test4)/6).values","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:58:44.83545Z","iopub.execute_input":"2024-07-21T05:58:44.835863Z","iopub.status.idle":"2024-07-21T05:58:48.941655Z","shell.execute_reply.started":"2024-07-21T05:58:44.835831Z","shell.execute_reply":"2024-07-21T05:58:48.940478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = sample.to_pandas()\nsubmission[TARGETS] = submission[TARGETS] * p_test_sub\npl.from_pandas(submission[[\"sample_id\"] + TARGETS]).write_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-21T05:58:48.943417Z","iopub.execute_input":"2024-07-21T05:58:48.943777Z","iopub.status.idle":"2024-07-21T05:59:05.47512Z","shell.execute_reply.started":"2024-07-21T05:58:48.943746Z","shell.execute_reply":"2024-07-21T05:59:05.474018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}