{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8877088,"sourceType":"competition"},{"sourceId":8409068,"sourceType":"datasetVersion","datasetId":5004471}],"dockerImageVersionId":30734,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-22T00:38:42.858223Z","iopub.execute_input":"2024-06-22T00:38:42.858662Z","iopub.status.idle":"2024-06-22T00:38:42.87154Z","shell.execute_reply.started":"2024-06-22T00:38:42.858628Z","shell.execute_reply":"2024-06-22T00:38:42.870244Z"},"collapsed":true,"jupyter":{"outputs_hidden":true,"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport os\nimport numpy as np\n\n# Paths for TFRecord files\nTFRec_path = '/kaggle/input/leap-train-tfrecords'\ntrain_files = [os.path.join(TFRec_path, f\"train_{i:03d}.tfrec\") for i in range(2)]\nvalid_files = [os.path.join(TFRec_path, \"train_100.tfrec\")]\n\n# Feature description\nfeature_description = {\n    'x': tf.io.FixedLenFeature([556], tf.float32),\n    'targets': tf.io.FixedLenFeature([368], tf.float32),\n}\n\ndef _parse_function(example_proto):\n    proto = tf.io.parse_single_example(example_proto, feature_description)\n    return proto['x'], proto['targets']\n\ndef create_dataset(file_list):\n    dataset = tf.data.TFRecordDataset(file_list)\n    dataset = dataset.map(_parse_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n    return dataset\n\n# Create dataset without batching or shuffling\nfull_train_dataset = create_dataset(train_files)\n\n# Normalize the features\nnorm_x = tf.keras.layers.Normalization(axis=-1)\nnorm_x.adapt(full_train_dataset.map(lambda x, y: x))\n\n# Normalize the targets\nnorm_y = tf.keras.layers.Normalization(axis=-1)\nnorm_y.adapt(full_train_dataset.map(lambda x, y: y))\n\n# Compute and save the computed statistics for future use\nmean_x = norm_x.mean.numpy()\nstd_x = norm_x.variance.numpy()**0.5\nmean_targets = norm_y.mean.numpy()\nstd_targets = norm_y.variance.numpy()**0.5\n\nmean_std_file_path = '/kaggle/working/final_mean_std_stats.npz'\nnp.savez(mean_std_file_path, mean_x=mean_x, std_x=std_x, mean_targets=mean_targets, std_targets=std_targets)\nprint(\"Computed and saved statistics.\")\n","metadata":{"execution":{"iopub.status.busy":"2024-06-23T01:35:30.03273Z","iopub.execute_input":"2024-06-23T01:35:30.03316Z","iopub.status.idle":"2024-06-23T01:35:48.04854Z","shell.execute_reply.started":"2024-06-23T01:35:30.033125Z","shell.execute_reply":"2024-06-23T01:35:48.04714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}