{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":180609034,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nfrom huggingface_hub import hf_hub_download\nimport os\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-29T21:49:16.327549Z","iopub.execute_input":"2024-05-29T21:49:16.327934Z","iopub.status.idle":"2024-05-29T21:49:31.478632Z","shell.execute_reply.started":"2024-05-29T21:49:16.327901Z","shell.execute_reply":"2024-05-29T21:49:31.477271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '/kaggle/input/leap-download-data-14'\nall_data_cols = np.load(f'{path}/all_data_cols.npy')\nall_data_cols_not = np.load(f'{path}/all_data_cols_not.npy')\nall_data_cols_not_target = np.load(f'{path}/all_data_cols_not_target.npy')\nall_data_cols_targets = np.load(f'{path}/all_data_cols_targets.npy')\ntimes = pickle.load(open(f'{path}/times.p', 'br'))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:49:31.480922Z","iopub.execute_input":"2024-05-29T21:49:31.48159Z","iopub.status.idle":"2024-05-29T21:51:55.748196Z","shell.execute_reply.started":"2024-05-29T21:49:31.481554Z","shell.execute_reply":"2024-05-29T21:51:55.746895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"year = [int(x[:4]) for x in times]\nmonth = [int(x[5:7]) for x in times]\nday = [int(x[8:10]) for x in times]\nsecs = [int(x[11:]) for x in times]","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:55.764086Z","iopub.execute_input":"2024-05-29T21:51:55.764436Z","iopub.status.idle":"2024-05-29T21:51:55.795565Z","shell.execute_reply.started":"2024-05-29T21:51:55.764407Z","shell.execute_reply":"2024-05-29T21:51:55.778698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"times_stacked = np.stack([year, month, day, secs], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:55.797624Z","iopub.execute_input":"2024-05-29T21:51:55.798562Z","iopub.status.idle":"2024-05-29T21:51:55.806972Z","shell.execute_reply.started":"2024-05-29T21:51:55.798529Z","shell.execute_reply":"2024-05-29T21:51:55.805477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(all_data_cols)/384)\nprint(len(year))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:55.808493Z","iopub.execute_input":"2024-05-29T21:51:55.809012Z","iopub.status.idle":"2024-05-29T21:51:55.817818Z","shell.execute_reply.started":"2024-05-29T21:51:55.808976Z","shell.execute_reply":"2024-05-29T21:51:55.816488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_num = len(all_data_cols)\nprint(samples_num)\nrand_seed = np.random.randint(10000)\nprint(rand_seed)\nrng = np.random.default_rng(rand_seed)\nrandomize_order_indices = rng.choice(range(samples_num), size=samples_num, replace=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:55.819591Z","iopub.execute_input":"2024-05-29T21:51:55.81994Z","iopub.status.idle":"2024-05-29T21:51:56.387373Z","shell.execute_reply.started":"2024-05-29T21:51:55.81991Z","shell.execute_reply":"2024-05-29T21:51:56.386223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = randomize_order_indices\ndata_chunked = []\nchunk_size = 5000\nfor i in range(len(data)//chunk_size+1):\n    data_chunked.append(data[i*chunk_size:(i+1)*chunk_size])\n\nprint(len(data))\nprint(np.sum([len(x) for x in data_chunked]))\n\nif not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\ntffile_names = [f\"tfds/{file_id}.tfrecord\" for file_id in range(len(data_chunked))]","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:56.388979Z","iopub.execute_input":"2024-05-29T21:51:56.389515Z","iopub.status.idle":"2024-05-29T21:51:56.399345Z","shell.execute_reply.started":"2024-05-29T21:51:56.389474Z","shell.execute_reply":"2024-05-29T21:51:56.398073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_to_tfrecord(file_name, randomize_order_indices, batch_index):\n    with tf.io.TFRecordWriter(file_name, 'GZIP') as file_writer:\n        for index in range(len(randomize_order_indices)):\n            random_index = randomize_order_indices[index]\n\n            data_cols = all_data_cols[random_index]\n            data_cols_not = all_data_cols_not[random_index]\n            data_cols_targets = all_data_cols_targets[random_index]\n            data_cols_not_target = all_data_cols_not_target[random_index]\n            time = times_stacked[random_index//384]\n            gidx = random_index%384\n            \n            data_cols = tf.io.serialize_tensor(data_cols).numpy()\n            data_cols_not = tf.io.serialize_tensor(data_cols_not).numpy()\n            data_cols_targets = tf.io.serialize_tensor(data_cols_targets).numpy()\n            data_cols_not_target = tf.io.serialize_tensor(data_cols_not_target).numpy()\n\n            features = {}\n            features['x1'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols]))\n            features['x2'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_not]))\n            features['x3'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_targets]))\n            features['x4'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_not_target]))\n            features['time'] = tf.train.Feature(int64_list=tf.train.Int64List(value=time))\n            features['gidx'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[-gidx-1]))\n            features['idx'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[random_index]))\n            features['batch_index'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[batch_index]))\n            record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n            file_writer.write(record_bytes)\n            \n        print(file_name)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:56.400889Z","iopub.execute_input":"2024-05-29T21:51:56.401298Z","iopub.status.idle":"2024-05-29T21:51:56.420004Z","shell.execute_reply.started":"2024-05-29T21:51:56.401264Z","shell.execute_reply":"2024-05-29T21:51:56.418754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(data_chunked))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:56.423821Z","iopub.execute_input":"2024-05-29T21:51:56.424222Z","iopub.status.idle":"2024-05-29T21:51:56.435116Z","shell.execute_reply.started":"2024-05-29T21:51:56.424188Z","shell.execute_reply":"2024-05-29T21:51:56.433962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tffiles_len = [len(x) for x in data_chunked]\npickle.dump(tffiles_len, open(f'tffiles_len.p', 'bw'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor i in range(len(data_chunked)):\n    save_to_tfrecord(tffile_names[i], data_chunked[i], i)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:51:56.436466Z","iopub.execute_input":"2024-05-29T21:51:56.43685Z","iopub.status.idle":"2024-05-29T21:55:51.857261Z","shell.execute_reply.started":"2024-05-29T21:51:56.436816Z","shell.execute_reply":"2024-05-29T21:55:51.856072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrec(record_bytes):\n    schema = {}\n    schema[\"x1\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x2\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x3\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x4\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"time\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"gidx\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"idx\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"batch_index\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    features = tf.io.parse_single_example(record_bytes, schema)\n\n    data_cols = tf.sparse.to_dense(features[\"x1\"])\n    data_cols = tf.io.parse_tensor(data_cols[0], out_type=tf.float64)\n    data_cols_not = tf.sparse.to_dense(features[\"x2\"])\n    data_cols_not = tf.io.parse_tensor(data_cols_not[0], out_type=tf.float64)\n    data_cols_targets = tf.sparse.to_dense(features[\"x3\"])\n    data_cols_targets = tf.io.parse_tensor(data_cols_targets[0], out_type=tf.float64)\n    data_cols_not_target = tf.sparse.to_dense(features[\"x4\"])\n    data_cols_not_target = tf.io.parse_tensor(data_cols_not_target[0], out_type=tf.float64)\n\n    time = tf.sparse.to_dense(features[\"time\"])\n    gidx = tf.sparse.to_dense(features[\"gidx\"])\n    idx = tf.sparse.to_dense(features[\"idx\"])\n    batch_index = tf.sparse.to_dense(features[\"batch_index\"])\n    \n    out = {}\n    out['data_cols']  = data_cols\n    out['data_cols_not']  = data_cols_not\n    out['data_cols_targets']  = data_cols_targets\n    out['data_cols_not_target']  = data_cols_not_target\n    out['time']  = time\n    out['gidx']  = gidx\n    out['idx']  = idx\n    out['batch_index']  = batch_index\n    return out","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:55:54.684599Z","iopub.execute_input":"2024-05-29T21:55:54.685049Z","iopub.status.idle":"2024-05-29T21:55:54.701521Z","shell.execute_reply.started":"2024-05-29T21:55:54.684996Z","shell.execute_reply":"2024-05-29T21:55:54.700122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nds = tf.data.TFRecordDataset(\n    ['/kaggle/working/tfds/0.tfrecord'], num_parallel_reads=tf.data.AUTOTUNE, compression_type = 'GZIP').prefetch(tf.data.AUTOTUNE)\nds = ds.map(decode_tfrec, tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:55:57.684638Z","iopub.execute_input":"2024-05-29T21:55:57.685079Z","iopub.status.idle":"2024-05-29T21:55:58.089291Z","shell.execute_reply.started":"2024-05-29T21:55:57.685026Z","shell.execute_reply":"2024-05-29T21:55:58.088102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlaa = [x for x in ds.take(5)]","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:55:59.453041Z","iopub.execute_input":"2024-05-29T21:55:59.45414Z","iopub.status.idle":"2024-05-29T21:55:59.511953Z","shell.execute_reply.started":"2024-05-29T21:55:59.454087Z","shell.execute_reply":"2024-05-29T21:55:59.510749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.mean(laa[0]['data_cols'] == all_data_cols[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_not'] ==  all_data_cols_not[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_targets'] ==  all_data_cols_targets[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_not_target'] ==  all_data_cols_not_target[data_chunked[0][0]]))\nprint(np.mean(laa[0]['time'] == times_stacked[data_chunked[0][0]//384]))\nprint(np.mean(laa[0]['gidx'] == (data_chunked[0][0])%384))\nprint(np.mean(laa[0]['idx'] == data_chunked[0][0]))\nprint(np.mean(laa[0]['batch_index'] == 0))","metadata":{"execution":{"iopub.status.busy":"2024-05-29T21:56:01.381162Z","iopub.execute_input":"2024-05-29T21:56:01.381681Z","iopub.status.idle":"2024-05-29T21:56:01.397156Z","shell.execute_reply.started":"2024-05-29T21:56:01.381641Z","shell.execute_reply":"2024-05-29T21:56:01.39587Z"},"trusted":true},"execution_count":null,"outputs":[]}]}