{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":56537,"databundleVersionId":8015876,"sourceType":"competition"},{"sourceId":180258298,"sourceType":"kernelVersion"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pickle\nfrom huggingface_hub import hf_hub_download\nimport os\nimport tensorflow as tf","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-14T21:31:26.888315Z","iopub.execute_input":"2024-06-14T21:31:26.888722Z","iopub.status.idle":"2024-06-14T21:31:44.920675Z","shell.execute_reply.started":"2024-06-14T21:31:26.888689Z","shell.execute_reply":"2024-06-14T21:31:44.919084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_cols = np.load('/kaggle/input/leap-download-data-1/all_data_cols.npy')\nall_data_cols_not = np.load('/kaggle/input/leap-download-data-1/all_data_cols_not.npy')\nall_data_cols_not_target = np.load('/kaggle/input/leap-download-data-1/all_data_cols_not_target.npy')\nall_data_cols_targets = np.load('/kaggle/input/leap-download-data-1/all_data_cols_targets.npy')","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:31:44.922875Z","iopub.execute_input":"2024-06-14T21:31:44.923715Z","iopub.status.idle":"2024-06-14T21:33:55.080057Z","shell.execute_reply.started":"2024-06-14T21:31:44.923673Z","shell.execute_reply":"2024-06-14T21:33:55.077583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"times = pickle.load(open('/kaggle/input/leap-download-data-1/times.p', 'br'))","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:38:07.628421Z","iopub.execute_input":"2024-06-14T21:38:07.628868Z","iopub.status.idle":"2024-06-14T21:38:07.637544Z","shell.execute_reply.started":"2024-06-14T21:38:07.628834Z","shell.execute_reply":"2024-06-14T21:38:07.635986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"year = [int(x[:4]) for x in times]\nmonth = [int(x[5:7]) for x in times]\nday = [int(x[8:10]) for x in times]\nsecs = [int(x[11:]) for x in times]","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.101269Z","iopub.execute_input":"2024-06-14T21:33:55.101794Z","iopub.status.idle":"2024-06-14T21:33:55.121624Z","shell.execute_reply.started":"2024-06-14T21:33:55.101751Z","shell.execute_reply":"2024-06-14T21:33:55.120223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"times_stacked = np.stack([year, month, day, secs], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.123288Z","iopub.execute_input":"2024-06-14T21:33:55.12378Z","iopub.status.idle":"2024-06-14T21:33:55.14045Z","shell.execute_reply.started":"2024-06-14T21:33:55.123746Z","shell.execute_reply":"2024-06-14T21:33:55.138996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(all_data_cols)/384)\nprint(len(year))","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.142406Z","iopub.execute_input":"2024-06-14T21:33:55.142897Z","iopub.status.idle":"2024-06-14T21:33:55.158564Z","shell.execute_reply.started":"2024-06-14T21:33:55.142862Z","shell.execute_reply":"2024-06-14T21:33:55.156974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"samples_num = len(all_data_cols)\nprint(samples_num)\n'''\nrand_seed = np.random.randint(10000)\nprint(rand_seed)\n'''\nrand_seed = 5412\nrng = np.random.default_rng(rand_seed)\nrandomize_order_indices = rng.choice(range(samples_num), size=samples_num, replace=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:36:18.754406Z","iopub.execute_input":"2024-06-14T21:36:18.754857Z","iopub.status.idle":"2024-06-14T21:36:19.324234Z","shell.execute_reply.started":"2024-06-14T21:36:18.754821Z","shell.execute_reply":"2024-06-14T21:36:19.322341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = randomize_order_indices\ndata_chunked = []\nchunk_size = 5000\nfor i in range(len(data)//chunk_size+1):\n    data_chunked.append(data[i*chunk_size:(i+1)*chunk_size])\n\nprint(len(data))\nprint(np.sum([len(x) for x in data_chunked]))\n\nif not os.path.isdir(\"tfds\"): os.mkdir(\"tfds\")\ntffile_names = [f\"tfds/{file_id}.tfrecord\" for file_id in range(len(data_chunked))]","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.759913Z","iopub.execute_input":"2024-06-14T21:33:55.760365Z","iopub.status.idle":"2024-06-14T21:33:55.770041Z","shell.execute_reply.started":"2024-06-14T21:33:55.760331Z","shell.execute_reply":"2024-06-14T21:33:55.768698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_to_tfrecord(file_name, randomize_order_indices, batch_index):\n    with tf.io.TFRecordWriter(file_name, 'GZIP') as file_writer:\n        for index in range(len(randomize_order_indices)):\n            random_index = randomize_order_indices[index]\n\n            data_cols = all_data_cols[random_index]\n            data_cols_not = all_data_cols_not[random_index]\n            data_cols_targets = all_data_cols_targets[random_index]\n            data_cols_not_target = all_data_cols_not_target[random_index]\n            time = times_stacked[random_index//384]\n            gidx = random_index%384\n            \n            data_cols = tf.io.serialize_tensor(data_cols).numpy()\n            data_cols_not = tf.io.serialize_tensor(data_cols_not).numpy()\n            data_cols_targets = tf.io.serialize_tensor(data_cols_targets).numpy()\n            data_cols_not_target = tf.io.serialize_tensor(data_cols_not_target).numpy()\n\n            features = {}\n            features['x1'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols]))\n            features['x2'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_not]))\n            features['x3'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_targets]))\n            features['x4'] = tf.train.Feature(bytes_list=tf.train.BytesList(value=[data_cols_not_target]))\n            features['time'] = tf.train.Feature(int64_list=tf.train.Int64List(value=time))\n            features['gidx'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[-gidx-1]))\n            features['idx'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[random_index]))\n            features['batch_index'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[batch_index]))\n            record_bytes = tf.train.Example(features=tf.train.Features(feature=features)).SerializeToString()\n            file_writer.write(record_bytes)\n            \n        print(file_name)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.772518Z","iopub.execute_input":"2024-06-14T21:33:55.772911Z","iopub.status.idle":"2024-06-14T21:33:55.789472Z","shell.execute_reply.started":"2024-06-14T21:33:55.772877Z","shell.execute_reply":"2024-06-14T21:33:55.788053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(data_chunked))","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.795004Z","iopub.execute_input":"2024-06-14T21:33:55.795411Z","iopub.status.idle":"2024-06-14T21:33:55.809797Z","shell.execute_reply.started":"2024-06-14T21:33:55.795381Z","shell.execute_reply":"2024-06-14T21:33:55.808278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tffiles_len = [len(x) for x in data_chunked]\npickle.dump(tffiles_len, open(f'tffiles_len.p', 'bw'))","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.811512Z","iopub.execute_input":"2024-06-14T21:33:55.812271Z","iopub.status.idle":"2024-06-14T21:33:55.822217Z","shell.execute_reply.started":"2024-06-14T21:33:55.8122Z","shell.execute_reply":"2024-06-14T21:33:55.820342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nfor i in range(len(data_chunked)):\n    save_to_tfrecord(tffile_names[i], data_chunked[i], i)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:33:55.824668Z","iopub.execute_input":"2024-06-14T21:33:55.825685Z","iopub.status.idle":"2024-06-14T21:34:31.55999Z","shell.execute_reply.started":"2024-06-14T21:33:55.825623Z","shell.execute_reply":"2024-06-14T21:34:31.558442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def decode_tfrec(record_bytes):\n    schema = {}\n    schema[\"x1\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x2\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x3\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"x4\"] = tf.io.VarLenFeature(dtype=tf.string)\n    schema[\"time\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"gidx\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"idx\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    schema[\"batch_index\"] = tf.io.VarLenFeature(dtype=tf.int64)\n    features = tf.io.parse_single_example(record_bytes, schema)\n\n    data_cols = tf.sparse.to_dense(features[\"x1\"])\n    data_cols = tf.io.parse_tensor(data_cols[0], out_type=tf.float64)\n    data_cols_not = tf.sparse.to_dense(features[\"x2\"])\n    data_cols_not = tf.io.parse_tensor(data_cols_not[0], out_type=tf.float64)\n    data_cols_targets = tf.sparse.to_dense(features[\"x3\"])\n    data_cols_targets = tf.io.parse_tensor(data_cols_targets[0], out_type=tf.float64)\n    data_cols_not_target = tf.sparse.to_dense(features[\"x4\"])\n    data_cols_not_target = tf.io.parse_tensor(data_cols_not_target[0], out_type=tf.float64)\n\n    time = tf.sparse.to_dense(features[\"time\"])\n    gidx = tf.sparse.to_dense(features[\"gidx\"])\n    idx = tf.sparse.to_dense(features[\"idx\"])\n    batch_index = tf.sparse.to_dense(features[\"batch_index\"])\n    \n    out = {}\n    out['data_cols']  = data_cols\n    out['data_cols_not']  = data_cols_not\n    out['data_cols_targets']  = data_cols_targets\n    out['data_cols_not_target']  = data_cols_not_target\n    out['time']  = time\n    out['gidx']  = gidx\n    out['idx']  = idx\n    out['batch_index']  = batch_index\n    return out","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:34:31.561769Z","iopub.execute_input":"2024-06-14T21:34:31.562187Z","iopub.status.idle":"2024-06-14T21:34:31.57794Z","shell.execute_reply.started":"2024-06-14T21:34:31.562139Z","shell.execute_reply":"2024-06-14T21:34:31.576704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nds = tf.data.TFRecordDataset(\n    ['/kaggle/working/tfds/0.tfrecord'], num_parallel_reads=tf.data.AUTOTUNE, compression_type = 'GZIP').prefetch(tf.data.AUTOTUNE)\nds = ds.map(decode_tfrec, tf.data.AUTOTUNE)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:34:31.579734Z","iopub.execute_input":"2024-06-14T21:34:31.580201Z","iopub.status.idle":"2024-06-14T21:34:32.41835Z","shell.execute_reply.started":"2024-06-14T21:34:31.580134Z","shell.execute_reply":"2024-06-14T21:34:32.417039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlaa = [x for x in ds.take(5)]","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:38:36.657842Z","iopub.execute_input":"2024-06-14T21:38:36.658315Z","iopub.status.idle":"2024-06-14T21:38:36.711435Z","shell.execute_reply.started":"2024-06-14T21:38:36.658278Z","shell.execute_reply":"2024-06-14T21:38:36.710132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(np.mean(laa[0]['data_cols'] == all_data_cols[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_not'] ==  all_data_cols_not[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_targets'] ==  all_data_cols_targets[data_chunked[0][0]]))\nprint(np.mean(laa[0]['data_cols_not_target'] ==  all_data_cols_not_target[data_chunked[0][0]]))\nprint(np.mean(laa[0]['time'] == times_stacked[data_chunked[0][0]//384]))\nprint(np.mean((-laa[0]['gidx']-1) == (data_chunked[0][0])%384))\nprint(np.mean(laa[0]['idx'] == data_chunked[0][0]))\nprint(np.mean(laa[0]['batch_index'] == 0))","metadata":{"execution":{"iopub.status.busy":"2024-06-14T21:36:05.637438Z","iopub.execute_input":"2024-06-14T21:36:05.637916Z","iopub.status.idle":"2024-06-14T21:36:05.660787Z","shell.execute_reply.started":"2024-06-14T21:36:05.637885Z","shell.execute_reply":"2024-06-14T21:36:05.659506Z"},"trusted":true},"execution_count":null,"outputs":[]}]}