{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"START = 0\nCOUNT = 1000\nSHARD_SIZE_MAX = 128\nDATA_PATH = '/kaggle/input/google-research-identify-contrails-reduce-global-warming'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-27T05:43:12.615665Z","iopub.execute_input":"2023-06-27T05:43:12.616082Z","iopub.status.idle":"2023-06-27T05:43:12.628365Z","shell.execute_reply.started":"2023-06-27T05:43:12.616045Z","shell.execute_reply":"2023-06-27T05:43:12.626987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport cv2\nimport json\nimport glob\nimport zipfile\nimport threading\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:12.635231Z","iopub.execute_input":"2023-06-27T05:43:12.635699Z","iopub.status.idle":"2023-06-27T05:43:16.002435Z","shell.execute_reply.started":"2023-06-27T05:43:12.635658Z","shell.execute_reply":"2023-06-27T05:43:16.000839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_fs = sorted(glob.glob(DATA_PATH + '/train/*'))\nvalid_fs = sorted(glob.glob(DATA_PATH + '/validation/*'))\n# len(train_fs, valid_fs)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.003956Z","iopub.execute_input":"2023-06-27T05:43:16.005413Z","iopub.status.idle":"2023-06-27T05:43:16.118098Z","shell.execute_reply.started":"2023-06-27T05:43:16.005331Z","shell.execute_reply":"2023-06-27T05:43:16.116839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load json file\nwith open('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json', 'r') as f:\n    train_json = json.load(f)\n\nwith open('/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json', 'r') as f:\n    valid_json = json.load(f)\ndef convert_json(js):\n    result = []\n    for record in js:\n        # print(record)\n        projection_wkt = record['projection_wkt']\n        s = projection_wkt.index('central_meridian')\n        str = projection_wkt[s:].split(',')[1]\n        central_meridian = float(re.findall(r'-?\\d+', str)[0])\n        result.append([record['record_id'], central_meridian, record['row_min'], record['row_size'], record['col_min'], record['col_size'], record['timestamp']])\n\n    result = pd.DataFrame(result, columns=['record_id', 'central_meridian', 'row_min', 'row_size', 'col_min', 'col_size', 'timestamp'])\n    result.set_index('record_id', inplace=True)\n        # break\n    return result\ntrain_df = convert_json(train_json)\nvalid_df = convert_json(valid_json)\n\ndf = pd.concat([train_df, valid_df], axis=0)\ndes = df.describe()\ndes","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.122307Z","iopub.execute_input":"2023-06-27T05:43:16.12273Z","iopub.status.idle":"2023-06-27T05:43:16.570725Z","shell.execute_reply.started":"2023-06-27T05:43:16.122696Z","shell.execute_reply":"2023-06-27T05:43:16.569175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_meta(path):\n    tp = path.split('/')[-2]\n    record_id = path.split('/')[-1]\n    if tp == 'train':\n        rs = train_df.loc[record_id]\n    else:\n        rs = valid_df.loc[record_id]\n    return rs\n\nget_meta(train_fs[0])","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.573489Z","iopub.execute_input":"2023-06-27T05:43:16.573969Z","iopub.status.idle":"2023-06-27T05:43:16.591056Z","shell.execute_reply.started":"2023-06-27T05:43:16.573926Z","shell.execute_reply":"2023-06-27T05:43:16.589574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef to_ash(band11, band14, band15):\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    return false_color","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.593248Z","iopub.execute_input":"2023-06-27T05:43:16.59365Z","iopub.status.idle":"2023-06-27T05:43:16.609588Z","shell.execute_reply.started":"2023-06-27T05:43:16.593613Z","shell.execute_reply":"2023-06-27T05:43:16.607629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_band(path):\n    with open(f'{path}/band_11.npy', 'rb') as f:\n        band11 = np.load(f)\n    with open(f'{path}/band_14.npy', 'rb') as f:\n        band14 = np.load(f)\n    with open(f'{path}/band_15.npy', 'rb') as f:\n        band15 = np.load(f)\n    ash = to_ash(band11, band14, band15) * 255\n    ash =  np.transpose(ash, [3, 0, 1, 2])\n    ash = ash.astype(np.uint8)\n    return ash","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.611281Z","iopub.execute_input":"2023-06-27T05:43:16.612188Z","iopub.status.idle":"2023-06-27T05:43:16.626521Z","shell.execute_reply.started":"2023-06-27T05:43:16.612139Z","shell.execute_reply":"2023-06-27T05:43:16.62551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ash = load_band(train_fs[0])\nplt.imshow(ash[4])","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:16.628002Z","iopub.execute_input":"2023-06-27T05:43:16.628559Z","iopub.status.idle":"2023-06-27T05:43:17.073679Z","shell.execute_reply.started":"2023-06-27T05:43:16.628522Z","shell.execute_reply":"2023-06-27T05:43:17.072464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_mask(path):\n    with open(f'{path}/human_pixel_masks.npy', 'rb') as f:\n        mask = np.load(f)\n    mask = mask.astype(np.uint8)\n    \n    if 'train' in path:\n        with open(f'{path}/human_individual_masks.npy', 'rb') as f:\n            mt = np.load(f)\n        mt = mt.astype(np.uint8)\n        mt = np.sum(mt, axis=3)\n        mask = mask + mt * 2\n        mask = mask.astype(np.uint8)\n    return mask","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:44:15.712299Z","iopub.execute_input":"2023-06-27T05:44:15.712793Z","iopub.status.idle":"2023-06-27T05:44:15.72086Z","shell.execute_reply.started":"2023-06-27T05:44:15.712758Z","shell.execute_reply":"2023-06-27T05:44:15.719529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = load_mask(train_fs[6])\nnp.max(m)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:45:47.52656Z","iopub.execute_input":"2023-06-27T05:45:47.526958Z","iopub.status.idle":"2023-06-27T05:45:47.540101Z","shell.execute_reply.started":"2023-06-27T05:45:47.526925Z","shell.execute_reply":"2023-06-27T05:45:47.5388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(15,5))\nax[0].imshow(m // 2)\nax[1].imshow(tf.bitwise.bitwise_and(m, 1))","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:45:17.360799Z","iopub.execute_input":"2023-06-27T05:45:17.361209Z","iopub.status.idle":"2023-06-27T05:45:17.7842Z","shell.execute_reply.started":"2023-06-27T05:45:17.361177Z","shell.execute_reply":"2023-06-27T05:45:17.782748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m = load_mask(valid_fs[6])\nnp.max(m)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:45:55.629316Z","iopub.execute_input":"2023-06-27T05:45:55.629739Z","iopub.status.idle":"2023-06-27T05:45:55.641621Z","shell.execute_reply.started":"2023-06-27T05:45:55.629706Z","shell.execute_reply":"2023-06-27T05:45:55.640312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"b = load_band(valid_fs[0])\nnp.shape(b)","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.612564Z","iopub.status.idle":"2023-06-27T05:43:17.613664Z","shell.execute_reply.started":"2023-06-27T05:43:17.613397Z","shell.execute_reply":"2023-06-27T05:43:17.613428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(9):\n#     plt.imshow(b[i][:,:,0])\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.615044Z","iopub.status.idle":"2023-06-27T05:43:17.616029Z","shell.execute_reply.started":"2023-06-27T05:43:17.615787Z","shell.execute_reply":"2023-06-27T05:43:17.615814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _bytes_feature(value):\n    \"\"\"Returns a bytes_list from a string / byte.\"\"\"\n    if isinstance(value, type(tf.constant(0))):\n        value = value.numpy() # BytesList won't unpack a string from an EagerTensor.\n    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))\n\ndef _float_feature(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))\n\ndef _float_features(value):\n    \"\"\"Returns a float_list from a float / double.\"\"\"\n    return tf.train.Feature(float_list=tf.train.FloatList(value=value))\n\ndef _int64_feature(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))  \n\ndef _int64_features(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))  \n\ndef to_tfrecord(ash, mask, meta):\n    feature = {\n        \"ash\": _bytes_feature(ash),\n        \"mask\": _bytes_feature(mask),\n        \"central_meridian\": _float_feature(meta['central_meridian']),\n        \"row_min\": _float_feature(meta['row_min']),\n        \"row_size\": _float_feature(meta['row_size']),\n        \"col_min\": _float_feature(meta['col_min']),\n        \"col_size\": _float_feature(meta['col_size']),\n        \"timestamp\": _float_feature(meta['timestamp']),\n    }\n    return tf.train.Example(features=tf.train.Features(feature=feature))","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.617804Z","iopub.status.idle":"2023-06-27T05:43:17.619267Z","shell.execute_reply.started":"2023-06-27T05:43:17.618725Z","shell.execute_reply":"2023-06-27T05:43:17.618788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = 'ds/'\n!rm -r {path}\n!mkdir {path}\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.62204Z","iopub.status.idle":"2023-06-27T05:43:17.622659Z","shell.execute_reply.started":"2023-06-27T05:43:17.622331Z","shell.execute_reply":"2023-06-27T05:43:17.622371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with zipfile.ZipFile(f'{path}/train.zip', 'w', compression=zipfile.ZIP_DEFLATED) as zf:\n    out_file = None\n    for i, f in enumerate(train_fs):\n        if i % SHARD_SIZE_MAX == 0:\n            if out_file is not None:\n                out_file.close()\n                zf.write(filename, arcname=filename)\n#                 !ls -al\n                !rm {filename}\n            filename = \"train_{:04d}.tfrec\".format(i // SHARD_SIZE_MAX)\n            print(filename)\n            out_file = tf.io.TFRecordWriter(filename)\n        ash = load_band(f)\n        ash = tf.io.encode_png(ash[4])\n        mask = load_mask(f)\n        mask = tf.io.encode_png(mask)\n        meta = get_meta(f)\n        example = to_tfrecord(ash, mask, meta)\n        out_file.write(example.SerializeToString())\n        \n    out_file.close()\n    zf.write(filename, arcname=filename)\n    !rm {filename}","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.626996Z","iopub.status.idle":"2023-06-27T05:43:17.628237Z","shell.execute_reply.started":"2023-06-27T05:43:17.627932Z","shell.execute_reply":"2023-06-27T05:43:17.627961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with zipfile.ZipFile(f'{path}/valid.zip', 'w', compression=zipfile.ZIP_DEFLATED) as zf:\n    out_file = None\n    for i, f in enumerate(valid_fs):\n        if i % SHARD_SIZE_MAX == 0:\n            if out_file is not None:\n                out_file.close()\n                zf.write(filename, arcname=filename)\n#                 !ls -al\n                !rm {filename}\n            filename = \"valid_{:04d}.tfrec\".format(i // SHARD_SIZE_MAX)\n            print(filename)\n            out_file = tf.io.TFRecordWriter(filename)\n        ash = load_band(f)\n        ash = tf.io.encode_png(ash[4])\n        mask = load_mask(f)\n        mask = tf.io.encode_png(mask)\n        meta = get_meta(f)\n        example = to_tfrecord(ash, mask, meta)\n        out_file.write(example.SerializeToString())\n        \n    out_file.close()\n    zf.write(filename, arcname=filename)\n    !rm {filename}","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.630084Z","iopub.status.idle":"2023-06-27T05:43:17.630706Z","shell.execute_reply.started":"2023-06-27T05:43:17.630411Z","shell.execute_reply":"2023-06-27T05:43:17.63044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !kaggle datasets create -r zip -p ds","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.63256Z","iopub.status.idle":"2023-06-27T05:43:17.633703Z","shell.execute_reply.started":"2023-06-27T05:43:17.633461Z","shell.execute_reply":"2023-06-27T05:43:17.633485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !kaggle datasets version -r zip -p ds -m ''","metadata":{"execution":{"iopub.status.busy":"2023-06-27T05:43:17.635574Z","iopub.status.idle":"2023-06-27T05:43:17.636436Z","shell.execute_reply.started":"2023-06-27T05:43:17.636164Z","shell.execute_reply":"2023-06-27T05:43:17.636188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}