{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## What is This Notebook\nIn this competition, you are provided with metadata such as timestamps and projection parameters to recreate satellite images.   \nThis notebook, we will use the information recorded in the metadata to visualize and verify the content.\n\nAs projection coordinates, row and column coordinates are given, and we will convert these into latitude and longitude.  \nCoordinate Reference System (CRS) information is required for the conversion, which is written in \"projection_wkt\".\n\nThe timestamp is UNIX Time. This is okay, right?","metadata":{}},{"cell_type":"code","source":"import json\nfrom datetime import datetime\nfrom tqdm.notebook import tqdm\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport random\n\nfrom PIL import Image\nimport io\nimport base64\n\nimport folium\nfrom pyproj import CRS, Transformer\nfrom geopy.distance import geodesic\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-01T13:43:38.993607Z","iopub.execute_input":"2023-08-01T13:43:38.994045Z","iopub.status.idle":"2023-08-01T13:43:39.000717Z","shell.execute_reply.started":"2023-08-01T13:43:38.994012Z","shell.execute_reply":"2023-08-01T13:43:38.99958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I'm creating a flag to indicate whether or not there are cloud labels. \n# Don't worry about it; we'll use it later.\nash_color_dir = Path(\"/kaggle/input/contrails-images-ash-color\")\ndf = (pd.concat([pd.read_csv(str(ash_color_dir / \"train_df.csv\")), pd.read_csv(str(ash_color_dir / \"valid_df.csv\"))], axis=0).reset_index(drop=True).drop(\"train\", axis=1))\ndf[\"path\"] = df.record_id.map(lambda x: ash_color_dir / \"contrails\" / \"{}.{}\".format(x, \"npy\"))\ndf[\"contrail\"] = df.path.map(lambda x: 0 if np.all(np.load(x)[..., -1] == 0) else 1)\ndf = df.set_index(\"record_id\")\ndf.index = df.index.astype(\"str\")","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-08-01T13:21:22.807803Z","iopub.execute_input":"2023-08-01T13:21:22.808157Z","iopub.status.idle":"2023-08-01T13:24:02.807044Z","shell.execute_reply.started":"2023-08-01T13:21:22.808132Z","shell.execute_reply":"2023-08-01T13:24:02.805889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### First, let's load the metafile.","metadata":{}},{"cell_type":"code","source":"metafiles = [\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train_metadata.json\", \"/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json\"]\ncombined_metadata = []\nfor file in metafiles:\n    with open(file) as f:\n        j = json.load(f)\n        combined_metadata.extend(j)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-01T13:24:02.808748Z","iopub.execute_input":"2023-08-01T13:24:02.809084Z","iopub.status.idle":"2023-08-01T13:24:02.962533Z","shell.execute_reply.started":"2023-08-01T13:24:02.809056Z","shell.execute_reply":"2023-08-01T13:24:02.960999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Next, we'll proceed with the conversion to latitude and longitude.\n\nWe will convert from the row and column information based on the CRS information written in projection_wkt.   \nThe destination of the conversion is the EPSG:4326 (WGS84) coordinate system.\n\nEPSG:4326 is a geographic coordinate system of latitude and longitude used by GPS.\nhttps://epsg.io/4326\n\nIt will take some time as there are many files.","metadata":{}},{"cell_type":"code","source":"converted_coords = []\nfor item in tqdm(combined_metadata):\n    crs = CRS.from_string(item['projection_wkt'])    \n    row = item['row_min']\n    col = item['col_min']\n    timestamp = item['timestamp']\n    \n    transformer = Transformer.from_crs(crs, CRS(\"EPSG:4326\"), always_xy=True)\n    \n    # Convert the coordinates\n    lon, lat = transformer.transform(col, row)\n    \n    # Add the converted coordinates to the list\n    converted_coords.append((item[\"record_id\"], lat, lon, timestamp))","metadata":{"execution":{"iopub.status.busy":"2023-08-01T13:38:11.678935Z","iopub.execute_input":"2023-08-01T13:38:11.679361Z","iopub.status.idle":"2023-08-01T13:41:18.530658Z","shell.execute_reply.started":"2023-08-01T13:38:11.679325Z","shell.execute_reply":"2023-08-01T13:41:18.529028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Last, Now that we have converted to latitude and longitude,\nlet's overlay them on a map. Since there's a lot of data, let's try to reduce it to just a few random ones.\n\n**Be careful, as having a large amount of data can take a long time to render and can cause the browser to become sluggish.**","metadata":{"execution":{"iopub.status.busy":"2023-07-26T08:41:08.656295Z","iopub.execute_input":"2023-07-26T08:41:08.656797Z","iopub.status.idle":"2023-07-26T08:41:08.665337Z","shell.execute_reply.started":"2023-07-26T08:41:08.656764Z","shell.execute_reply":"2023-07-26T08:41:08.663693Z"}}},{"cell_type":"code","source":"html = \"\"\"\n        <p>RecordID: {}</p>\n        <p>TimeStamp: {}</p>\n        <p>lat:{}, lon:{}</p>\n        <img src=\"data:image/png;base64,{}\">\n        \"\"\"\n\ndef convert_base64(path):\n    data = np.load(path)[..., :-1] * 255\n    image_data = Image.fromarray(data.astype(np.uint8))\n    \n    image_file = io.BytesIO()\n    image_data.save(image_file, format='PNG')\n    image_file = image_file.getvalue()\n    return base64.b64encode(image_file).decode()\n\ndef utc2datetime(d):\n    dt = datetime.utcfromtimestamp(d)\n    return dt.strftime('%m-%d-%Y %H:%M:%S')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-08-01T13:35:19.058458Z","iopub.execute_input":"2023-08-01T13:35:19.058873Z","iopub.status.idle":"2023-08-01T13:35:19.067886Z","shell.execute_reply.started":"2023-08-01T13:35:19.058843Z","shell.execute_reply":"2023-08-01T13:35:19.065138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"choiced = random.sample(converted_coords, 500)\nm = folium.Map(location=[0, -70], zoom_start=3)\n\nfor record_id, latitude,longitude, timestamp in choiced:\n    color = \"red\" if df.loc[record_id].contrail == 0 else \"blue\"\n    popup = folium.Popup(html.format(record_id, utc2datetime(timestamp), latitude,longitude, convert_base64(df.loc[record_id].path)), max_width=320)\n    folium.Marker(location=[latitude, longitude], icon=folium.Icon(color=color), popup=popup).add_to(m)\nm","metadata":{"execution":{"iopub.status.busy":"2023-08-01T13:52:10.717019Z","iopub.execute_input":"2023-08-01T13:52:10.717394Z","iopub.status.idle":"2023-08-01T13:52:49.174469Z","shell.execute_reply.started":"2023-08-01T13:52:10.717362Z","shell.execute_reply":"2023-08-01T13:52:49.172616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You might be able to gain further insights by overlaying additional information from aviation information APIs, like finding out where areas with a lot of contrails are!  \n\nhttps://ja.flightaware.com/\n    \nGood Luck!","metadata":{"execution":{"iopub.status.busy":"2023-07-26T09:37:11.554048Z","iopub.execute_input":"2023-07-26T09:37:11.55455Z","iopub.status.idle":"2023-07-26T09:37:11.563661Z","shell.execute_reply.started":"2023-07-26T09:37:11.55451Z","shell.execute_reply":"2023-07-26T09:37:11.56188Z"}}}]}