{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# packages\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-11T17:03:30.976744Z","iopub.execute_input":"2023-05-11T17:03:30.977127Z","iopub.status.idle":"2023-05-11T17:03:30.982553Z","shell.execute_reply.started":"2023-05-11T17:03:30.977096Z","shell.execute_reply":"2023-05-11T17:03:30.98128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# configs\ndefault_col_1 = 'darkblue'\ndefault_col_2 = 'darkgreen'","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:30.989622Z","iopub.execute_input":"2023-05-11T17:03:30.990069Z","iopub.status.idle":"2023-05-11T17:03:31.002456Z","shell.execute_reply.started":"2023-05-11T17:03:30.990036Z","shell.execute_reply":"2023-05-11T17:03:31.001611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"# load JSON files\nwith open('../input/google-research-identify-contrails-reduce-global-warming/train_metadata.json', 'r') as f:\n  dict_train_meta = json.load(f)\n\nwith open('../input/google-research-identify-contrails-reduce-global-warming/validation_metadata.json', 'r') as f:\n  dict_valid_meta = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.003969Z","iopub.execute_input":"2023-05-11T17:03:31.004661Z","iopub.status.idle":"2023-05-11T17:03:31.285258Z","shell.execute_reply.started":"2023-05-11T17:03:31.004629Z","shell.execute_reply":"2023-05-11T17:03:31.283962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# look at first entry (train)\ndict_train_meta[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.287187Z","iopub.execute_input":"2023-05-11T17:03:31.287564Z","iopub.status.idle":"2023-05-11T17:03:31.29735Z","shell.execute_reply.started":"2023-05-11T17:03:31.287519Z","shell.execute_reply":"2023-05-11T17:03:31.296444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert to data frame\ndf_train_meta = pd.DataFrame.from_dict(dict_train_meta)\ndf_valid_meta = pd.DataFrame.from_dict(dict_valid_meta)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.299092Z","iopub.execute_input":"2023-05-11T17:03:31.299702Z","iopub.status.idle":"2023-05-11T17:03:31.380595Z","shell.execute_reply.started":"2023-05-11T17:03:31.299659Z","shell.execute_reply":"2023-05-11T17:03:31.377721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# First Glance","metadata":{}},{"cell_type":"code","source":"# preview\ndf_train_meta.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.384399Z","iopub.execute_input":"2023-05-11T17:03:31.384778Z","iopub.status.idle":"2023-05-11T17:03:31.401016Z","shell.execute_reply.started":"2023-05-11T17:03:31.384746Z","shell.execute_reply":"2023-05-11T17:03:31.39998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure of data frame - train\ndf_train_meta.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.402291Z","iopub.execute_input":"2023-05-11T17:03:31.402637Z","iopub.status.idle":"2023-05-11T17:03:31.443433Z","shell.execute_reply.started":"2023-05-11T17:03:31.402608Z","shell.execute_reply":"2023-05-11T17:03:31.442517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# structure of data frame - valid\ndf_valid_meta.info()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.44482Z","iopub.execute_input":"2023-05-11T17:03:31.445486Z","iopub.status.idle":"2023-05-11T17:03:31.45933Z","shell.execute_reply.started":"2023-05-11T17:03:31.445454Z","shell.execute_reply":"2023-05-11T17:03:31.458393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate projection feature. The categories are only different at the parameter \"central_meridian\".","metadata":{}},{"cell_type":"code","source":"# extract central_meridian as a number\ndf_train_meta['central_meridian'] = df_train_meta.projection_wkt.apply(lambda x : x[347:351])\ndf_train_meta.central_meridian = df_train_meta.central_meridian.apply(lambda x : x.replace(']',''))\ndf_train_meta.central_meridian = pd.to_numeric(df_train_meta.central_meridian)\n\ndf_valid_meta['central_meridian'] = df_valid_meta.projection_wkt.apply(lambda x : x[347:351])\ndf_valid_meta.central_meridian = df_valid_meta.central_meridian.apply(lambda x : x.replace(']',''))\ndf_valid_meta.central_meridian = pd.to_numeric(df_valid_meta.central_meridian)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.460931Z","iopub.execute_input":"2023-05-11T17:03:31.461462Z","iopub.status.idle":"2023-05-11T17:03:31.518705Z","shell.execute_reply.started":"2023-05-11T17:03:31.461429Z","shell.execute_reply":"2023-05-11T17:03:31.517185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"# numerical features\nfeatures_num = ['row_min','row_size','col_min','col_size','timestamp','central_meridian']","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.520187Z","iopub.execute_input":"2023-05-11T17:03:31.520614Z","iopub.status.idle":"2023-05-11T17:03:31.532298Z","shell.execute_reply.started":"2023-05-11T17:03:31.520576Z","shell.execute_reply":"2023-05-11T17:03:31.530969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats - training data\ndf_train_meta[features_num].describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.533942Z","iopub.execute_input":"2023-05-11T17:03:31.534927Z","iopub.status.idle":"2023-05-11T17:03:31.593549Z","shell.execute_reply.started":"2023-05-11T17:03:31.534886Z","shell.execute_reply":"2023-05-11T17:03:31.592303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pairwise scatterplots - training data\nsns.pairplot(df_train_meta[features_num],\n             plot_kws = {'alpha' : 0.2,\n                         'color' : default_col_1},\n             diag_kws = {'color' : default_col_1})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:31.597745Z","iopub.execute_input":"2023-05-11T17:03:31.598098Z","iopub.status.idle":"2023-05-11T17:03:43.08484Z","shell.execute_reply.started":"2023-05-11T17:03:31.598068Z","shell.execute_reply":"2023-05-11T17:03:43.083872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic stats - validation data\ndf_valid_meta[features_num].describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:43.08646Z","iopub.execute_input":"2023-05-11T17:03:43.087058Z","iopub.status.idle":"2023-05-11T17:03:43.12359Z","shell.execute_reply.started":"2023-05-11T17:03:43.087022Z","shell.execute_reply":"2023-05-11T17:03:43.122143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pairwise scatterplots - validation data\nsns.pairplot(df_valid_meta[features_num],\n             plot_kws = {'alpha' : 0.2,\n                         'color' : default_col_2},\n             diag_kws = {'color' : default_col_2})\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:43.125384Z","iopub.execute_input":"2023-05-11T17:03:43.125776Z","iopub.status.idle":"2023-05-11T17:03:52.204641Z","shell.execute_reply.started":"2023-05-11T17:03:43.125743Z","shell.execute_reply":"2023-05-11T17:03:52.2036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export\ndf_train_meta.to_csv('df_train_meta.csv', index=False)\ndf_valid_meta.to_csv('df_valid_meta.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-11T17:03:52.206087Z","iopub.execute_input":"2023-05-11T17:03:52.207096Z","iopub.status.idle":"2023-05-11T17:03:52.988771Z","shell.execute_reply.started":"2023-05-11T17:03:52.207058Z","shell.execute_reply":"2023-05-11T17:03:52.98758Z"},"trusted":true},"execution_count":null,"outputs":[]}]}