{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Basic EDA\n\nThis notebook implements the following items:\n\n\n- Compute mean/std of each false color image (full and per channel)\n- Compute number of pixels per mask (ground truth annotations)\n- Find outliers based on mean/std\n- Compute empty mask balance\n- Merge meta-data\n\nTrain data includes 20529 records, validation includes 1856 records. Total is 22385 (Train = 91.7% , Validation = 8.3%).<br>Train non-empty/empty masks balance is 55%/45%. Validation non-empty/empty masks balance is 70%/30%.<br>Train coverage is up to 16% whereas validation coverage is up to 7%.<br>One year data in both starting on 2019-04.\n\n\nSOTA available in this [OpenContrails](https://arxiv.org/pdf/2304.02122.pdf) paper.\n\nSome bands definitions:\n- 11: \"Cloud-Top Phase\" Band (8.5)\n- 14: IR Longwave Window Band (11.2)\n- 15: \"Dirty\" Longwave Window Band (12.3)\n\n","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/input/google-research-identify-contrails-reduce-global-warming/train | wc -l\n!ls /kaggle/input/google-research-identify-contrails-reduce-global-warming/validation | wc -l","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:13:29.459852Z","iopub.execute_input":"2023-06-03T20:13:29.460265Z","iopub.status.idle":"2023-06-03T20:13:32.43638Z","shell.execute_reply.started":"2023-06-03T20:13:29.460233Z","shell.execute_reply":"2023-06-03T20:13:32.434734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 100)\npd.set_option('display.max_colwidth', None)\nimport glob\nfrom tqdm.auto import tqdm\nimport seaborn as sns\nsns.set_theme(style=\"whitegrid\")\nsns.set_context(\"paper\", font_scale=1.2)\nimport matplotlib.pyplot as plt\nimport json","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:13:32.439091Z","iopub.execute_input":"2023-06-03T20:13:32.439554Z","iopub.status.idle":"2023-06-03T20:13:34.006056Z","shell.execute_reply.started":"2023-06-03T20:13:32.439511Z","shell.execute_reply":"2023-06-03T20:13:34.004726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATA_HOME = \"/kaggle/input/google-research-identify-contrails-reduce-global-warming\"\nTRAIN_HOME = os.path.join(DATA_HOME, \"train\")\nVALID_HOME = os.path.join(DATA_HOME, \"validation\")\nRECORD_ID = \"record_id\"\nIMAGE_SIZE = 256","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:13:34.014701Z","iopub.execute_input":"2023-06-03T20:13:34.015451Z","iopub.status.idle":"2023-06-03T20:13:34.022071Z","shell.execute_reply.started":"2023-06-03T20:13:34.01541Z","shell.execute_reply":"2023-06-03T20:13:34.020618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\nN_TIMES_BEFORE = 4\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\ndef false_color_image(band11, band14, band15):\n    r = normalize_range(band15 - band14, _TDIFF_BOUNDS)\n    g = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\n    b = normalize_range(band14, _T11_BOUNDS)\n    false_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)\n    return false_color\n\n\ndef load_frame_from_record(record_path, frame=N_TIMES_BEFORE):\n    # Read Image\n    human_pixel_mask, fc_image = None, None\n    band11, band14, band15 = None, None, None\n    with open(os.path.join(record_path, 'band_11.npy'), 'rb') as f:\n        band11 = np.load(f)\n    with open(os.path.join(record_path, 'band_14.npy'), 'rb') as f:\n        band14 = np.load(f)\n    with open(os.path.join(record_path, 'band_15.npy'), 'rb') as f:\n        band15 = np.load(f)\n    # Convert to false color\n    false_color = false_color_image(band11, band14, band15)                    \n    # Pick one frame\n    fc_image = false_color[..., frame]        \n    # Load mask\n    mask_path = os.path.join(record_path, 'human_pixel_masks.npy')\n    if os.path.exists(mask_path):\n        with open(mask_path, 'rb') as f:\n            human_pixel_mask = np.load(f)\n    return fc_image, human_pixel_mask\n        \n\ndef plot_image_mask(img, mask, title=\"\"):\n    fig, ax = plt.subplots(1,2,figsize=(12,5))\n    d = ax[0].imshow(img, interpolation='none')\n    d = ax[0].grid(False)\n    d = ax[0].set_title(title)\n    d = ax[1].imshow(mask, interpolation='none')\n    d = ax[1].grid(False)\n    d = ax[1].set_title(\"Total contrail pixels: %d\" % (mask.sum()))\n    plt.tight_layout()    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:13:34.024617Z","iopub.execute_input":"2023-06-03T20:13:34.025093Z","iopub.status.idle":"2023-06-03T20:13:34.042898Z","shell.execute_reply.started":"2023-06-03T20:13:34.025053Z","shell.execute_reply":"2023-06-03T20:13:34.041533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_folder(folder, total=None):\n    displayed = False\n    results = []\n    for dirname, _, filenames in tqdm(os.walk(folder), total=total):\n        record_id, fc_image_mean, fc_image_std = None, None, None\n        for filename in filenames:\n            if \".npy\" in filename:\n                record_id = dirname      \n                # Read Image\n                fc_image, human_pixel_mask = load_frame_from_record(record_id)\n                fc_image_mean = fc_image.mean()\n                fc_image_std = fc_image.std()\n                \n                fc_image_red_mean = fc_image[:,:,0].mean()\n                fc_image_red_std = fc_image[:,:,0].std()\n                fc_image_green_mean = fc_image[:,:,1].mean()\n                fc_image_green_std = fc_image[:,:,1].std()\n                fc_image_blue_mean = fc_image[:,:,2].mean()\n                fc_image_blue_std = fc_image[:,:,2].std()\n                \n                mask_sum = human_pixel_mask.astype(np.int32).sum() if human_pixel_mask is not None else None          \n        if record_id is not None:\n            record_id = record_id.replace(folder + \"/\", \"\")\n            results.append((record_id, fc_image_red_mean, fc_image_red_std, fc_image_green_mean, fc_image_green_std, fc_image_blue_mean, fc_image_blue_std, fc_image_mean, fc_image_std, mask_sum))\n            if (displayed is False) and (mask_sum > 0):\n                plot_image_mask(fc_image, human_pixel_mask, title=\"%s\" % record_id)\n                displayed = True\n        # if len(results) > 1000:\n        #     break\n    results_pd = pd.DataFrame(results, columns=[RECORD_ID, \"fc_image_red_mean\", \"fc_image_red_std\", \"fc_image_green_mean\", \"fc_image_green_std\", \"fc_image_blue_mean\", \"fc_image_blue_std\", \"fc_image_mean\", \"fc_image_std\", \"mask_sum\"])\n    return results_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:13:34.044702Z","iopub.execute_input":"2023-06-03T20:13:34.045064Z","iopub.status.idle":"2023-06-03T20:13:34.059616Z","shell.execute_reply.started":"2023-06-03T20:13:34.045034Z","shell.execute_reply":"2023-06-03T20:13:34.058202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Some weird mask\nEven if:\n- Contrails must contain at least 10 pixels\n- At some time in their life, Contrails must be at least 3x longer than they are wide\n- Contrails must either appear suddenly or enter from the sides of the image\n- Contrails should be visible in at least two image\n\nAt then end: Ground truth was determined by (generally) 4+ different labelers annotating each image. Pixels were considered a contrail when **>50%** of the labelers annotated it as such.\n\nAnd it can lead to such mask below that models with will unable to predict.","metadata":{}},{"cell_type":"code","source":"img, mask = load_frame_from_record(os.path.join(TRAIN_HOME, \"5913668717246725061\"))\nplot_image_mask(img, mask, title=\"5913668717246725061\")","metadata":{"execution":{"iopub.status.busy":"2023-06-03T20:19:02.011676Z","iopub.execute_input":"2023-06-03T20:19:02.012079Z","iopub.status.idle":"2023-06-03T20:19:02.919518Z","shell.execute_reply.started":"2023-06-03T20:19:02.01205Z","shell.execute_reply":"2023-06-03T20:19:02.918531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load training data and compute mean/std metrics.","metadata":{}},{"cell_type":"code","source":"train_pd = load_folder(TRAIN_HOME, total=20529)\ntrain_pd.to_parquet(\"train.parquet\")\ntrain_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T18:46:23.638533Z","iopub.execute_input":"2023-06-02T18:46:23.638921Z","iopub.status.idle":"2023-06-02T20:51:01.362184Z","shell.execute_reply.started":"2023-06-02T18:46:23.638888Z","shell.execute_reply":"2023-06-02T20:51:01.357947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pd.describe(percentiles=[0.01, 0.1, 0.25, 0.5, 0.55, 0.6, 0.70, 0.71, 0.725, 0.75, 0.95, 0.975, 0.99])","metadata":{"execution":{"iopub.status.busy":"2023-06-02T20:51:01.378893Z","iopub.execute_input":"2023-06-02T20:51:01.381629Z","iopub.status.idle":"2023-06-02T20:51:01.488557Z","shell.execute_reply.started":"2023-06-02T20:51:01.381501Z","shell.execute_reply":"2023-06-02T20:51:01.487336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have a look to non-empty masks","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(20, 5))\nd = sns.histplot(data=train_pd[train_pd[\"mask_sum\"] > 0], x = \"mask_sum\", ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T20:51:01.490502Z","iopub.execute_input":"2023-06-02T20:51:01.491013Z","iopub.status.idle":"2023-06-02T20:51:02.756433Z","shell.execute_reply.started":"2023-06-02T20:51:01.49097Z","shell.execute_reply":"2023-06-02T20:51:02.755152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20, 5))\nd = sns.histplot(data=train_pd, x = \"fc_image_mean\", ax=ax[0])\nd = ax[0].set_title(\"Train: FC mean\")\nd = sns.histplot(data=train_pd, x = \"fc_image_std\", ax=ax[1])\nd = ax[1].set_title(\"Train: FC std\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T20:51:02.758016Z","iopub.execute_input":"2023-06-02T20:51:02.758493Z","iopub.status.idle":"2023-06-02T20:51:03.882993Z","shell.execute_reply.started":"2023-06-02T20:51:02.758461Z","shell.execute_reply":"2023-06-02T20:51:03.881852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20, 5))\nd = sns.histplot(data=train_pd, x = \"fc_image_red_mean\", ax=ax[0], color=\"red\", alpha=0.5)\nd = sns.histplot(data=train_pd, x = \"fc_image_green_mean\", ax=ax[0], color=\"green\", alpha=0.5)\nd = sns.histplot(data=train_pd, x = \"fc_image_blue_mean\", ax=ax[0], color=\"blue\", alpha=0.5)\nd = ax[0].set_title(\"Train: FC mean by channel\")\nd = sns.histplot(data=train_pd, x = \"fc_image_red_std\", ax=ax[1], color=\"red\", alpha=0.5)\nd = sns.histplot(data=train_pd, x = \"fc_image_green_std\", ax=ax[1], color=\"green\", alpha=0.5)\nd = sns.histplot(data=train_pd, x = \"fc_image_blue_std\", ax=ax[1], color=\"blue\", alpha=0.5)\nd = ax[1].set_title(\"Train: FC std by channel\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:05:02.864022Z","iopub.execute_input":"2023-06-02T21:05:02.864499Z","iopub.status.idle":"2023-06-02T21:05:04.839518Z","shell.execute_reply.started":"2023-06-02T21:05:02.864465Z","shell.execute_reply":"2023-06-02T21:05:04.838234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Do the same for validation","metadata":{}},{"cell_type":"code","source":"valid_pd = load_folder(VALID_HOME, total=1856)\nvalid_pd.to_parquet(\"validation.parquet\")\nvalid_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T20:51:05.562668Z","iopub.execute_input":"2023-06-02T20:51:05.563054Z","iopub.status.idle":"2023-06-02T21:01:39.171461Z","shell.execute_reply.started":"2023-06-02T20:51:05.56302Z","shell.execute_reply":"2023-06-02T21:01:39.169916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pd.describe(percentiles=[0.01, 0.1, 0.25, 0.5, 0.55, 0.6, 0.70, 0.71, 0.725, 0.75, 0.95, 0.975, 0.99])","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:39.174678Z","iopub.execute_input":"2023-06-02T21:01:39.175232Z","iopub.status.idle":"2023-06-02T21:01:39.234971Z","shell.execute_reply.started":"2023-06-02T21:01:39.175189Z","shell.execute_reply":"2023-06-02T21:01:39.233671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(20, 5))\nd = sns.histplot(data=valid_pd[valid_pd[\"mask_sum\"] > 0], x = \"mask_sum\", ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:39.236557Z","iopub.execute_input":"2023-06-02T21:01:39.237473Z","iopub.status.idle":"2023-06-02T21:01:39.810384Z","shell.execute_reply.started":"2023-06-02T21:01:39.237433Z","shell.execute_reply":"2023-06-02T21:01:39.809003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20, 5))\nd = sns.histplot(data=valid_pd, x = \"fc_image_mean\", ax=ax[0])\nd = ax[0].set_title(\"Validation: FC mean\")\nd = sns.histplot(data=valid_pd, x = \"fc_image_std\", ax=ax[1])\nd = ax[1].set_title(\"Validation: FC std\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:39.812449Z","iopub.execute_input":"2023-06-02T21:01:39.812929Z","iopub.status.idle":"2023-06-02T21:01:41.012004Z","shell.execute_reply.started":"2023-06-02T21:01:39.812875Z","shell.execute_reply":"2023-06-02T21:01:41.010685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20, 5))\nd = sns.histplot(data=valid_pd, x = \"fc_image_red_mean\", ax=ax[0], color=\"red\", alpha=0.5)\nd = sns.histplot(data=valid_pd, x = \"fc_image_green_mean\", ax=ax[0], color=\"green\", alpha=0.5)\nd = sns.histplot(data=valid_pd, x = \"fc_image_blue_mean\", ax=ax[0], color=\"blue\", alpha=0.5)\nd = ax[0].set_title(\"Validation: FC mean by channel\")\nd = sns.histplot(data=valid_pd, x = \"fc_image_red_std\", ax=ax[1], color=\"red\", alpha=0.5)\nd = sns.histplot(data=valid_pd, x = \"fc_image_green_std\", ax=ax[1], color=\"green\", alpha=0.5)\nd = sns.histplot(data=valid_pd, x = \"fc_image_blue_std\", ax=ax[1], color=\"blue\", alpha=0.5)\nd = ax[1].set_title(\"Validation: FC std by channel\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:19:15.525129Z","iopub.execute_input":"2023-06-02T21:19:15.525688Z","iopub.status.idle":"2023-06-02T21:19:16.763714Z","shell.execute_reply.started":"2023-06-02T21:19:15.525646Z","shell.execute_reply":"2023-06-02T21:19:16.762516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Find some outliers based on mean/std IQR.","metadata":{}},{"cell_type":"code","source":"def find_outliers_index(col, coef=1.5):\n    q1 = col.quantile(.25)\n    q3 = col.quantile(.75)\n    IQR = q3 - q1\n    ll = q1 - (coef*IQR)\n    ul = q3 + (coef*IQR)\n    upper_outliers = col[col > ul].index.tolist()\n    lower_outliers = col[col < ll].index.tolist()\n    bad_indices = list(set(upper_outliers + lower_outliers))\n    return bad_indices\n\ndef compute_outliers(data, col):\n    bad_indexes = []\n    bad_indexes.append(find_outliers_index(data[col]))\n    bad_indexes = list(set(list(np.concatenate(bad_indexes).flat)))\n    data[\"outlier_%s\" % col] = 0\n    data.loc[bad_indexes, \"outlier_%s\" % col] = 1\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.313413Z","iopub.execute_input":"2023-06-02T21:01:42.314051Z","iopub.status.idle":"2023-06-02T21:01:42.323888Z","shell.execute_reply.started":"2023-06-02T21:01:42.314005Z","shell.execute_reply":"2023-06-02T21:01:42.321988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pd = compute_outliers(train_pd, \"fc_image_mean\")\ntrain_pd = compute_outliers(train_pd, \"fc_image_std\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.325717Z","iopub.execute_input":"2023-06-02T21:01:42.326174Z","iopub.status.idle":"2023-06-02T21:01:42.351833Z","shell.execute_reply.started":"2023-06-02T21:01:42.326137Z","shell.execute_reply":"2023-06-02T21:01:42.35049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pd[train_pd[\"outlier_fc_image_mean\"] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.355109Z","iopub.execute_input":"2023-06-02T21:01:42.355509Z","iopub.status.idle":"2023-06-02T21:01:42.380556Z","shell.execute_reply.started":"2023-06-02T21:01:42.35548Z","shell.execute_reply":"2023-06-02T21:01:42.379513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pd[train_pd[\"outlier_fc_image_std\"] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.381919Z","iopub.execute_input":"2023-06-02T21:01:42.383856Z","iopub.status.idle":"2023-06-02T21:01:42.414671Z","shell.execute_reply.started":"2023-06-02T21:01:42.38378Z","shell.execute_reply":"2023-06-02T21:01:42.413117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pd = compute_outliers(valid_pd, \"fc_image_mean\")\nvalid_pd = compute_outliers(valid_pd, \"fc_image_std\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.417035Z","iopub.execute_input":"2023-06-02T21:01:42.418608Z","iopub.status.idle":"2023-06-02T21:01:42.439778Z","shell.execute_reply.started":"2023-06-02T21:01:42.418562Z","shell.execute_reply":"2023-06-02T21:01:42.437501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pd[valid_pd[\"outlier_fc_image_mean\"] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.442938Z","iopub.execute_input":"2023-06-02T21:01:42.443622Z","iopub.status.idle":"2023-06-02T21:01:42.458991Z","shell.execute_reply.started":"2023-06-02T21:01:42.443575Z","shell.execute_reply":"2023-06-02T21:01:42.457635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_pd[valid_pd[\"outlier_fc_image_std\"] == 1]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.461216Z","iopub.execute_input":"2023-06-02T21:01:42.462675Z","iopub.status.idle":"2023-06-02T21:01:42.492777Z","shell.execute_reply.started":"2023-06-02T21:01:42.462628Z","shell.execute_reply":"2023-06-02T21:01:42.491365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for record_id in valid_pd[valid_pd[\"outlier_fc_image_std\"] == 1][RECORD_ID].unique():\n    fc_image, mask = load_frame_from_record(os.path.join(VALID_HOME, record_id))\n    plot_image_mask(fc_image, mask, title=record_id)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:42.493894Z","iopub.execute_input":"2023-06-02T21:01:42.494316Z","iopub.status.idle":"2023-06-02T21:01:49.558285Z","shell.execute_reply.started":"2023-06-02T21:01:42.494285Z","shell.execute_reply":"2023-06-02T21:01:49.557118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for record_id in valid_pd[valid_pd[\"mask_sum\"] > 3000][RECORD_ID].unique():\n    fc_image, mask = load_frame_from_record(os.path.join(VALID_HOME, record_id))\n    plot_image_mask(fc_image, mask, title=record_id)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:49.559625Z","iopub.execute_input":"2023-06-02T21:01:49.559961Z","iopub.status.idle":"2023-06-02T21:01:51.938875Z","shell.execute_reply.started":"2023-06-02T21:01:49.55993Z","shell.execute_reply":"2023-06-02T21:01:51.93745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inspect empty masks balance","metadata":{}},{"cell_type":"code","source":"train_pd[\"has_mask\"] = train_pd[\"mask_sum\"].apply(lambda x: 1 if x > 0 else 0)\nvalid_pd[\"has_mask\"] = valid_pd[\"mask_sum\"].apply(lambda x: 1 if x > 0 else 0)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:51.940594Z","iopub.execute_input":"2023-06-02T21:01:51.941052Z","iopub.status.idle":"2023-06-02T21:01:51.969823Z","shell.execute_reply.started":"2023-06-02T21:01:51.94102Z","shell.execute_reply":"2023-06-02T21:01:51.968643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20, 5))\n\nd = sns.countplot(data=train_pd, x=\"has_mask\", ax=ax[0])\ntrain_labels = [\"%d (%.1f%%)\" % (len(train_pd[train_pd[\"has_mask\"] == 0]), len(train_pd[train_pd[\"has_mask\"] == 0])*100/len(train_pd)),\n                \"%d (%.1f%%)\" % (len(train_pd[train_pd[\"has_mask\"] == 1]), len(train_pd[train_pd[\"has_mask\"] == 1])*100/len(train_pd))]\nfor i in ax[0].containers:\n    ax[0].bar_label(i, fmt='%.0f', labels=train_labels, fontsize=8)\nd = ax[0].set_title(\"Train: %d\" % len(train_pd))\n\nd = sns.countplot(data=valid_pd, x=\"has_mask\", ax=ax[1])\nvalid_labels = [\"%d (%.1f%%)\" % (len(valid_pd[valid_pd[\"has_mask\"] == 0]), len(valid_pd[valid_pd[\"has_mask\"] == 0])*100/len(valid_pd)),\n                \"%d (%.1f%%)\" % (len(valid_pd[valid_pd[\"has_mask\"] == 1]), len(valid_pd[valid_pd[\"has_mask\"] == 1])*100/len(valid_pd))]\nfor i in ax[1].containers:\n    ax[1].bar_label(i, fmt='%.0f', labels=valid_labels, fontsize=8)\nd = ax[1].set_title(\"Validation: %d\" % len(valid_pd))","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:51.971199Z","iopub.execute_input":"2023-06-02T21:01:51.971547Z","iopub.status.idle":"2023-06-02T21:01:52.870358Z","shell.execute_reply.started":"2023-06-02T21:01:51.971509Z","shell.execute_reply":"2023-06-02T21:01:52.869159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Meta-data\n\nFrom: https://www.kaggle.com/code/docxian/contrails-metadata","metadata":{}},{"cell_type":"code","source":"def read_meta_data(file):\n    with open(os.path.join(DATA_HOME, file), 'r') as f:\n        dict_valid_meta = json.load(f)\n        df_valid_meta = pd.DataFrame.from_dict(dict_valid_meta)\n        df_valid_meta['central_meridian'] = df_valid_meta.projection_wkt.apply(lambda x : x[347:351])\n        df_valid_meta.central_meridian = df_valid_meta.central_meridian.apply(lambda x : x.replace(']',''))\n        df_valid_meta.central_meridian = pd.to_numeric(df_valid_meta.central_meridian)    \n    return df_valid_meta[[RECORD_ID, \"row_min\", \"row_size\", \"col_min\", \"col_size\", \"timestamp\", \"central_meridian\"]]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:52.872456Z","iopub.execute_input":"2023-06-02T21:01:52.873048Z","iopub.status.idle":"2023-06-02T21:01:52.881858Z","shell.execute_reply.started":"2023-06-02T21:01:52.872997Z","shell.execute_reply":"2023-06-02T21:01:52.880344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_meta_pd = read_meta_data('train_metadata.json')\ntrain_meta_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:52.888776Z","iopub.execute_input":"2023-06-02T21:01:52.889239Z","iopub.status.idle":"2023-06-02T21:01:53.371753Z","shell.execute_reply.started":"2023-06-02T21:01:52.889205Z","shell.execute_reply":"2023-06-02T21:01:53.370363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(20, 5))\nd = sns.histplot(data=train_meta_pd, x = \"central_meridian\", ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:53.373351Z","iopub.execute_input":"2023-06-02T21:01:53.373672Z","iopub.status.idle":"2023-06-02T21:01:53.877322Z","shell.execute_reply.started":"2023-06-02T21:01:53.373644Z","shell.execute_reply":"2023-06-02T21:01:53.876019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_meta_pd = read_meta_data('validation_metadata.json')\nvalid_meta_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:53.878965Z","iopub.execute_input":"2023-06-02T21:01:53.880012Z","iopub.status.idle":"2023-06-02T21:01:53.94643Z","shell.execute_reply.started":"2023-06-02T21:01:53.879963Z","shell.execute_reply":"2023-06-02T21:01:53.945148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(20, 5))\nd = sns.histplot(data=valid_meta_pd, x = \"central_meridian\", ax=ax)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:53.948488Z","iopub.execute_input":"2023-06-02T21:01:53.948847Z","iopub.status.idle":"2023-06-02T21:01:54.405246Z","shell.execute_reply.started":"2023-06-02T21:01:53.948819Z","shell.execute_reply":"2023-06-02T21:01:54.402381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_pd = pd.merge(train_pd, train_meta_pd, on=RECORD_ID, how=\"left\")\ntrain_final_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:54.407304Z","iopub.execute_input":"2023-06-02T21:01:54.40773Z","iopub.status.idle":"2023-06-02T21:01:54.483385Z","shell.execute_reply.started":"2023-06-02T21:01:54.407694Z","shell.execute_reply":"2023-06-02T21:01:54.482105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_final_pd = pd.merge(valid_pd, valid_meta_pd, on=RECORD_ID, how=\"left\")\nvalid_final_pd","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:54.485104Z","iopub.execute_input":"2023-06-02T21:01:54.486296Z","iopub.status.idle":"2023-06-02T21:01:54.524801Z","shell.execute_reply.started":"2023-06-02T21:01:54.486254Z","shell.execute_reply":"2023-06-02T21:01:54.523653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_final_pd.to_parquet(\"train.parquet\")\nvalid_final_pd.to_parquet(\"validation.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:54.526636Z","iopub.execute_input":"2023-06-02T21:01:54.527121Z","iopub.status.idle":"2023-06-02T21:01:54.595868Z","shell.execute_reply.started":"2023-06-02T21:01:54.527048Z","shell.execute_reply":"2023-06-02T21:01:54.594456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_features(df):\n    # Commpute mask coverage\n    df[\"coverage\"] = np.round(df[\"mask_sum\"]*100./(IMAGE_SIZE*IMAGE_SIZE), 1)\n    # Convert timestamp\n    df[\"timestamp\"] = df[\"timestamp\"].apply(lambda x: pd.Timestamp(x, unit='s'))\n    df[\"year_month\"] = df[\"timestamp\"].dt.to_period(\"M\").astype(str)\n\napply_features(train_final_pd)\napply_features(valid_final_pd)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:54.597542Z","iopub.execute_input":"2023-06-02T21:01:54.59789Z","iopub.status.idle":"2023-06-02T21:01:54.968603Z","shell.execute_reply.started":"2023-06-02T21:01:54.597851Z","shell.execute_reply":"2023-06-02T21:01:54.967375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plot some distributions","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1,4, figsize=(30,5))\nd = sns.histplot(data=train_final_pd[train_final_pd[\"coverage\"] > 0], x=\"coverage\", kde=True, log_scale=(False, True), ax=ax[0])\nd = sns.histplot(data=train_final_pd, x=\"fc_image_mean\", kde=True, log_scale=(False, False), ax=ax[1])\nd = sns.histplot(data=train_final_pd, x=\"fc_image_std\", kde=True, log_scale=(False, False), ax=ax[1])\nd = sns.histplot(data=train_final_pd, x=\"central_meridian\", kde=True, log_scale=(False, False), ax=ax[2])\nd = sns.histplot(data=train_final_pd.sort_values([\"year_month\"]), x=\"year_month\", ax=ax[3])\nd = ax[3].tick_params(axis='x', labelrotation=45)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:54.970069Z","iopub.execute_input":"2023-06-02T21:01:54.970395Z","iopub.status.idle":"2023-06-02T21:01:58.372356Z","shell.execute_reply.started":"2023-06-02T21:01:54.970367Z","shell.execute_reply":"2023-06-02T21:01:58.371161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,4, figsize=(30,5))\nd = sns.histplot(data=valid_final_pd[valid_final_pd[\"coverage\"] > 0], x=\"coverage\", kde=True, log_scale=(False, True), ax=ax[0])\nd = sns.histplot(data=valid_final_pd, x=\"fc_image_mean\", kde=True, log_scale=(False, False), ax=ax[1])\nd = sns.histplot(data=valid_final_pd, x=\"fc_image_std\", kde=True, log_scale=(False, False), ax=ax[1])\nd = sns.histplot(data=valid_final_pd, x=\"central_meridian\", kde=True, log_scale=(False, False), ax=ax[2])\nd = sns.histplot(data=valid_final_pd.sort_values([\"year_month\"]), x=\"year_month\", ax=ax[3])\nd = ax[3].tick_params(axis='x', labelrotation=45)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:01:58.374094Z","iopub.execute_input":"2023-06-02T21:01:58.374469Z","iopub.status.idle":"2023-06-02T21:02:00.459277Z","shell.execute_reply.started":"2023-06-02T21:01:58.374436Z","shell.execute_reply":"2023-06-02T21:02:00.458048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have a look to images with highest coverage in training and validation","metadata":{}},{"cell_type":"code","source":"train_final_pd[train_final_pd[\"coverage\"] > 16]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:02:00.461038Z","iopub.execute_input":"2023-06-02T21:02:00.461502Z","iopub.status.idle":"2023-06-02T21:02:00.491339Z","shell.execute_reply.started":"2023-06-02T21:02:00.461462Z","shell.execute_reply":"2023-06-02T21:02:00.489973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_final_pd[valid_final_pd[\"coverage\"] > 7]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:02:00.49282Z","iopub.execute_input":"2023-06-02T21:02:00.493205Z","iopub.status.idle":"2023-06-02T21:02:00.51697Z","shell.execute_reply.started":"2023-06-02T21:02:00.493167Z","shell.execute_reply":"2023-06-02T21:02:00.516129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, mask =  load_frame_from_record(os.path.join(TRAIN_HOME, \"4766623185976161051\"))\nplot_image_mask(img, mask, title=\"4766623185976161051\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:02:00.518028Z","iopub.execute_input":"2023-06-02T21:02:00.518414Z","iopub.status.idle":"2023-06-02T21:02:01.588691Z","shell.execute_reply.started":"2023-06-02T21:02:00.518382Z","shell.execute_reply":"2023-06-02T21:02:01.587425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img, mask =  load_frame_from_record(os.path.join(VALID_HOME, \"552609781892851211\"))\nplot_image_mask(img, mask, title=\"552609781892851211\")","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:02:01.590548Z","iopub.execute_input":"2023-06-02T21:02:01.590892Z","iopub.status.idle":"2023-06-02T21:02:02.371469Z","shell.execute_reply.started":"2023-06-02T21:02:01.590862Z","shell.execute_reply":"2023-06-02T21:02:02.37018Z"},"trusted":true},"execution_count":null,"outputs":[]}]}