{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports (All the datasets used are public)","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:44.598966Z","iopub.execute_input":"2023-06-06T23:54:44.599551Z","iopub.status.idle":"2023-06-06T23:54:44.98551Z","shell.execute_reply.started":"2023-06-06T23:54:44.599517Z","shell.execute_reply":"2023-06-06T23:54:44.983944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class fastnumpyio:\n    def load(file):\n        file=open(file,\"rb\")\n        header = file.read(128)\n        descr = str(header[19:25], 'utf-8').replace(\"'\",\"\").replace(\" \",\"\")\n        shape = tuple(int(num) for num in str(header[60:120], 'utf-8').replace(', }', '').replace('(', '').replace(')', '').split(','))\n        datasize = np.lib.format.descr_to_dtype(descr).itemsize\n        for dimension in shape:\n            datasize *= dimension\n        return np.ndarray(shape, dtype=descr, buffer=file.read(datasize))","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:44.988087Z","iopub.execute_input":"2023-06-06T23:54:44.989485Z","iopub.status.idle":"2023-06-06T23:54:44.99733Z","shell.execute_reply.started":"2023-06-06T23:54:44.989418Z","shell.execute_reply":"2023-06-06T23:54:44.995704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_csv('/kaggle/input/train-metadata-boolean/train_metadata_with_bool_contrail.csv')\nvalid_metadata = pd.read_csv('/kaggle/input/train-metadata-boolean/valid_metadata_with_bool_contrail.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:44.998733Z","iopub.execute_input":"2023-06-06T23:54:44.999133Z","iopub.status.idle":"2023-06-06T23:54:45.026422Z","shell.execute_reply.started":"2023-06-06T23:54:44.999103Z","shell.execute_reply":"2023-06-06T23:54:45.024956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    train_image_path = '/kaggle/input/ash-rgb-58-dataset-wo-clipping/train/'\n    valid_image_path = '/kaggle/input/ash-rgb-58-dataset-wo-clipping/validation/'\n    \n    train_label_path = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/'\n    valid_label_path = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/validation/'\n    \n    train_image_file_name = '/ash_rgb_sequence_4.npy'\n    valid_image_file_name = '/ash_rgb_sequence_4.npy'\n    \n    train_label_file_name = '/human_pixel_masks.npy'\n    valid_label_file_name = '/human_pixel_masks.npy'","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:45.072922Z","iopub.execute_input":"2023-06-06T23:54:45.073301Z","iopub.status.idle":"2023-06-06T23:54:45.0794Z","shell.execute_reply.started":"2023-06-06T23:54:45.073266Z","shell.execute_reply":"2023-06-06T23:54:45.077873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_metadata = pd.read_csv('/kaggle/input/train-metadata-boolean/train_metadata_with_bool_contrail.csv')\ntrain_metadata['image_path'] = train_metadata['record_id'].apply(lambda x: Config.train_image_path + str(x) + Config.train_image_file_name)\ntrain_metadata['label_path'] = train_metadata['record_id'].apply(lambda x: Config.train_label_path + str(x) + Config.train_label_file_name)\ntrain_metadata = train_metadata.sort_values(by='record_id')\n\nvalid_metadata = pd.read_csv('/kaggle/input/train-metadata-boolean/valid_metadata_with_bool_contrail.csv')\nvalid_metadata['image_path'] = valid_metadata['record_id'].apply(lambda x: Config.valid_image_path + str(x) + Config.valid_image_file_name)\nvalid_metadata['label_path'] = valid_metadata['record_id'].apply(lambda x: Config.valid_label_path + str(x) + Config.valid_label_file_name)\nvalid_metadata = valid_metadata.sort_values(by='record_id')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:45.248167Z","iopub.execute_input":"2023-06-06T23:54:45.248519Z","iopub.status.idle":"2023-06-06T23:54:45.305312Z","shell.execute_reply.started":"2023-06-06T23:54:45.248492Z","shell.execute_reply":"2023-06-06T23:54:45.303834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_image_label(record_id, image, label):\n    img_mean = image.mean()\n    label_ones = label.sum()\n    img_std = image.std()\n    \n    print(f'Record ID: {record_id}')\n    fig, axs = plt.subplots(1, 2, figsize=(10, 5))\n\n    axs[0].imshow(image)\n    axs[0].set_title(f'Image (Mean={img_mean:.2f}, Std ={img_std:.2f})')\n\n    axs[1].imshow(label, cmap='gray')\n    axs[1].set_title(f'Label (Positive_pixels={label_ones})')\n\n    plt.suptitle(f'Record ID: {record_id}')\n    plt.tight_layout()\n    plt.show()\n    plt.close(fig)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-06T23:54:49.007582Z","iopub.execute_input":"2023-06-06T23:54:49.007966Z","iopub.status.idle":"2023-06-06T23:54:49.014556Z","shell.execute_reply.started":"2023-06-06T23:54:49.007942Z","shell.execute_reply":"2023-06-06T23:54:49.013842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Set - All images are sorted by record_id ","metadata":{}},{"cell_type":"code","source":"for index,row in train_metadata.iterrows():\n    record_id = str(row['record_id'])\n    image = fastnumpyio.load(row['image_path'])\n    label = fastnumpyio.load(row['label_path'])\n    \n    plot_image_label(record_id,image,label)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-07T00:03:04.073051Z","iopub.execute_input":"2023-06-07T00:03:04.073457Z","iopub.status.idle":"2023-06-07T00:03:10.436074Z","shell.execute_reply.started":"2023-06-07T00:03:04.073422Z","shell.execute_reply":"2023-06-07T00:03:10.435039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Validation Set - All images are sorted by record_id ","metadata":{}},{"cell_type":"code","source":"for index,row in valid_metadata.iterrows():\n    record_id = str(row['record_id'])\n    image = fastnumpyio.load(row['image_path'])\n    label = fastnumpyio.load(row['label_path'])\n    \n    plot_image_label(record_id,image,label)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-06-07T00:04:37.622719Z","iopub.execute_input":"2023-06-07T00:04:37.623174Z","iopub.status.idle":"2023-06-07T00:04:38.227643Z","shell.execute_reply.started":"2023-06-07T00:04:37.623141Z","shell.execute_reply":"2023-06-07T00:04:38.224003Z"},"trusted":true},"execution_count":null,"outputs":[]}]}