{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nfrom matplotlib import animation\nimport matplotlib.pyplot as plt\nfrom IPython import display\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, Dropout, UpSampling2D, concatenate","metadata":{"_uuid":"3ddfdbda-6bea-47d1-ba9d-d4acaa379ad1","_cell_guid":"c7665dd4-f4e4-48e1-93c8-8d162f664b2c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-08-30T19:49:07.495298Z","iopub.execute_input":"2023-08-30T19:49:07.495736Z","iopub.status.idle":"2023-08-30T19:49:16.884067Z","shell.execute_reply.started":"2023-08-30T19:49:07.495696Z","shell.execute_reply":"2023-08-30T19:49:16.882556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ORGINAL","metadata":{}},{"cell_type":"code","source":"import os\nimport numpy as np\nfrom matplotlib import animation\nimport matplotlib.pyplot as plt\nfrom IPython import display","metadata":{"execution":{"iopub.status.busy":"2023-08-29T14:20:28.134916Z","iopub.execute_input":"2023-08-29T14:20:28.135326Z","iopub.status.idle":"2023-08-29T14:20:28.174325Z","shell.execute_reply.started":"2023-08-29T14:20:28.135289Z","shell.execute_reply":"2023-08-29T14:20:28.172552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '1704010292581573769'\n\nwith open(os.path.join(BASE_DIR, record_id, 'band_11.npy'), 'rb') as f:\n    band11 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_14.npy'), 'rb') as f:\n    band14 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_15.npy'), 'rb') as f:\n    band15 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-08-29T14:20:43.483991Z","iopub.execute_input":"2023-08-29T14:20:43.484435Z","iopub.status.idle":"2023-08-29T14:20:43.860808Z","shell.execute_reply.started":"2023-08-29T14:20:43.484397Z","shell.execute_reply":"2023-08-29T14:20:43.859474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nr = normalize_range(band15 - band14, _TDIFF_BOUNDS)\ng = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(band14, _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-29T14:20:45.724238Z","iopub.execute_input":"2023-08-29T14:20:45.724668Z","iopub.status.idle":"2023-08-29T14:20:45.745Z","shell.execute_reply.started":"2023-08-29T14:20:45.72463Z","shell.execute_reply":"2023-08-29T14:20:45.743473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"false_color[..., N_TIMES_BEFORE].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-29T14:22:17.64644Z","iopub.execute_input":"2023-08-29T14:22:17.646832Z","iopub.status.idle":"2023-08-29T14:22:17.655027Z","shell.execute_reply.started":"2023-08-29T14:22:17.646797Z","shell.execute_reply":"2023-08-29T14:22:17.653029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = false_color[..., N_TIMES_BEFORE]\n\nplt.figure(figsize=(18, 6))\nax = plt.subplot(1, 3, 1)\nax.imshow(img)\nax.set_title('False color image')\n\nax = plt.subplot(1, 3, 2)\nax.imshow(human_pixel_mask, interpolation='none')\nax.set_title('Ground truth contrail mask')\n\nax = plt.subplot(1, 3, 3)\nax.imshow(img)\nax.imshow(human_pixel_mask, cmap='Reds', alpha=.4, interpolation='none')\nax.set_title('Contrail mask on false color image');","metadata":{"execution":{"iopub.status.busy":"2023-08-29T14:21:03.358754Z","iopub.execute_input":"2023-08-29T14:21:03.359124Z","iopub.status.idle":"2023-08-29T14:21:04.194386Z","shell.execute_reply.started":"2023-08-29T14:21:03.359096Z","shell.execute_reply":"2023-08-29T14:21:04.193179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_DIR = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train'\nN_TIMES_BEFORE = 4\nrecord_id = '1704010292581573769'\n\nwith open(os.path.join(BASE_DIR, record_id, 'band_11.npy'), 'rb') as f:\n    band11 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_14.npy'), 'rb') as f:\n    band14 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'band_15.npy'), 'rb') as f:\n    band15 = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_pixel_masks.npy'), 'rb') as f:\n    human_pixel_mask = np.load(f)\nwith open(os.path.join(BASE_DIR, record_id, 'human_individual_masks.npy'), 'rb') as f:\n    human_individual_mask = np.load(f)","metadata":{"execution":{"iopub.status.busy":"2023-08-21T08:44:58.212255Z","iopub.execute_input":"2023-08-21T08:44:58.213402Z","iopub.status.idle":"2023-08-21T08:44:58.501853Z","shell.execute_reply.started":"2023-08-21T08:44:58.213354Z","shell.execute_reply":"2023-08-21T08:44:58.500578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"band_data = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/band_11.npy')\nhuman_individual_masks = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/human_individual_masks.npy')\nhuman_pixel_masks = np.load('/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769/human_pixel_masks.npy')","metadata":{"execution":{"iopub.status.busy":"2023-08-30T19:56:23.405111Z","iopub.execute_input":"2023-08-30T19:56:23.40713Z","iopub.status.idle":"2023-08-30T19:56:23.517418Z","shell.execute_reply.started":"2023-08-30T19:56:23.407085Z","shell.execute_reply":"2023-08-30T19:56:23.516407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"band_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-30T19:56:36.422888Z","iopub.execute_input":"2023-08-30T19:56:36.423952Z","iopub.status.idle":"2023-08-30T19:56:36.43169Z","shell.execute_reply.started":"2023-08-30T19:56:36.423911Z","shell.execute_reply":"2023-08-30T19:56:36.430772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type(band_data)","metadata":{"execution":{"iopub.status.busy":"2023-08-30T19:56:58.400407Z","iopub.execute_input":"2023-08-30T19:56:58.400783Z","iopub.status.idle":"2023-08-30T19:56:58.408531Z","shell.execute_reply.started":"2023-08-30T19:56:58.400751Z","shell.execute_reply":"2023-08-30T19:56:58.407043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"band_data[:,:,0].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-30T20:00:59.295464Z","iopub.execute_input":"2023-08-30T20:00:59.29586Z","iopub.status.idle":"2023-08-30T20:00:59.303987Z","shell.execute_reply.started":"2023-08-30T20:00:59.295829Z","shell.execute_reply":"2023-08-30T20:00:59.302701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 7, figsize=(15, 5))\n\nax[0].imshow(band_data[:,:,0])\nax[1].imshow(band_data[:,:,1])\nax[2].imshow(band_data[:,:,2])\nax[3].imshow(band_data[:,:,3])\nax[4].imshow(band_data[:,:,4])\nax[5].imshow(band_data[:,:,5])\nax[6].imshow(band_data[:,:,6])\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-08-30T20:00:09.430979Z","iopub.execute_input":"2023-08-30T20:00:09.431404Z","iopub.status.idle":"2023-08-30T20:00:10.150187Z","shell.execute_reply.started":"2023-08-30T20:00:09.431367Z","shell.execute_reply":"2023-08-30T20:00:10.149189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# Assuming n_times_before and n_times_after are constants\nn_times_before = 4\nn_times_after = 3\n\n# Extract the labeled frame from the band data\nlabeled_frames = band_data[:, :, n_times_before]\n\n# Create a figure and subplots\nfig, axes = plt.subplots(1, 3, figsize=(15, 5))\n\n# Plot the band data\naxes[0].imshow(labeled_frames)\naxes[0].set_title('Band Data')\naxes[0].axis('off')\n\n# Plot the human individual masks\naxes[1].imshow(human_individual_masks[:, :, 0, 0])  # Plotting the first labeler's mask\naxes[1].set_title('Human Individual Masks')\naxes[1].axis('off')\n\n# Plot the human pixel masks\naxes[2].imshow(human_pixel_masks[:, :, 0])\naxes[2].set_title('Human Pixel Masks')\naxes[2].axis('off')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-28T08:23:07.301526Z","iopub.execute_input":"2023-08-28T08:23:07.303434Z","iopub.status.idle":"2023-08-28T08:23:08.064247Z","shell.execute_reply.started":"2023-08-28T08:23:07.303376Z","shell.execute_reply":"2023-08-28T08:23:08.062862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeled_frames.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:34:00.723787Z","iopub.execute_input":"2023-08-23T10:34:00.724215Z","iopub.status.idle":"2023-08-23T10:34:00.731093Z","shell.execute_reply.started":"2023-08-23T10:34:00.724177Z","shell.execute_reply":"2023-08-23T10:34:00.729974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1704010292581573769'\n\nband_8_data = np.load(os.path.join(data_directory, 'band_16.npy'))\nband_8_data = (band_8_data - np.min(band_8_data)) / (np.max(band_8_data) - np.min(band_8_data))\n\nn_times_before = 4\nn_times_after = 3\nT = n_times_before + 1 + n_times_after\n\nfig, ax = plt.subplots()\n\ndef update(frame):\n    ax.clear()\n    ax.imshow(band_8_data[:, :, frame], cmap='binary_r')\n    ax.set_title(f'Frame {frame + 1}/{T}')\n    ax.axis('off')\n\nanimation = FuncAnimation(fig, update, frames=T, interval=500)\ndisplay.HTML(animation.to_jshtml())\n","metadata":{"execution":{"iopub.status.busy":"2023-08-21T09:13:33.883193Z","iopub.execute_input":"2023-08-21T09:13:33.883641Z","iopub.status.idle":"2023-08-21T09:13:35.359449Z","shell.execute_reply.started":"2023-08-21T09:13:33.883581Z","shell.execute_reply":"2023-08-21T09:13:35.358055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Combine bands into a false color image\nIn order to view contrails in GOES, we use the \"ash\" color scheme. This color scheme was originally developed for viewing volcanic ash in the atmosphere but is also useful for viewing thin cirrus, including contrails. In this color scheme, contrails appear in the image as dark blue.\n\nNote that we use a modified version of the ash color scheme here, developed by Kulik et al., which uses slightly different bands and bounds tuned for contrails.\n\nReferences:\n - Ash Color Scheme (page 7): https://eumetrain.org/sites/default/files/2020-05/RGB_recipes.pdf","metadata":{}},{"cell_type":"code","source":"_T11_BOUNDS = (243, 303)\n_CLOUD_TOP_TDIFF_BOUNDS = (-4, 5)\n_TDIFF_BOUNDS = (-4, 2)\n\ndef normalize_range(data, bounds):\n    \"\"\"Maps data to the range [0, 1].\"\"\"\n    return (data - bounds[0]) / (bounds[1] - bounds[0])\n\nr = normalize_range(band15 - band14, _TDIFF_BOUNDS)\ng = normalize_range(band14 - band11, _CLOUD_TOP_TDIFF_BOUNDS)\nb = normalize_range(band14, _T11_BOUNDS)\nfalse_color = np.clip(np.stack([r, g, b], axis=2), 0, 1)","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:54.46358Z","iopub.execute_input":"2023-08-20T13:57:54.464662Z","iopub.status.idle":"2023-08-20T13:57:54.492299Z","shell.execute_reply.started":"2023-08-20T13:57:54.464601Z","shell.execute_reply":"2023-08-20T13:57:54.491031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = false_color[..., N_TIMES_BEFORE]\n\nplt.figure(figsize=(18, 6))\nax = plt.subplot(1, 3, 1)\nax.imshow(img)\nax.set_title('False color image')\n\nax = plt.subplot(1, 3, 2)\nax.imshow(human_pixel_mask, interpolation='none')\nax.set_title('Ground truth contrail mask')\n\nax = plt.subplot(1, 3, 3)\nax.imshow(img)\nax.imshow(human_pixel_mask, cmap='Reds', alpha=.4, interpolation='none')\nax.set_title('Contrail mask on false color image');","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:54.493747Z","iopub.execute_input":"2023-08-20T13:57:54.496289Z","iopub.status.idle":"2023-08-20T13:57:55.452538Z","shell.execute_reply.started":"2023-08-20T13:57:54.496236Z","shell.execute_reply":"2023-08-20T13:57:55.451376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Individual human masks\nn = human_individual_mask.shape[-1]\nplt.figure(figsize=(16, 4))\nfor i in range(n):\n    plt.subplot(1, n, i+1)\n    plt.imshow(human_individual_mask[..., i], interpolation='none')","metadata":{"execution":{"iopub.status.busy":"2023-08-20T13:57:55.455172Z","iopub.execute_input":"2023-08-20T13:57:55.455888Z","iopub.status.idle":"2023-08-20T13:57:56.207736Z","shell.execute_reply.started":"2023-08-20T13:57:55.455844Z","shell.execute_reply":"2023-08-20T13:57:56.206676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef create_unet(input_shape, num_classes):\n    inputs = Input(shape=input_shape)\n    \n    # Contracting Path (Downsampling)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(inputs)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n    \n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(pool1)\n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(conv2)\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n    \n    # Bottleneck\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(pool2)\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(conv3)\n    \n    # Expansive Path (Upsampling)\n    up4 = UpSampling2D(size=(2, 2))(conv3)\n    up4 = Conv2D(128, 2, activation='relu', padding='same')(up4)\n    merge4 = concatenate([conv2, up4], axis=3)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(merge4)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(conv4)\n    \n    up5 = UpSampling2D(size=(2, 2))(conv4)\n    up5 = Conv2D(64, 2, activation='relu', padding='same')(up5)\n    merge5 = concatenate([conv1, up5], axis=3)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(merge5)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(conv5)\n    \n    # Output layer\n    outputs = Conv2D(2, 1, activation='softmax')(conv5)\n    \n    model = Model(inputs=inputs, outputs=outputs)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:38:57.189939Z","iopub.execute_input":"2023-08-23T09:38:57.191006Z","iopub.status.idle":"2023-08-23T09:38:57.207455Z","shell.execute_reply.started":"2023-08-23T09:38:57.190964Z","shell.execute_reply":"2023-08-23T09:38:57.206352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the paths to the numpy files\ndata_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077'\nband_08_path = os.path.join(data_directory, 'band_08.npy')\nindividual_masks_path = os.path.join(data_directory, 'human_individual_masks.npy')\npixel_masks_path = os.path.join(data_directory, 'human_pixel_masks.npy')\n\n# Load the numpy arrays\nband_08_data = np.load(band_08_path)\nindividual_masks_data = np.load(individual_masks_path)\npixel_masks_data = np.load(pixel_masks_path)\n\n# Get the shapes of the loaded numpy arrays\ninput_shape = band_08_data.shape  # Add channel dimension\nnum_classes = 2  # Number of classes (contrail and non-contrail)\n\n# Create the U-Net model\nmodel = create_unet(input_shape, num_classes)\n\n# Compile the model\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Print the model summary\nmodel.summary()\n","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:53:34.722248Z","iopub.execute_input":"2023-08-23T09:53:34.723411Z","iopub.status.idle":"2023-08-23T09:53:34.951843Z","shell.execute_reply.started":"2023-08-23T09:53:34.723363Z","shell.execute_reply":"2023-08-23T09:53:34.950981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nbatch_size = 16\nepochs = 20\n\n# Train the model\nhistory = model.fit(\n    train_data, train_masks_categorical,\n    batch_size=batch_size,\n    epochs=epochs,\n    verbose=1,\n    validation_data=(val_data, val_masks_categorical)\n)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T09:53:41.25822Z","iopub.execute_input":"2023-08-23T09:53:41.259294Z","iopub.status.idle":"2023-08-23T09:58:07.081791Z","shell.execute_reply.started":"2023-08-23T09:53:41.259251Z","shell.execute_reply":"2023-08-23T09:58:07.080519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/test'\n\n# List to store test data\ntest_data_list = []\n\n# Loop through test directories and load data\nfor dir_name in os.listdir(test_directory):\n    dir_path = os.path.join(test_directory, dir_name)\n    if os.path.isdir(dir_path):\n        test_band_08_path = os.path.join(dir_path, 'band_08.npy')\n        \n        if os.path.exists(test_band_08_path):\n            test_band_08_data = np.load(test_band_08_path)\n            test_data_list.append(test_band_08_data)\n\n# Convert list to numpy array\ntest_contrail_time_series = np.array(test_data_list)\n\n# Normalize test data\nnormalized_test_contrail_data = (test_contrail_time_series - np.min(test_contrail_time_series)) / (np.max(test_contrail_time_series) - np.min(test_contrail_time_series))\n\n# Evaluate the model on the test data\ntest_predictions = model.predict(normalized_test_contrail_data)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:27:48.778468Z","iopub.execute_input":"2023-08-23T10:27:48.778908Z","iopub.status.idle":"2023-08-23T10:27:49.850276Z","shell.execute_reply.started":"2023-08-23T10:27:48.77887Z","shell.execute_reply":"2023-08-23T10:27:49.849127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:34:30.693411Z","iopub.execute_input":"2023-08-23T10:34:30.694076Z","iopub.status.idle":"2023-08-23T10:34:30.701864Z","shell.execute_reply.started":"2023-08-23T10:34:30.69402Z","shell.execute_reply":"2023-08-23T10:34:30.700562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Prediction\",test_predictions[0].shape)\nprint(\"Band\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/band_08.npy\").shape)\nprint(\"Human Individual\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_individual_masks.npy\").shape)\nprint(\"Human pixel\",np.load(\"/kaggle/input/google-research-identify-contrails-reduce-global-warming/train/1000216489776414077/human_pixel_masks.npy\").shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:36:50.952813Z","iopub.execute_input":"2023-08-23T10:36:50.953566Z","iopub.status.idle":"2023-08-23T10:36:50.987886Z","shell.execute_reply.started":"2023-08-23T10:36:50.953524Z","shell.execute_reply":"2023-08-23T10:36:50.986658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:32:57.165941Z","iopub.execute_input":"2023-08-23T10:32:57.166735Z","iopub.status.idle":"2023-08-23T10:32:57.173942Z","shell.execute_reply.started":"2023-08-23T10:32:57.166695Z","shell.execute_reply":"2023-08-23T10:32:57.172844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the threshold value\nthreshold = 0.5  # You can adjust this threshold value based on your needs\n\n# Apply thresholding to generate binary predictions\nbinary_test_predictions = (test_predictions > threshold).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:38:44.330072Z","iopub.execute_input":"2023-08-23T10:38:44.331205Z","iopub.status.idle":"2023-08-23T10:38:44.337725Z","shell.execute_reply.started":"2023-08-23T10:38:44.331143Z","shell.execute_reply":"2023-08-23T10:38:44.336431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"binary_test_predictions[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-08-23T10:39:46.755665Z","iopub.execute_input":"2023-08-23T10:39:46.756696Z","iopub.status.idle":"2023-08-23T10:39:46.763878Z","shell.execute_reply.started":"2023-08-23T10:39:46.756653Z","shell.execute_reply":"2023-08-23T10:39:46.762768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pickle\nimport numpy as np\nimport random\nfrom tqdm.auto import tqdm # to show progress bar\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, concatenate\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:14.917201Z","iopub.execute_input":"2023-08-28T06:51:14.917939Z","iopub.status.idle":"2023-08-28T06:51:14.930747Z","shell.execute_reply.started":"2023-08-28T06:51:14.917863Z","shell.execute_reply":"2023-08-28T06:51:14.929335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Directories**","metadata":{}},{"cell_type":"markdown","source":"# Creating u-net model","metadata":{}},{"cell_type":"code","source":"def create_unet(input_shape, num_classes):\n    inputs = Input(input_shape)\n    \n    # Encoder\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(inputs)\n    conv1 = Conv2D(64, 3, activation='relu', padding='same')(conv1)\n    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)\n    \n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(pool1)\n    conv2 = Conv2D(128, 3, activation='relu', padding='same')(conv2)\n    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)\n    \n    # Middle\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(pool2)\n    conv3 = Conv2D(256, 3, activation='relu', padding='same')(conv3)\n    \n    # Decoder\n    up4 = UpSampling2D(size=(2, 2))(conv3)\n    concat4 = concatenate([conv2, up4], axis=-1)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(concat4)\n    conv4 = Conv2D(128, 3, activation='relu', padding='same')(conv4)\n    \n    up5 = UpSampling2D(size=(2, 2))(conv4)\n    concat5 = concatenate([conv1, up5], axis=-1)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(concat5)\n    conv5 = Conv2D(64, 3, activation='relu', padding='same')(conv5)\n    \n    # Output layer\n    outputs = Conv2D(1, 1, activation='sigmoid')(conv5)    \n    model = Model(inputs, outputs)\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:39.139635Z","iopub.execute_input":"2023-08-28T06:51:39.140116Z","iopub.status.idle":"2023-08-28T06:51:39.155092Z","shell.execute_reply.started":"2023-08-28T06:51:39.140073Z","shell.execute_reply":"2023-08-28T06:51:39.153403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (256, 256, 3)  # Adjust based on your data\nnum_classes = 1  # Binary classification (contrail vs. non-contrail)\n\n# Create the U-Net model\nmodel = create_unet(input_shape, num_classes)\n\n# Compile the model\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:39.350462Z","iopub.execute_input":"2023-08-28T06:51:39.351017Z","iopub.status.idle":"2023-08-28T06:51:39.590513Z","shell.execute_reply.started":"2023-08-28T06:51:39.350954Z","shell.execute_reply":"2023-08-28T06:51:39.588799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:39.592633Z","iopub.execute_input":"2023-08-28T06:51:39.593049Z","iopub.status.idle":"2023-08-28T06:51:39.670206Z","shell.execute_reply.started":"2023-08-28T06:51:39.593004Z","shell.execute_reply":"2023-08-28T06:51:39.668766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model using pre-processed data","metadata":{}},{"cell_type":"code","source":"data_dir='/kaggle/input/contrails-ash-color-dataset/contrails'\nall_files = os.listdir(data_dir)\n\ntrain_files, valid_files = train_test_split(all_files, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:47.654529Z","iopub.execute_input":"2023-08-28T06:51:47.655013Z","iopub.status.idle":"2023-08-28T06:51:47.668821Z","shell.execute_reply.started":"2023-08-28T06:51:47.654957Z","shell.execute_reply":"2023-08-28T06:51:47.66708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seperate_data(ids):\n    img = []\n    mask = []\n    for i in tqdm(ids):\n        path = '/kaggle/input/contrails-ash-color-dataset/contrails/' + str(i)\n        data = np.load(path)\n        img.append(data[:, :, :3])\n        mask.append(data[:, :, 3:])\n    return img, mask","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:49.734044Z","iopub.execute_input":"2023-08-28T06:51:49.734481Z","iopub.status.idle":"2023-08-28T06:51:49.741903Z","shell.execute_reply.started":"2023-08-28T06:51:49.734444Z","shell.execute_reply":"2023-08-28T06:51:49.74041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_contrail , train_mask = seperate_data(train_files)\nvalid_contrail , valid_mask = seperate_data(valid_files)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:51:49.931701Z","iopub.execute_input":"2023-08-28T06:51:49.93218Z","iopub.status.idle":"2023-08-28T06:52:43.98493Z","shell.execute_reply.started":"2023-08-28T06:51:49.932136Z","shell.execute_reply":"2023-08-28T06:52:43.983352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_contrail[0].shape)\nprint(train_mask[0].shape)\nprint(valid_contrail[0].shape)\nprint(valid_mask[0].shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:52:43.987511Z","iopub.execute_input":"2023-08-28T06:52:43.987974Z","iopub.status.idle":"2023-08-28T06:52:43.996666Z","shell.execute_reply.started":"2023-08-28T06:52:43.98793Z","shell.execute_reply":"2023-08-28T06:52:43.995012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_contrail =np.array(train_contrail)\ntrain_mask = np.array(train_mask)\nvalid_contrail=np.array(valid_contrail)\nvalid_mask=np.array(valid_mask)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:52:43.99852Z","iopub.execute_input":"2023-08-28T06:52:43.999577Z","iopub.status.idle":"2023-08-28T06:52:50.929235Z","shell.execute_reply.started":"2023-08-28T06:52:43.999516Z","shell.execute_reply":"2023-08-28T06:52:50.927658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.save('train_contrail.npy', train_contrail)\nnp.save('train_mask.npy', train_mask)\nnp.save('valid_contrail.npy', valid_contrail)\nnp.save('valid_mask.npy', valid_mask)","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:52:50.932443Z","iopub.execute_input":"2023-08-28T06:52:50.933139Z","iopub.status.idle":"2023-08-28T06:52:53.282424Z","shell.execute_reply.started":"2023-08-28T06:52:50.933083Z","shell.execute_reply":"2023-08-28T06:52:53.280513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the number of epochs and batch size\nepochs = 15\nbatch_size = 16\n\n# Train the model\nhistory = model.fit(train_contrail, train_mask, \n                    batch_size=batch_size, \n                    epochs=epochs, \n                    verbose=1, \n                    validation_data=(valid_contrail, valid_mask))","metadata":{"execution":{"iopub.status.busy":"2023-08-28T06:53:13.585239Z","iopub.execute_input":"2023-08-28T06:53:13.585721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the model ","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test the model","metadata":{}},{"cell_type":"code","source":"# Set the directory path for the test data\ntest_data_directory = '/kaggle/input/google-research-identify-contrails-reduce-global-warming/test'\n\n# Preprocess the test data\npreprocessed_test_data = preprocess_test_data(test_data_directory)\n\n# Print the shape of the preprocessed test data\nprint(\"Preprocessed Test Data Shape:\", preprocessed_test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T22:32:04.341724Z","iopub.execute_input":"2023-08-26T22:32:04.343222Z","iopub.status.idle":"2023-08-26T22:32:04.389703Z","shell.execute_reply.started":"2023-08-26T22:32:04.343138Z","shell.execute_reply":"2023-08-26T22:32:04.388387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_masks = model.predict(preprocessed_test_data)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T22:36:56.355434Z","iopub.execute_input":"2023-08-26T22:36:56.356449Z","iopub.status.idle":"2023-08-26T22:36:56.439202Z","shell.execute_reply.started":"2023-08-26T22:36:56.356397Z","shell.execute_reply":"2023-08-26T22:36:56.438105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.5\nbinary_predicted_masks = (predicted_masks > threshold).astype(np.uint8)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T22:36:51.38184Z","iopub.execute_input":"2023-08-26T22:36:51.382605Z","iopub.status.idle":"2023-08-26T22:36:51.388932Z","shell.execute_reply.started":"2023-08-26T22:36:51.38256Z","shell.execute_reply":"2023-08-26T22:36:51.387735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = accuracy_score(flat_ground_truth, flat_predictions)\nprecision = precision_score(flat_ground_truth, flat_predictions)\nrecall = recall_score(flat_ground_truth, flat_predictions)\nf1 = f1_score(flat_ground_truth, flat_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-08-26T22:16:28.402378Z","iopub.status.idle":"2023-08-26T22:16:28.403273Z","shell.execute_reply.started":"2023-08-26T22:16:28.402955Z","shell.execute_reply":"2023-08-26T22:16:28.402983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-08-26T22:16:28.40488Z","iopub.status.idle":"2023-08-26T22:16:28.40573Z","shell.execute_reply.started":"2023-08-26T22:16:28.40544Z","shell.execute_reply":"2023-08-26T22:16:28.405469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}